metric for heap fragmentation (#5090)

* +Esp.getHeapUnfragness()

* only in debug mode

* default value

* always enable, 64->32, light 32 integer square root, comments

* fix when debugging is disabled

* give credits

* cosmetics

* fragmentation metric updates (doc, better api, added getMaxFreeBlockSize())

* api reworked, +example

* fixe types, fix names

* coding style fix

* use astyle for example

Author: d-a-v

Diff for: cores/esp8266/Esp-frag.cpp

@@ -0,0 +1,48 @@
+/*
+ Esp.cpp - ESP8266-specific APIs
+ Copyright (c) 2015 Ivan Grokhotkov. All rights reserved.
+ This file is part of the esp8266 core for Arduino environment.
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include "umm_malloc/umm_malloc.h"
+#include "umm_malloc/umm_malloc_cfg.h"
+#include "coredecls.h"
+#include "Esp.h"
+
+void EspClass::getHeapStats(uint32_t* hfree, uint16_t* hmax, uint8_t* hfrag)
+{
+    // L2 / Euclidian norm of free block sizes.
+    // Having getFreeHeap()=sum(hole-size), fragmentation is given by
+    // 100 * (1 - sqrt(sum(hole-size²)) / sum(hole-size))
+
+    umm_info(NULL, 0);
+    uint8_t block_size = umm_block_size();
+    uint32_t fh = ummHeapInfo.freeBlocks * block_size;
+    if (hfree)
+        *hfree = fh;
+    if (hmax)
+        *hmax = ummHeapInfo.maxFreeContiguousBlocks * block_size;
+    if (hfrag)
+        *hfrag = 100 - (sqrt32(ummHeapInfo.freeSize2) * 100) / fh;
+}
+
+uint8_t EspClass::getHeapFragmentation()
+{
+    uint8_t hfrag;
+    getHeapStats(nullptr, nullptr, &hfrag);
+    return hfrag;
+}

Diff for: cores/esp8266/Esp.cpp

@@ -24,6 +24,7 @@
 #include <memory>
 #include "interrupts.h"
 #include "MD5Builder.h"
+#include "umm_malloc/umm_malloc.h"
 
 extern "C" {
 #include "user_interface.h"
@@ -171,6 +172,11 @@ uint32_t EspClass::getFreeHeap(void)
     return system_get_free_heap_size();
 }
 
+uint16_t EspClass::getMaxFreeBlockSize(void)
+{
+    return umm_max_block_size();
+}
+
 uint32_t EspClass::getChipId(void)
 {
     return system_get_chip_id();

Diff for: cores/esp8266/Esp.h

@@ -103,10 +103,13 @@ class EspClass {
         void restart();
 
         uint16_t getVcc();
-        uint32_t getFreeHeap();
-
         uint32_t getChipId();
 
+        uint32_t getFreeHeap();
+        uint16_t getMaxFreeBlockSize();
+        uint8_t getHeapFragmentation(); // in %
+        void getHeapStats(uint32_t* free = nullptr, uint16_t* max = nullptr, uint8_t* frag = nullptr);
+
         const char * getSdkVersion();
         String getCoreVersion();
         String getFullVersion();

Diff for: cores/esp8266/coredecls.h

@@ -8,6 +8,7 @@ extern "C" {
 
 // TODO: put declarations here, get rid of -Wno-implicit-function-declaration
 
+#include <stdint.h>
 #include <cont.h> // g_pcont declaration
 
 extern bool timeshift64_is_set;
@@ -18,6 +19,8 @@ void tune_timeshift64 (uint64_t now_us);
 void settimeofday_cb (void (*cb)(void));
 void disable_extra4k_at_link_time (void) __attribute__((noinline));
 
+uint32_t sqrt32 (uint32_t n);
+
 #ifdef __cplusplus
 }
 #endif

Diff for: cores/esp8266/sqrt32.c

@@ -0,0 +1,56 @@
+
+#include <coredecls.h>
+#include <stdint.h>
+
+uint32_t sqrt32 (uint32_t n)
+{
+    // http://www.codecodex.com/wiki/Calculate_an_integer_square_root#C
+    // Another very fast algorithm donated by [email protected]
+    // (note: tested across the full 32 bits range, see comment below)
+
+    // 15 iterations (c=1<<15)
+
+    unsigned int c = 0x8000;
+    unsigned int g = 0x8000;
+
+    for(;;)
+    {
+        if (g*g > n)
+            g ^= c;
+        c >>= 1;
+        if (!c)
+            return g;
+        g |= c;
+    }
+}
+
+/*
+ * tested with:
+ *
+
+#include <stdio.h>
+#include <stdint.h>
+#include <math.h>
+
+int main (void)
+{
+    for (uint32_t i = 0; ++i; )
+    {
+        uint32_t sr = sqrt32(i);
+        uint32_t ifsr = sqrt(i);
+
+        if (ifsr != sr)
+            printf("%d: i%d f%d\n", i, sr, ifsr);
+
+        if (!(i & 0xffffff))
+        {
+            printf("%i%% (0x%08x)\r", ((i >> 16) * 100) >> 16, i);
+            fflush(stdout);
+        }
+    }
+
+    printf("\n");
+}
+
+ *
+ */

Diff for: cores/esp8266/umm_malloc/umm_malloc.c

@@ -1024,6 +1024,10 @@ void ICACHE_FLASH_ATTR *umm_info( void *ptr, int force ) {
     if( UMM_NBLOCK(blockNo) & UMM_FREELIST_MASK ) {
       ++ummHeapInfo.freeEntries;
       ummHeapInfo.freeBlocks += curBlocks;
+      ummHeapInfo.freeSize2 += (unsigned int)curBlocks
+                             * (unsigned int)sizeof(umm_block)
+                             * (unsigned int)curBlocks
+                             * (unsigned int)sizeof(umm_block);
 
       if (ummHeapInfo.maxFreeContiguousBlocks < curBlocks) {
         ummHeapInfo.maxFreeContiguousBlocks = curBlocks;
@@ -1761,4 +1765,13 @@ size_t ICACHE_FLASH_ATTR umm_free_heap_size( void ) {
   return (size_t)ummHeapInfo.freeBlocks * sizeof(umm_block);
 }
 
+size_t ICACHE_FLASH_ATTR umm_max_block_size( void ) {
+  umm_info(NULL, 0);
+  return ummHeapInfo.maxFreeContiguousBlocks * sizeof(umm_block);
+}
+
+size_t ICACHE_FLASH_ATTR umm_block_size( void ) {
+  return sizeof(umm_block);
+}
+
 /* ------------------------------------------------------------------------ */

Diff for: cores/esp8266/umm_malloc/umm_malloc.h

@@ -26,6 +26,8 @@ typedef struct UMM_HEAP_INFO_t {
   unsigned short int freeBlocks;
 
   unsigned short int maxFreeContiguousBlocks;
+
+  unsigned int freeSize2;
 }
 UMM_HEAP_INFO;
 
@@ -41,6 +43,8 @@ void *umm_realloc( void *ptr, size_t size );
 void umm_free( void *ptr );
 
 size_t umm_free_heap_size( void );
+size_t umm_max_block_size( void );
+size_t umm_block_size( void );
 
 #ifdef __cplusplus
 }

Diff for: doc/faq/a02-my-esp-crashes.rst

@@ -291,9 +291,12 @@ Memory, memory, memory
    rely on exceptions for error handling, which is not available for the ESP, and in any
    case there is no access to the underlying code.
 
-   Instrumenting the code with the OOM debug option and calls to ``ESP.getFreeHeap()`` will
-   help the process of finding leaks. Now is time to re-read about the
-   `exception decoder <#exception-decoder>`__.
+   Instrumenting the code with the OOM debug option and calls to
+   ``ESP.getFreeHeap()`` / ``ESP.getHeapFragmentation()`` /
+   ``ESP.getMaxFreeBlockSize()`` will help the process of finding memory issues.
+
+   Now is time to re-read about the `exception decoder
+   <#exception-decoder>`__.
 
 
 *Some techniques for reducing memory usage*

Diff for: doc/libraries.rst

@@ -83,6 +83,10 @@ Some ESP-specific APIs related to deep sleep, RTC and flash memories are availab
 
 ``ESP.getFreeHeap()`` returns the free heap size.
 
+``ESP.getHeapFragmentation()`` returns the fragmentation metric (0% is clean, more than ~50% is not harmless)
+
+``ESP.getMaxFreeBlockSize()`` returns the maximum allocatable ram block regarding heap fragmentation
+
 ``ESP.getChipId()`` returns the ESP8266 chip ID as a 32-bit integer.
 
 ``ESP.getCoreVersion()`` returns a String containing the core version.

Diff for: libraries/esp8266/examples/HeapMetric/HeapMetric.ino

@@ -0,0 +1,79 @@
+
+// nothing else than showing heap metric usage
+// released to public domain
+
+#include <ESP8266WiFi.h>
+
+void stats(const char* what) {
+  // we could use getFreeHeap() getMaxFreeBlockSize() and getHeapFragmentation()
+  // or all at once:
+  uint32_t free;
+  uint16_t max;
+  uint8_t frag;
+  ESP.getHeapStats(&free, &max, &frag);
+
+  Serial.printf("free: %5d - max: %5d - frag: %3d%% <- ", free, max, frag);
+  // %s requires a malloc that could fail, using println instead:
+  Serial.println(what);
+}
+
+void tryit(int blocksize) {
+  void** p;
+  int blocks;
+
+  // heap-used ~= blocks*sizeof(void*) + blocks*blocksize
+  blocks = ((ESP.getMaxFreeBlockSize() / (blocksize + sizeof(void*))) + 3) & ~3; // rounded up, multiple of 4
+
+  Serial.printf("\nFilling memory with blocks of %d bytes each\n", blocksize);
+  stats("before");
+
+  p = (void**)malloc(sizeof(void*) * blocks);
+  for (int i = 0; i < blocks; i++) {
+    p[i] = malloc(blocksize);
+  }
+  stats("array and blocks allocation");
+
+  for (int i = 0; i < blocks; i += 2) {
+    if (p[i]) {
+      free(p[i]);
+    }
+    p[i] = nullptr;
+  }
+  stats("freeing every other blocks");
+
+  for (int i = 0; i < blocks; i += 4) {
+    if (p[i + 1]) {
+      free(p[i + 1]);
+    }
+    p[i + 1] = nullptr;
+  }
+  stats("freeing every other remaining blocks");
+
+  for (int i = 0; i < blocks; i++) {
+    if (p[i]) {
+      free(p[i]);
+    }
+  }
+  stats("freeing array");
+
+  free(p);
+  stats("after");
+}
+
+void setup() {
+  Serial.begin(115200);
+  WiFi.mode(WIFI_OFF);
+
+  tryit(8000);
+  tryit(4000);
+  tryit(2000);
+  tryit(1000);
+  tryit(500);
+  tryit(200);
+  tryit(100);
+  tryit(50);
+  tryit(15);
+}
+
+void loop() {
+}

Diff for: libraries/esp8266/keywords.txt

@@ -29,9 +29,12 @@ reset	KEYWORD2
 restart	KEYWORD2
 getVcc	KEYWORD2
 getFreeHeap	KEYWORD2
+getHeapFragmentation	KEYWORD2
+getMaxFreeBlockSize	KEYWORD2
 getChipId	KEYWORD2
 getSdkVersion	KEYWORD2
 getCoreVersion	KEYWORD2
+getFullVersion	KEYWORD2
 getBootVersion	KEYWORD2
 getBootMode	KEYWORD2
 getCpuFreqMHz	KEYWORD2