Add a CRC32 over progmem and ESP.checkFlashCRC (#6566)

* Add a CRC32 over progmem and ESP.checkFlashCRC

Automatically embed a CRC32 of the program memory (including bootloader
but excluding any filesystems) in all images in unused space in the
bootloader block.

Add a call, ESP.checkFlashCRC() which returns false if the calculated
CRC doesn't match the one stored in the image (i.e. flash corruption).

Fixes #4165

* Add example that corrupts itself, comments

Show CRC checking catch a 1-bit error in program code by corrupting a
large array, and then return it to clean and verify the CRC matches once
again.

Add comments to the CRC check routine

Clean up pylint complaints on crc32bin.py

* Check linker script for CRC space in bootsector

Add an assertion in the eboot linker file to guarantee that we have at
least 8 bytes of unused space at the end of the boot sector to patch in
the CRC.  If not, the eboot link will fail.

* Add note about what to do if CRC check fails

Per discussion with @d-a-v.

When the CRC check fails, you could *try* to do certain things (but may
not succeed since there is known flash corruption at that point).  List
a few ideas for application authors.

* Only single, flash/ram friendly crc32() function

* Combine the CRC calc and bin generation in 1 step

Per discussion w/@mcspr, combine the CRC calculation with the binary
generation, removing the additional build step.

Author: earlephilhower

Diff for: bootloaders/eboot/eboot.ld

@@ -154,6 +154,12 @@ SECTIONS
     *(.gnu.linkonce.b.*)
     . = ALIGN (8);
     _bss_end = ABSOLUTE(.);
+    /* CRC stored in last 8 bytes */
+    ASSERT((. < 4096 - 8), "Error: No space for CRC in bootloader sector.");
+    . = _stext + 4096 - 8;
+    _crc_size = .;
+    . = . + 4;
+    _crc_val = .;
   } >iram1_0_seg :iram1_0_phdr
 
   .lit4 : ALIGN(4)

Diff for: cores/esp8266/Esp.cpp

@@ -26,6 +26,7 @@
 #include "MD5Builder.h"
 #include "umm_malloc/umm_malloc.h"
 #include "cont.h"
+#include "coredecls.h"
 
 extern "C" {
 #include "user_interface.h"
@@ -447,6 +448,25 @@ bool EspClass::checkFlashConfig(bool needsEquals) {
     return false;
 }
 
+bool EspClass::checkFlashCRC() {
+    // The CRC and total length are placed in extra space at the end of the 4K chunk
+    // of flash occupied by the bootloader.  If the bootloader grows to >4K-8 bytes,
+    // we'll need to adjust this.
+    uint32_t flashsize = *((uint32_t*)(0x40200000 + 4088)); // Start of PROGMEM plus 4K-8
+    uint32_t flashcrc = *((uint32_t*)(0x40200000 + 4092)); // Start of PROGMEM plus 4K-4
+    uint32_t z[2];
+    z[0] = z[1] = 0;
+
+    // Start the checksum
+    uint32_t crc = crc32((const void*)0x40200000, 4096-8, 0xffffffff);
+    // Pretend the 2 words of crc/len are zero to be idempotent
+    crc = crc32(z, 8, crc);
+    // Finish the CRC calculation over the rest of flash
+    crc = crc32((const void*)0x40201000, flashsize-4096, crc);
+    return crc == flashcrc;
+}
+
+
 String EspClass::getResetReason(void) {
     char buff[32];
     if (resetInfo.reason == REASON_DEFAULT_RST) { // normal startup by power on

Diff for: cores/esp8266/Esp.h

@@ -176,6 +176,8 @@ class EspClass {
 
         bool checkFlashConfig(bool needsEquals = false);
 
+        bool checkFlashCRC();
+
         bool flashEraseSector(uint32_t sector);
         bool flashWrite(uint32_t offset, uint32_t *data, size_t size);
         bool flashRead(uint32_t offset, uint32_t *data, size_t size);

Diff for: cores/esp8266/crc32.cpp

@@ -20,14 +20,15 @@
  */
 
 #include "coredecls.h"
+#include "pgmspace.h"
 
 // moved from core_esp8266_eboot_command.cpp
 uint32_t crc32 (const void* data, size_t length, uint32_t crc /*= 0xffffffff*/)
 {
     const uint8_t* ldata = (const uint8_t*)data;
     while (length--)
     {
-        uint8_t c = *ldata++;
+        uint8_t c = pgm_read_byte(ldata++);
         for (uint32_t i = 0x80; i > 0; i >>= 1)
         {
             bool bit = crc & 0x80000000;

Diff for: doc/libraries.rst

@@ -113,6 +113,8 @@ Some ESP-specific APIs related to deep sleep, RTC and flash memories are availab
 
 ``ESP.getCycleCount()`` returns the cpu instruction cycle count since start as an unsigned 32-bit. This is useful for accurate timing of very short actions like bit banging.
 
+``ESP.checkFlashCRC()`` calculates the CRC of the program memory (not including any filesystems) and compares it to the one embedded in the image.  If this call returns ``false`` then the flash has been corrupted.  At that point, you may want to consider trying to send a MQTT message, to start a re-download of the application, blink a LED in an `SOS` pattern, etc.  However, since the flash is known corrupted at this point there is no guarantee the app will be able to perform any of these operations, so in safety critical deployments an immediate shutdown to a fail-safe mode may be indicated.
+
 ``ESP.getVcc()`` may be used to measure supply voltage. ESP needs to reconfigure the ADC at startup in order for this feature to be available. Add the following line to the top of your sketch to use ``getVcc``:
 
 .. code:: cpp

Diff for: libraries/esp8266/examples/CheckFlashCRC/CheckFlashCRC.ino

@@ -0,0 +1,45 @@
+/*
+   Demonstrate CRC check passing and failing by simulating a bit flip in flash.
+   WARNING!!!  You would never want to actually do this in a real application!
+
+   Released to the Public Domain by Earle F. Philhower, III <[email protected]>
+*/
+
+extern "C" {
+#include "spi_flash.h"
+}
+// Artificially create a space in PROGMEM that fills multipe sectors so
+// we can corrupt one without crashing the system
+const int corruptme[SPI_FLASH_SEC_SIZE * 4] PROGMEM = { 0 };
+
+void setup() {
+  Serial.begin(115200);
+  Serial.printf("Starting\n");
+  Serial.printf("CRC check: %s\n", ESP.checkFlashCRC() ? "OK" : "ERROR");
+  Serial.printf("...Corrupting a portion of flash in the array...\n");
+
+  uint32_t ptr = (uint32_t)corruptme;
+  // Find a page aligned spot inside the array
+  ptr += 2 * SPI_FLASH_SEC_SIZE;
+  ptr &= ~(SPI_FLASH_SEC_SIZE - 1); // Sectoralign
+  uint32_t sector = ((((uint32_t)ptr - 0x40200000) / SPI_FLASH_SEC_SIZE));
+
+  // Create a sector with 1 bit set (i.e. fake corruption)
+  uint32_t *space = (uint32_t*)calloc(SPI_FLASH_SEC_SIZE, 1);
+  space[42] = 64;
+
+  // Write it into flash at the spot in question
+  spi_flash_erase_sector(sector);
+  spi_flash_write(sector * SPI_FLASH_SEC_SIZE, (uint32_t*)space, SPI_FLASH_SEC_SIZE);
+  Serial.printf("CRC check: %s\n", ESP.checkFlashCRC() ? "OK" : "ERROR");
+
+  Serial.printf("...Correcting the flash...\n");
+  memset(space, 0, SPI_FLASH_SEC_SIZE);
+  spi_flash_erase_sector(sector);
+  spi_flash_write(sector * SPI_FLASH_SEC_SIZE, (uint32_t*)space, SPI_FLASH_SEC_SIZE);
+  Serial.printf("CRC check: %s\n", ESP.checkFlashCRC() ? "OK" : "ERROR");
+}
+
+
+void loop() {
+}

Diff for: tools/elf2bin.py

@@ -30,6 +30,9 @@
 ffreqb = { '40': 0, '26': 1, '20': 2, '80': 15 }
 fsizeb = { '512K': 0, '256K': 1, '1M': 2, '2M': 3, '4M': 4, '8M': 8, '16M': 9 }
 
+crcsize_offset = 4088
+crcval_offset = 4092
+
 def get_elf_entry(elf, path):
     p = subprocess.Popen([path + "/xtensa-lx106-elf-readelf", '-h', elf], stdout=subprocess.PIPE, universal_newlines=True )
     lines = p.stdout.readlines()
@@ -94,6 +97,47 @@ def write_bin(out, elf, segments, to_addr, flash_mode, flash_size, flash_freq, p
             out.write(bytearray([0xaa]))
             total_size += 1
 
+def crc8266(ldata):
+    "Return the CRC of ldata using same algorithm as eboot"
+    crc = 0xffffffff
+    idx = 0
+    while idx < len(ldata):
+        byte = int(ldata[idx])
+        idx = idx + 1
+        for i in [0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01]:
+            bit = crc & 0x80000000
+            if (byte & i) != 0:
+                if bit == 0:
+                    bit = 1
+                else:
+                    bit = 0
+            crc = int(crc << 1) & 0xffffffff
+            if bit != 0:
+                crc = int(crc ^ 0x04c11db7)
+    return crc
+
+def store_word(raw, offset, val):
+    "Place a 4-byte word in 8266-dependent order in the raw image"
+    raw[offset] = val & 255
+    raw[offset + 1] = (val >> 8) & 255
+    raw[offset + 2] = (val >> 16) & 255
+    raw[offset + 3] = (val >> 24) & 255
+    return raw
+
+def add_crc(out):
+    with open(out, "rb") as binfile:
+        raw = bytearray(binfile.read())
+
+    # Zero out the spots we're going to overwrite to be idempotent
+    raw = store_word(raw, crcsize_offset, 0)
+    raw = store_word(raw, crcval_offset, 0)
+    crc = crc8266(raw)
+    raw = store_word(raw, crcsize_offset, len(raw))
+    raw = store_word(raw, crcval_offset, int(crc))
+
+    with open(out, "wb") as binfile:
+        binfile.write(raw)
+
 def main():
     parser = argparse.ArgumentParser(description='Create a BIN file from eboot.elf and Arduino sketch.elf for upload by esptool.py')
     parser.add_argument('-e', '--eboot', action='store', required=True, help='Path to the Arduino eboot.elf bootloader')
@@ -113,6 +157,9 @@ def main():
     write_bin(out, args.app, ['.irom0.text', '.text', '.text1', '.data', '.rodata'], 0, args.flash_mode, args.flash_size, args.flash_freq, args.path)
     out.close()
 
+    # Because the CRC includes both eboot and app, can only calculate it after the entire BIN generated
+    add_crc(args.out)
+
     return 0