flash-size agnostic builds (#6690)

* flash: mapping definition by sketch at runtime depending on flash chip size and user configuration

Author: d-a-v

Diff for: boards.txt

@@ -186,6 +186,9 @@ void attachInterrupt(uint8_t pin, void (*)(void), int mode);
 void detachInterrupt(uint8_t pin);
 void attachInterruptArg(uint8_t pin, void (*)(void*), void* arg, int mode);
 
+#if FLASH_MAP_SUPPORT
+#include "flash_hal.h"
+#endif
 void preinit(void);
 void setup(void);
 void loop(void);

Diff for: cores/esp8266/Arduino.h

@@ -26,7 +26,7 @@
 #include "MD5Builder.h"
 #include "umm_malloc/umm_malloc.h"
 #include "cont.h"
-
+#include "flash_hal.h"
 #include "coredecls.h"
 #include "umm_malloc/umm_malloc.h"
 #include <pgmspace.h>
@@ -291,13 +291,17 @@ uint32_t EspClass::getFlashChipRealSize(void)
 
 uint32_t EspClass::getFlashChipSize(void)
 {
+#if FLASH_MAP_SUPPORT
+    return getFlashChipRealSize();
+#else
     uint32_t data;
     uint8_t * bytes = (uint8_t *) &data;
     // read first 4 byte (magic byte + flash config)
     if(spi_flash_read(0x0000, &data, 4) == SPI_FLASH_RESULT_OK) {
         return magicFlashChipSize((bytes[3] & 0xf0) >> 4);
     }
     return 0;
+#endif
 }
 
 uint32_t EspClass::getFlashChipSpeed(void)
@@ -323,6 +327,7 @@ FlashMode_t EspClass::getFlashChipMode(void)
     return mode;
 }
 
+#if !FLASH_MAP_SUPPORT
 uint32_t EspClass::magicFlashChipSize(uint8_t byte) {
     switch(byte & 0x0F) {
         case 0x0: // 4 Mbit (512KB)
@@ -343,6 +348,7 @@ uint32_t EspClass::magicFlashChipSize(uint8_t byte) {
             return 0;
     }
 }
+#endif
 
 uint32_t EspClass::magicFlashChipSpeed(uint8_t byte) {
     switch(byte & 0x0F) {
@@ -612,14 +618,12 @@ uint32_t EspClass::getSketchSize() {
     return result;
 }
 
-extern "C" uint32_t _FS_start;
-
 uint32_t EspClass::getFreeSketchSpace() {
 
     uint32_t usedSize = getSketchSize();
     // round one sector up
     uint32_t freeSpaceStart = (usedSize + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1));
-    uint32_t freeSpaceEnd = (uint32_t)&_FS_start - 0x40200000;
+    uint32_t freeSpaceEnd = (uint32_t)FS_start - 0x40200000;
 
 #ifdef DEBUG_SERIAL
     DEBUG_SERIAL.printf("usedSize=%u freeSpaceStart=%u freeSpaceEnd=%u\r\n", usedSize, freeSpaceStart, freeSpaceEnd);

Diff for: cores/esp8266/Esp.cpp

@@ -0,0 +1,59 @@
+
+// - do not edit - autogenerated by boards.txt.py
+
+#ifndef __FLASH_MAP_H
+#define __FLASH_MAP_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+typedef struct
+{
+    uint32_t eeprom_start;
+    uint32_t fs_start;
+    uint32_t fs_end;
+    uint32_t fs_block_size;
+    uint32_t fs_page_size;
+    uint32_t flash_size_kb;
+} flash_map_s;
+
+/*
+  Following definitions map the above structure, one per line.
+  FLASH_MAP_* is a user choice in sketch:
+      `FLASH_MAP_SETUP_CONFIG(FLASH_MAP_OTA_FS)`
+  Configuration is made at boot with detected flash chip size (last argument 512..16384)
+  Other values are defined from `tools/boards.txt.py`.
+*/
+
+#define FLASH_MAP_OTA_FS \
+    { \
+        { .eeprom_start = 0x402fb000, .fs_start = 0x402eb000, .fs_end = 0x402fb000, .fs_block_size = 0x1000, .fs_page_size = 0x100, .flash_size_kb = 1024 }, \
+        { .eeprom_start = 0x403fb000, .fs_start = 0x403c0000, .fs_end = 0x403fb000, .fs_block_size = 0x1000, .fs_page_size = 0x100, .flash_size_kb = 2048 }, \
+        { .eeprom_start = 0x405fb000, .fs_start = 0x40400000, .fs_end = 0x405fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 4096 }, \
+        { .eeprom_start = 0x409fb000, .fs_start = 0x40400000, .fs_end = 0x409fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 8192 }, \
+        { .eeprom_start = 0x411fb000, .fs_start = 0x40400000, .fs_end = 0x411fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 16384 }, \
+        { .eeprom_start = 0x4027b000, .fs_start = 0x40273000, .fs_end = 0x4027b000, .fs_block_size = 0x1000, .fs_page_size = 0x100, .flash_size_kb = 512 }, \
+    }
+
+#define FLASH_MAP_MAX_FS \
+    { \
+        { .eeprom_start = 0x402fb000, .fs_start = 0x4027b000, .fs_end = 0x402fb000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 1024 }, \
+        { .eeprom_start = 0x403fb000, .fs_start = 0x40300000, .fs_end = 0x403fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 2048 }, \
+        { .eeprom_start = 0x405fb000, .fs_start = 0x40300000, .fs_end = 0x405fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 4096 }, \
+        { .eeprom_start = 0x409fb000, .fs_start = 0x40300000, .fs_end = 0x409fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 8192 }, \
+        { .eeprom_start = 0x411fb000, .fs_start = 0x40300000, .fs_end = 0x411fa000, .fs_block_size = 0x2000, .fs_page_size = 0x100, .flash_size_kb = 16384 }, \
+        { .eeprom_start = 0x4027b000, .fs_start = 0x4025b000, .fs_end = 0x4027b000, .fs_block_size = 0x1000, .fs_page_size = 0x100, .flash_size_kb = 512 }, \
+    }
+
+#define FLASH_MAP_NO_FS \
+    { \
+        { .eeprom_start = 0x402fb000, .fs_start = 0x402fb000, .fs_end = 0x402fb000, .fs_block_size = 0x0, .fs_page_size = 0x0, .flash_size_kb = 1024 }, \
+        { .eeprom_start = 0x403fb000, .fs_start = 0x403fb000, .fs_end = 0x403fb000, .fs_block_size = 0x0, .fs_page_size = 0x0, .flash_size_kb = 2048 }, \
+        { .eeprom_start = 0x405fb000, .fs_start = 0x405fb000, .fs_end = 0x405fb000, .fs_block_size = 0x0, .fs_page_size = 0x0, .flash_size_kb = 4096 }, \
+        { .eeprom_start = 0x409fb000, .fs_start = 0x409fb000, .fs_end = 0x409fb000, .fs_block_size = 0x0, .fs_page_size = 0x0, .flash_size_kb = 8192 }, \
+        { .eeprom_start = 0x411fb000, .fs_start = 0x411fb000, .fs_end = 0x411fb000, .fs_block_size = 0x0, .fs_page_size = 0x0, .flash_size_kb = 16384 }, \
+        { .eeprom_start = 0x4027b000, .fs_start = 0x4027b000, .fs_end = 0x4027b000, .fs_block_size = 0x0, .fs_page_size = 0x0, .flash_size_kb = 512 }, \
+    }
+
+#endif // __FLASH_MAP_H
+

Diff for: cores/esp8266/FlashMap.h

@@ -24,8 +24,7 @@ extern "C" {
     #include "user_interface.h"
 }
 
-extern "C" uint32_t _FS_start;
-extern "C" uint32_t _FS_end;
+#include <flash_hal.h> // not "flash_hal.h": can use hijacked MOCK version
 
 UpdaterClass::UpdaterClass()
 {
@@ -118,7 +117,7 @@ bool UpdaterClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
 
   if (command == U_FLASH) {
     //address of the end of the space available for sketch and update
-    uintptr_t updateEndAddress = (uintptr_t)&_FS_start - 0x40200000;
+    uintptr_t updateEndAddress = FS_start - 0x40200000;
 
     updateStartAddress = (updateEndAddress > roundedSize)? (updateEndAddress - roundedSize) : 0;
 
@@ -135,14 +134,14 @@ bool UpdaterClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
     }
   }
   else if (command == U_FS) {
-    if((uintptr_t)&_FS_start + roundedSize > (uintptr_t)&_FS_end) {
+    if(FS_start + roundedSize > FS_end) {
       _setError(UPDATE_ERROR_SPACE);
       return false;
     }
 
 #ifdef ATOMIC_FS_UPDATE
     //address of the end of the space available for update
-    uintptr_t updateEndAddress = (uintptr_t)&_FS_start - 0x40200000;
+    uintptr_t updateEndAddress = FS_start - 0x40200000;
 
     updateStartAddress = (updateEndAddress > roundedSize)? (updateEndAddress - roundedSize) : 0;
 
@@ -151,7 +150,7 @@ bool UpdaterClass::begin(size_t size, int command, int ledPin, uint8_t ledOn) {
       return false;
     }
 #else
-    updateStartAddress = (uintptr_t)&_FS_start - 0x40200000;
+    updateStartAddress = FS_start - 0x40200000;
 #endif
   }
   else {
@@ -314,7 +313,7 @@ bool UpdaterClass::end(bool evenIfRemaining){
     eboot_command ebcmd;
     ebcmd.action = ACTION_COPY_RAW;
     ebcmd.args[0] = _startAddress;
-    ebcmd.args[1] = (uintptr_t)&_FS_start - 0x40200000;
+    ebcmd.args[1] = FS_start - 0x40200000;
     ebcmd.args[2] = _size;
     eboot_command_write(&ebcmd);
 #endif
@@ -460,6 +459,9 @@ bool UpdaterClass::_verifyEnd() {
             return false;
         }
 
+// it makes no sense to check flash size in auto flash mode
+// (sketch size would have to be set in bin header, instead of flash size)
+#if !FLASH_MAP_SUPPORT
         uint32_t bin_flash_size = ESP.magicFlashChipSize((buf[3] & 0xf0) >> 4);
 
         // check if new bin fits to SPI flash
@@ -468,6 +470,7 @@ bool UpdaterClass::_verifyEnd() {
             _setError(UPDATE_ERROR_NEW_FLASH_CONFIG);            
             return false;
         }
+#endif
 
         return true;
     } else if(_command == U_FS) {

Diff for: cores/esp8266/Updater.cpp

@@ -393,6 +393,12 @@ extern "C" void __disableWiFiAtBootTime (void)
     wifi_fpm_do_sleep(0xFFFFFFF);
 }
 
+#if FLASH_MAP_SUPPORT
+#include "flash_hal.h"
+extern "C" void flashinit (void);
+uint32_t __flashindex;
+#endif
+
 extern "C" void user_init(void) {
     struct rst_info *rtc_info_ptr = system_get_rst_info();
     memcpy((void *) &resetInfo, (void *) rtc_info_ptr, sizeof(resetInfo));
@@ -421,6 +427,9 @@ extern "C" void user_init(void) {
 
 #if defined(MMU_IRAM_HEAP)
     umm_init_iram();
+#endif
+#if FLASH_MAP_SUPPORT
+    flashinit();
 #endif
     preinit(); // Prior to C++ Dynamic Init (not related to above init() ). Meant to be user redefinable.
     __disableWiFiAtBootTime(); // default weak function disables WiFi

Diff for: cores/esp8266/core_esp8266_main.cpp

@@ -68,3 +68,13 @@ int32_t flash_hal_erase(uint32_t addr, uint32_t size) {
     }
     return FLASH_HAL_OK;
 }
+
+#if FLASH_MAP_SUPPORT
+
+// default weak configuration:
+FLASH_MAP_SETUP_CONFIG_ATTR(__attribute__((weak)), FLASH_MAP_OTA_FS)
+
+// can be overridden by user with:
+//FLASH_MAP_SETUP_CONFIG(FLASH_MAP_some_configuration)
+
+#endif

Diff for: cores/esp8266/flash_hal.cpp

@@ -24,18 +24,58 @@
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
-#ifdef ARDUINO
-extern "C" uint32_t _FS_start;
-extern "C" uint32_t _FS_end;
-extern "C" uint32_t _FS_page;
-extern "C" uint32_t _FS_block;
-
-#define FS_PHYS_ADDR ((uint32_t) (&_FS_start) - 0x40200000)
-#define FS_PHYS_SIZE ((uint32_t) (&_FS_end) - (uint32_t) (&_FS_start))
-#define FS_PHYS_PAGE ((uint32_t) &_FS_page)
-#define FS_PHYS_BLOCK ((uint32_t) &_FS_block)
+
+#ifdef __cplusplus
+extern "C" {
 #endif
 
+#if FLASH_MAP_SUPPORT
+#include <FlashMap.h>
+
+extern uint32_t spi_flash_get_id (void); // <user_interface.h>
+extern void flashinit(void);
+extern uint32_t __flashindex;
+extern const flash_map_s __flashdesc[];
+
+#define FLASH_MAP_SETUP_CONFIG(conf) FLASH_MAP_SETUP_CONFIG_ATTR(,conf)
+#define FLASH_MAP_SETUP_CONFIG_ATTR(attr, conf...) \
+  const flash_map_s __flashdesc[] PROGMEM = conf; \
+  void flashinit (void) attr; \
+  void flashinit (void) \
+  { \
+    uint32_t flash_chip_size_kb = 1 << (((spi_flash_get_id() >> 16) & 0xff) - 10); \
+    for (__flashindex = 0; __flashindex < sizeof(__flashdesc) / sizeof(__flashdesc[0]); __flashindex++) \
+      if (__flashdesc[__flashindex].flash_size_kb == flash_chip_size_kb) \
+        return; \
+    panic(); /* configuration not found */ \
+  }
+
+#define EEPROM_start (__flashdesc[__flashindex].eeprom_start)
+#define FS_start     (__flashdesc[__flashindex].fs_start)
+#define FS_end       (__flashdesc[__flashindex].fs_end)
+#define FS_block     (__flashdesc[__flashindex].fs_block_size)
+#define FS_page      (__flashdesc[__flashindex].fs_page_size)
+
+#else // !FLASH_MAP_SUPPORT
+
+extern uint32_t _FS_start;
+extern uint32_t _FS_end;
+extern uint32_t _FS_page;
+extern uint32_t _FS_block;
+extern uint32_t _EEPROM_start;
+#define EEPROM_start ((uint32_t)&_EEPROM_start)
+#define FS_start ((uint32_t)&_FS_start)
+#define FS_end ((uint32_t)&_FS_end)
+#define FS_page ((uint32_t)&_FS_page)
+#define FS_block ((uint32_t)&_FS_block)
+
+#endif // FLASH_MAP_SUPPORT
+
+#define FS_PHYS_ADDR ((uint32_t)FS_start - 0x40200000)
+#define FS_PHYS_SIZE ((uint32_t)(FS_end - FS_start))
+#define FS_PHYS_PAGE ((uint32_t)FS_page)
+#define FS_PHYS_BLOCK ((uint32_t)FS_block)
+
 // Return values of the following functions
 #define FLASH_HAL_OK          (0)
 #define FLASH_HAL_READ_ERROR  (-1)
@@ -46,4 +86,8 @@ extern int32_t flash_hal_write(uint32_t addr, uint32_t size, const uint8_t *src)
 extern int32_t flash_hal_erase(uint32_t addr, uint32_t size);
 extern int32_t flash_hal_read(uint32_t addr, uint32_t size, uint8_t *dst);
 
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
 #endif // !defined(flash_hal_h)

Diff for: cores/esp8266/flash_hal.h

@@ -186,3 +186,8 @@ indeed off at boot and is powered on only when starting to be used with the
 regular API.
 
 Read more at `former WiFi persistent mode <../esp8266wifi/generic-class.rst#persistent>`__.
+
+How to resolve "undefined reference to ``flashinit`'" error ?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Please read `flash layout <../filesystem.rst>`__ documentation entry.