From 02149a16d1305d0e9b11cea3c4a40a223b0e6ff8 Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Thu, 30 Nov 2023 10:15:56 +0100 Subject: [PATCH 1/7] removed functions that are not used anywhere, they are probably required for encoding --- src/decompress/lzss.cpp | 46 ----------------------------------------- 1 file changed, 46 deletions(-) diff --git a/src/decompress/lzss.cpp b/src/decompress/lzss.cpp index 8ae0917..48a57e5 100644 --- a/src/decompress/lzss.cpp +++ b/src/decompress/lzss.cpp @@ -121,52 +121,6 @@ int lzss_fgetc() LZSS FUNCTIONS **************************************************************************************/ -void putbit1(void) -{ - bit_buffer |= bit_mask; - if ((bit_mask >>= 1) == 0) { - lzss_fputc(bit_buffer); - bit_buffer = 0; bit_mask = 128; - } -} - -void putbit0(void) -{ - if ((bit_mask >>= 1) == 0) { - lzss_fputc(bit_buffer); - bit_buffer = 0; bit_mask = 128; - } -} - -void output1(int c) -{ - int mask; - - putbit1(); - mask = 256; - while (mask >>= 1) { - if (c & mask) putbit1(); - else putbit0(); - } -} - -void output2(int x, int y) -{ - int mask; - - putbit0(); - mask = N; - while (mask >>= 1) { - if (x & mask) putbit1(); - else putbit0(); - } - mask = (1 << EJ); - while (mask >>= 1) { - if (y & mask) putbit1(); - else putbit0(); - } -} - int getbit(int n) /* get n bits */ { int i, x; From 1a47bcb1f1dc76e0ec816e46c62874ddfcb02511 Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Fri, 1 Dec 2023 15:25:47 +0100 Subject: [PATCH 2/7] implementing lzss decode into a decoder class The objective of this implementation is to provide a more versatile way of handling decompression of lzss streams of data. The main reason is to allow streaming decompression --- src/decompress/lzss.cpp | 290 +++++++++++++++++++++++----------------- src/decompress/lzss.h | 86 ++++++++++++ 2 files changed, 254 insertions(+), 122 deletions(-) diff --git a/src/decompress/lzss.cpp b/src/decompress/lzss.cpp index 48a57e5..fc96a5f 100644 --- a/src/decompress/lzss.cpp +++ b/src/decompress/lzss.cpp @@ -5,159 +5,205 @@ #include "lzss.h" #include -#include /************************************************************************************** - DEFINE + GLOBAL VARIABLES **************************************************************************************/ -#define EI 11 /* typically 10..13 */ -#define EJ 4 /* typically 4..5 */ -#define P 1 /* If match length <= P then output one character */ -#define N (1 << EI) /* buffer size */ -#define F ((1 << EJ) + 1) /* lookahead buffer size */ - -#define LZSS_EOF (-1) +static FILE * update_file = nullptr; +static FILE * target_file = nullptr; -#define FPUTC_BUF_SIZE (64) -#define FGETC_BUF_SIZE (64) +static ArduinoPortentaOtaWatchdogResetFuncPointer wdog_feed_func = nullptr; +static LZSSDecoder* decoder = nullptr; /************************************************************************************** - GLOBAL VARIABLES + PUBLIC FUNCTIONS **************************************************************************************/ -static uint32_t LZSS_FILE_SIZE = 0; -static FILE * update_file = 0; -static FILE * target_file = 0; +void lzss_init( + FILE * update_file_ptr, + FILE * target_file_ptr, + uint32_t const lzss_file_size, + ArduinoPortentaOtaWatchdogResetFuncPointer wdog_feed_func_ptr) { + update_file = update_file_ptr; + target_file = target_file_ptr; + wdog_feed_func = wdog_feed_func_ptr; + + if(decoder != nullptr) { + delete decoder; + decoder = nullptr; + } + + decoder = new LZSSDecoder( + [target_file](const uint8_t c) { + fwrite(&c, 1, 1, target_file); + } + ); +} + +void lzss_flush() { + fflush(target_file); +} + +void lzss_decode() { + if(decoder == nullptr) { + return; + } + const size_t buf_size = 64; + uint8_t buffer[buf_size]; + size_t res = 0; -int bit_buffer = 0, bit_mask = 128; -unsigned char buffer[N * 2]; + do { + if(wdog_feed_func) { + wdog_feed_func(); + } + res = fread(buffer, sizeof(uint8_t), buf_size, update_file); + decoder->decompress(buffer, res); + } while(res == buf_size); +} -static char write_buf[FPUTC_BUF_SIZE]; -static size_t write_buf_num_bytes = 0; -static size_t bytes_written_fputc = 0; -static ArduinoPortentaOtaWatchdogResetFuncPointer wdog_feed_func = 0; /************************************************************************************** - PUBLIC FUNCTIONS + LZSS DECODER CLASS IMPLEMENTATION **************************************************************************************/ -void lzss_init(FILE * update_file_ptr, FILE * target_file_ptr, uint32_t const lzss_file_size, ArduinoPortentaOtaWatchdogResetFuncPointer wdog_feed_func_ptr) -{ - update_file = update_file_ptr; - target_file = target_file_ptr; - LZSS_FILE_SIZE = lzss_file_size; - wdog_feed_func = wdog_feed_func_ptr; +// get the number of bits the algorithm will try to get given the state +uint8_t LZSSDecoder::bits_required(LZSSDecoder::FSM_STATES s) { + switch(s) { + case FSM_0: + return 1; + case FSM_1: + return 8; + case FSM_2: + return EI; + case FSM_3: + return EJ; + default: + return 0; + } } -void lzss_flush() -{ - bytes_written_fputc += write_buf_num_bytes; - - if (wdog_feed_func) - wdog_feed_func(); +LZSSDecoder::LZSSDecoder(std::function getc_cbk, std::function putc_cbk) +: available(0), state(FSM_0), put_char_cbk(putc_cbk), get_char_cbk(getc_cbk) { + for (int i = 0; i < N - F; i++) buffer[i] = ' '; + r = N - F; +} - fwrite(write_buf, 1, write_buf_num_bytes, target_file); - write_buf_num_bytes = 0; +LZSSDecoder::LZSSDecoder(std::function putc_cbk) +: available(0), state(FSM_0), put_char_cbk(putc_cbk), get_char_cbk(nullptr) { + for (int i = 0; i < N - F; i++) buffer[i] = ' '; + r = N - F; } -/************************************************************************************** - PRIVATE FUNCTIONS - **************************************************************************************/ +LZSSDecoder::status LZSSDecoder::handle_state() { + LZSSDecoder::status res = IN_PROGRESS; + + int c = getbit(bits_required(this->state)); + + if(c == LZSS_BUFFER_EMPTY) { + res = NOT_COMPLETED; + } else if (c == LZSS_EOF) { + res = DONE; + this->state = FSM_EOF; + } else { + switch(this->state) { + case FSM_0: + if(c) { + this->state = FSM_1; + } else { + this->state = FSM_2; + } + break; + case FSM_1: + putc(c); + buffer[r++] = c; + r &= (N - 1); // equivalent to r = r % N when N is a power of 2 + + this->state = FSM_0; + break; + case FSM_2: + this->i = c; + this->state = FSM_3; + break; + case FSM_3: { + int j = c; + + // This is where the actual decompression takes place: we look into the local buffer for reuse + // of byte chunks. This can be improved by means of memcpy and by changing the putc function + // into a put_buf function in order to avoid buffering on the other end. + // TODO improve this section of code + for (int k = 0; k <= j + 1; k++) { + c = buffer[(this->i + k) & (N - 1)]; // equivalent to buffer[(i+k) % N] when N is a power of 2 + putc(c); + buffer[r++] = c; + r &= (N - 1); // equivalent to r = r % N + } + this->state = FSM_0; + + break; + } + case FSM_EOF: + break; + } + } -void lzss_fputc(int const c) -{ - /* Buffer the decompressed data into a buffer so - * we can perform block writes and don't need to - * write every byte singly on the flash (which - * wouldn't be possible anyway). - */ - write_buf[write_buf_num_bytes] = static_cast(c); - write_buf_num_bytes++; - - /* The write buffer is full of decompressed - * data, write it to the flash now. - */ - if (write_buf_num_bytes == FPUTC_BUF_SIZE) - lzss_flush(); + return res; } -int lzss_fgetc() -{ - static uint8_t read_buf[FGETC_BUF_SIZE]; - static size_t read_buf_pos = FGETC_BUF_SIZE; - static size_t bytes_read_fgetc = 0; - static size_t bytes_read_from_modem = 0; - - /* lzss_file_size is set within SSUBoot:main - * and contains the size of the LZSS file. Once - * all those bytes have been read its time to return - * LZSS_EOF in order to signal that the end of - * the file has been reached. - */ - if (bytes_read_fgetc == LZSS_FILE_SIZE) - return LZSS_EOF; - - /* If there is no data left to be read from the read buffer - * than read a new block and store it into the read buffer. - */ - if (read_buf_pos == FGETC_BUF_SIZE) - { - /* Read the next block from the flash memory. */ - bytes_read_from_modem += fread(read_buf, 1, FGETC_BUF_SIZE, update_file); - /* Reset the read buffer position. */ - read_buf_pos = 0; - } - - uint8_t const c = read_buf[read_buf_pos]; - read_buf_pos++; - bytes_read_fgetc++; - - return c; +LZSSDecoder::status LZSSDecoder::decompress(uint8_t* const buffer, uint32_t size) { + if(!get_char_cbk) { + this->in_buffer = buffer; + this->available += size; + } + + status res = IN_PROGRESS; + + while((res = handle_state()) == IN_PROGRESS); + + this->in_buffer = nullptr; + + return res; } -/************************************************************************************** - LZSS FUNCTIONS - **************************************************************************************/ +int LZSSDecoder::getbit(uint8_t n) { // get n bits from buffer + int x=0, c; -int getbit(int n) /* get n bits */ -{ - int i, x; - static int buf, mask = 0; - - x = 0; - for (i = 0; i < n; i++) { - if (mask == 0) { - if ((buf = lzss_fgetc()) == LZSS_EOF) return LZSS_EOF; - mask = 128; + // if the local bit buffer doesn't have enough bit get them + while(buf_size < n) { + switch(c=getc()) { + case LZSS_EOF: + case LZSS_BUFFER_EMPTY: + return c; } - x <<= 1; - if (buf & mask) x++; - mask >>= 1; + buf <<= 8; + + buf |= (uint8_t)c; + buf_size += sizeof(uint8_t)*8; } + + // the result is the content of the buffer starting from msb to n successive bits + x = buf >> (buf_size-n); + + // remove from the buffer the read bits with a mask + buf &= (1<<(buf_size-n))-1; + + buf_size-=n; + return x; } -void lzss_decode(void) -{ - int i, j, k, r, c; - - for (i = 0; i < N - F; i++) buffer[i] = ' '; - r = N - F; - while ((c = getbit(1)) != LZSS_EOF) { - if (c) { - if ((c = getbit(8)) == LZSS_EOF) break; - lzss_fputc(c); - buffer[r++] = c; r &= (N - 1); - } else { - if ((i = getbit(EI)) == LZSS_EOF) break; - if ((j = getbit(EJ)) == LZSS_EOF) break; - for (k = 0; k <= j + 1; k++) { - c = buffer[(i + k) & (N - 1)]; - lzss_fputc(c); - buffer[r++] = c; r &= (N - 1); - } - } +int LZSSDecoder::getc() { + int c; + + if(get_char_cbk) { + c = get_char_cbk(); + } else if(in_buffer == nullptr || available == 0) { + c = LZSS_BUFFER_EMPTY; + } else { + c = *in_buffer; + in_buffer++; + available--; } + return c; } diff --git a/src/decompress/lzss.h b/src/decompress/lzss.h index 0907600..4f56a79 100644 --- a/src/decompress/lzss.h +++ b/src/decompress/lzss.h @@ -17,4 +17,90 @@ void lzss_init(FILE * update_file_ptr, FILE * target_file_ptr, uint32_t const lz void lzss_decode(); void lzss_flush(); +/************************************************************************************** + LZSS DECODER CLASS + **************************************************************************************/ + + +class LZSSDecoder { +public: + + /** + * Build an LZSS decoder by providing a callback for storing the decoded bytes + * @param putc_cbk: a callback that takes a char and stores it e.g. a callback to fwrite + */ + LZSSDecoder(std::function putc_cbk); + + /** + * Build an LZSS decoder providing a callback for getting a char and putting a char + * in this way you need to call decompress with no parameters + * @param putc_cbk: a callback that takes a char and stores it e.g. a callback to fwrite + * @param getc_cbk: a callback that returns the next char to consume + * -1 means EOF, -2 means buffer is temporairly finished + */ + LZSSDecoder(std::function getc_cbk, std::function putc_cbk); + + /** + * this enum describes the result of the computation of a single FSM computation + * DONE: the decompression is completed + * IN_PROGRESS: the decompression cycle completed successfully, ready to compute next + * NOT_COMPLETED: the current cycle didn't complete because the available data is not enough + */ + enum status: uint8_t { + DONE, + IN_PROGRESS, + NOT_COMPLETED + }; + + /** + * decode the provided buffer until buffer ends, then pause the process + * @return DONE if the decompression is completed, NOT_COMPLETED if not + */ + status decompress(uint8_t* const buffer=nullptr, uint32_t size=0); + + static const int LZSS_EOF = -1; + static const int LZSS_BUFFER_EMPTY = -2; +private: + // TODO provide a way for the user to set these parameters + static const int EI = 11; /* typically 10..13 */ + static const int EJ = 4; /* typically 4..5 */ + static const int N = (1 << EI); /* buffer size */ + static const int F = ((1 << EJ) + 1); /* lookahead buffer size */ + + // algorithm specific buffer used to store text that could be later referenced and copied + uint8_t buffer[N * 2]; + + // this function gets 1 single char from the input buffer + int getc(); + uint8_t* in_buffer = nullptr; + uint32_t available = 0; + + status handle_state(); + + // get 1 bit from the available input buffer + int getbit(uint8_t n); + // the following 2 are variables used by getbits + uint32_t buf, buf_size=0; + + enum FSM_STATES: uint8_t { + FSM_0 = 0, + FSM_1 = 1, + FSM_2 = 2, + FSM_3 = 3, + FSM_EOF + } state; + + // these variable are used in a decode session and specific to the old C implementation + // there is no documentation about their meaning + int i, r; + + std::function put_char_cbk; + std::function get_char_cbk; + + inline void putc(const uint8_t c) { if(put_char_cbk) { put_char_cbk(c); } } + + // get the number of bits the FSM will require given its state + uint8_t bits_required(FSM_STATES s); +}; + #endif /* SSU_LZSS_H_ */ From f35648ad6c95bb723c2a697f19aa258d886f541d Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Fri, 1 Dec 2023 15:27:37 +0100 Subject: [PATCH 3/7] Added an example for using the LZSSDecoder class with comparison wrt old implementation --- examples/LZSS/LZSS.ino | 187 ++++++++++++++++++++++++++++++++ examples/LZSS/arduino_secrets.h | 2 + 2 files changed, 189 insertions(+) create mode 100644 examples/LZSS/LZSS.ino create mode 100644 examples/LZSS/arduino_secrets.h diff --git a/examples/LZSS/LZSS.ino b/examples/LZSS/LZSS.ino new file mode 100644 index 0000000..02fd0bd --- /dev/null +++ b/examples/LZSS/LZSS.ino @@ -0,0 +1,187 @@ +/* + * This example demonstrates how to download a lzss file and decompress it in two ways: + * -1 download the file on the filesystem and then decompress the downloaded file on the filesystem + * -2 download and decompress the file on the fly + * this sketch also provides a comparison in terms of speed and execution time + * + */ + +/****************************************************************************** + * INCLUDE + ******************************************************************************/ + +#include + +#include + +#include "arduino_secrets.h" +#include + +/****************************************************************************** + * CONSTANT + ******************************************************************************/ + +/* Please enter your sensitive data in the Secret tab/arduino_secrets.h */ +static char const SSID[] = SECRET_SSID; /* your network SSID (name) */ +static char const PASS[] = SECRET_PASS; /* your network password (use for WPA, or use as key for WEP) */ + + +#if defined(ARDUINO_NICLA_VISION) +static char const URL_FILE[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.NICLA_VISION.ota"; +#elif defined(ARDUINO_PORTENTA_H7_M7) +static char const URL_FILE[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.PORTENTA_H7_M7.ota"; +#elif defined(ARDUINO_OPTA) +static char const URL_FILE[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.OPTA.ota"; +#elif defined(ARDUINO_GIGA) +static char const URL_FILE[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.GIGA.ota"; +#else +#error "Board not supported" +#endif + +static char const DOWNLOAD_DESTINATION[] = "/fs/UPDATE.BIN.LZSS"; +static char const DECOMPRESSED_DESTINATION[] = "/fs/UPDATE.BIN"; + +LZSSDecoder *decoder = nullptr; +FILE* download_target = nullptr; + +/****************************************************************************** + * SETUP/LOOP + ******************************************************************************/ +void decompress_on_the_fly_cbk(const char*, uint32_t); +void putc_file(const uint8_t c); + +void setup() { + Serial.begin(115200); + while (!Serial) {} + + if (WiFi.status() == WL_NO_SHIELD) + { + Serial.println("Communication with WiFi module failed!"); + return; + } + + int status = WL_IDLE_STATUS; + while (status != WL_CONNECTED) + { + Serial.print ("Attempting to connect to '"); + Serial.print (SSID); + Serial.println("'"); + status = WiFi.begin(SSID, PASS); + if(status != WL_CONNECTED) { + delay(10000); + } + } + Serial.print ("You're connected to '"); + Serial.print (WiFi.SSID()); + Serial.println("'"); + + // Init fs + mbed::BlockDevice * _bd_raw_qspi = mbed::BlockDevice::get_default_instance();; + auto _bd_qspi = new mbed::MBRBlockDevice(_bd_raw_qspi, 2); + auto _fs_qspi = new mbed::FATFileSystem("fs"); + int const err_mount = _fs_qspi->mount(_bd_qspi); + if (err_mount) { + Serial.print("Error while mounting the filesystem. Err = "); + Serial.println(err_mount); + return; + } + + MbedSocketClass * socket = static_cast(&WiFi); + remove(DOWNLOAD_DESTINATION); + remove(DECOMPRESSED_DESTINATION); + + uint32_t start; + int bytes; + float elapsed, speed; + start = millis(); + Serial.println("Starting download to QSPI ..."); + bytes = socket->download(URL_FILE, DOWNLOAD_DESTINATION, true /* is_https */); + if (bytes <= 0) + { + Serial.print ("MbedSocketClass::download failed with error code "); + Serial.println(bytes); + return; + } + Serial.print (bytes); + Serial.println(" bytes stored."); + + elapsed = (millis()-start)/1000.0; // elapsed expressed in seconds + speed = (bytes/elapsed)/1024; + + Serial.print("download elapsed "); + Serial.print(elapsed); + Serial.print("s speed: "); + Serial.print(speed); + Serial.println("KBps"); + + FILE* downloaded_file = fopen(DOWNLOAD_DESTINATION, "rb"); + FILE* decompressed_file = fopen(DECOMPRESSED_DESTINATION, "wb"); + + start = millis(); + lzss_init(downloaded_file, decompressed_file, bytes, nullptr); + + lzss_decode(); + /* Write the data remaining in the write buffer to + * the file. + */ + lzss_flush(); + + elapsed = (millis()-start)/1000.0; // elapsed expressed in seconds + + Serial.print("decompress elapsed "); + Serial.print(elapsed); + Serial.print("s"); + Serial.print(" size "); + Serial.println(ftell(decompressed_file)); + + fclose(downloaded_file); + fclose(decompressed_file); + + // On the fly decompression + remove(DOWNLOAD_DESTINATION); + remove(DECOMPRESSED_DESTINATION); + + download_target = fopen(DECOMPRESSED_DESTINATION, "wb"); + decoder = new LZSSDecoder(putc_file); + + Serial.println("Starting download & decompress on the fly"); + start = millis(); + bytes = socket->download(URL_FILE, true /* is_https */, decompress_on_the_fly_cbk); + if (bytes <= 0) + { + Serial.print ("MbedSocketClass::download failed with error code "); + Serial.println(bytes); + return; + } + + Serial.print("downloaded "); + Serial.print(bytes); + Serial.print(" bytes "); + + elapsed = (millis()-start)/1000.0; // elapsed expressed in seconds + speed = (bytes/elapsed)/1024; + + Serial.print (ftell(download_target)); + Serial.println(" bytes stored."); + + Serial.print("download elapsed "); + Serial.print(elapsed); + Serial.print("s speed: "); + Serial.print(speed); + Serial.println("KBps"); + + delete decoder; + fclose(download_target); +} + +void loop() { +} + +void decompress_on_the_fly_cbk(const char* buffer, uint32_t size) { + decoder->decompress((uint8_t*)buffer, size); +} + +void putc_file(const uint8_t c) { + fwrite(&c, 1, 1, download_target); +} + diff --git a/examples/LZSS/arduino_secrets.h b/examples/LZSS/arduino_secrets.h new file mode 100644 index 0000000..0c9fdd5 --- /dev/null +++ b/examples/LZSS/arduino_secrets.h @@ -0,0 +1,2 @@ +#define SECRET_SSID "" +#define SECRET_PASS "" From 16037f8280b4d12c44fb83306ce097a8297117e6 Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Mon, 4 Dec 2023 15:54:40 +0100 Subject: [PATCH 4/7] added new error code --- src/Arduino_Portenta_OTA.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Arduino_Portenta_OTA.h b/src/Arduino_Portenta_OTA.h index 8a1a524..6d5b0f0 100644 --- a/src/Arduino_Portenta_OTA.h +++ b/src/Arduino_Portenta_OTA.h @@ -86,6 +86,7 @@ class Arduino_Portenta_OTA OtaHeaterMagicNumber = -7, CaStorageInit = -8, CaStorageOpen = -9, + OtaDownload = -12, }; Arduino_Portenta_OTA(StorageTypePortenta const storage_type, uint32_t const data_offset); From 20855e80c88c65c1667f312f28a06ac980820c4d Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Mon, 4 Dec 2023 15:55:56 +0100 Subject: [PATCH 5/7] added downloadAndDecompress function --- src/Arduino_Portenta_OTA.h | 2 + src/decompress/utility.cpp | 169 ++++++++++++++++++++++++++++++------- 2 files changed, 141 insertions(+), 30 deletions(-) diff --git a/src/Arduino_Portenta_OTA.h b/src/Arduino_Portenta_OTA.h index 6d5b0f0..0724e3e 100644 --- a/src/Arduino_Portenta_OTA.h +++ b/src/Arduino_Portenta_OTA.h @@ -103,6 +103,8 @@ class Arduino_Portenta_OTA */ int download(const char * url, bool const is_https, MbedSocketClass * socket = static_cast(&WiFi)); int decompress(); + int downloadAndDecompress(const char * url, bool const is_https, MbedSocketClass * socket = static_cast(&WiFi)); + void setFeedWatchdogFunc(ArduinoPortentaOtaWatchdogResetFuncPointer func); void feedWatchdog(); diff --git a/src/decompress/utility.cpp b/src/decompress/utility.cpp index 4b2449b..2640924 100644 --- a/src/decompress/utility.cpp +++ b/src/decompress/utility.cpp @@ -89,11 +89,149 @@ uint32_t crc_update(uint32_t crc, const void * data, size_t data_len) MAIN **************************************************************************************/ +union HeaderVersion +{ + struct __attribute__((packed)) + { + uint32_t header_version : 6; + uint32_t compression : 1; + uint32_t signature : 1; + uint32_t spare : 4; + uint32_t payload_target : 4; + uint32_t payload_major : 8; + uint32_t payload_minor : 8; + uint32_t payload_patch : 8; + uint32_t payload_build_num : 24; + } field; + uint8_t buf[sizeof(field)]; + static_assert(sizeof(buf) == 8, "Error: sizeof(HEADER.VERSION) != 8"); +}; + +union OTAHeader +{ + struct __attribute__((packed)) + { + uint32_t len; + uint32_t crc32; + uint32_t magic_number; + HeaderVersion hdr_version; + } header; + uint8_t buf[sizeof(header)]; + static_assert(sizeof(buf) == 20, "Error: sizeof(HEADER) != 20"); +}; + int Arduino_Portenta_OTA::download(const char * url, bool const is_https, MbedSocketClass * socket) { return socket->download((char *)url, UPDATE_FILE_NAME_LZSS, is_https); } +int Arduino_Portenta_OTA::downloadAndDecompress(const char * url, bool const is_https, MbedSocketClass * socket) { + int res=0; + + FILE* decompressed = fopen(UPDATE_FILE_NAME, "wb"); + OTAHeader ota_header; + + LZSSDecoder decoder([&decompressed](const uint8_t c) { + fwrite(&c, 1, 1, decompressed); + }); + + enum OTA_DOWNLOAD_STATE: uint8_t { + OTA_DOWNLOAD_HEADER=0, + OTA_DOWNLOAD_FILE, + OTA_DOWNLOAD_ERR + }; + + // since mbed::Callback requires a function to not exceed a certain size, we group the following parameters in a struct + struct { + uint32_t crc32 = 0xFFFFFFFF; + uint32_t header_copied_bytes = 0; + OTA_DOWNLOAD_STATE state=OTA_DOWNLOAD_HEADER; + } ota_progress; + + int bytes = socket->download(url, is_https, [&decoder, &ota_header, &ota_progress](const char* buffer, uint32_t size) { + for(char* cursor=(char*)buffer; cursor(Error::OtaDownload); + goto exit; + } + + if(ota_header.header.len == (bytes-sizeof(ota_header.buf))) { + res = static_cast(Error::OtaHeaderLength); + goto exit; + } + + // verify magic number: it may be done in the download function and stop the download immediately + if(ota_header.header.magic_number != ARDUINO_PORTENTA_OTA_MAGIC) { + res = static_cast(Error::OtaHeaterMagicNumber); + goto exit; + } + + // finalize CRC and verify it + ota_progress.crc32 ^= 0xFFFFFFFF; + if(ota_header.header.crc32 != ota_progress.crc32) { + res = static_cast(Error::OtaHeaderCrc); + goto exit; + } + + res = ftell(decompressed); + +exit: + fclose(decompressed); + + if(res < 0) { + remove(UPDATE_FILE_NAME); + } + + return res; +} + + int Arduino_Portenta_OTA::decompress() { struct stat stat_buf; @@ -103,36 +241,7 @@ int Arduino_Portenta_OTA::decompress() /* For UPDATE.BIN.LZSS - LZSS compressed binary files. */ FILE* update_file = fopen(UPDATE_FILE_NAME_LZSS, "rb"); - union HeaderVersion - { - struct __attribute__((packed)) - { - uint32_t header_version : 6; - uint32_t compression : 1; - uint32_t signature : 1; - uint32_t spare : 4; - uint32_t payload_target : 4; - uint32_t payload_major : 8; - uint32_t payload_minor : 8; - uint32_t payload_patch : 8; - uint32_t payload_build_num : 24; - } field; - uint8_t buf[sizeof(field)]; - static_assert(sizeof(buf) == 8, "Error: sizeof(HEADER.VERSION) != 8"); - }; - - union - { - struct __attribute__((packed)) - { - uint32_t len; - uint32_t crc32; - uint32_t magic_number; - HeaderVersion hdr_version; - } header; - uint8_t buf[sizeof(header)]; - static_assert(sizeof(buf) == 20, "Error: sizeof(HEADER) != 20"); - } ota_header; + OTAHeader ota_header; uint32_t crc32, bytes_read; uint8_t crc_buf[128]; From 23b820fa53fb97728ba46e702a96792980cc40e6 Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Mon, 4 Dec 2023 15:57:58 +0100 Subject: [PATCH 6/7] added example for download and decompress for ota images --- .../OTA_Qspi_Flash_download_onthefly.ino | 137 ++++++++++++++++++ .../arduino_secrets.h | 2 + 2 files changed, 139 insertions(+) create mode 100644 examples/OTA_Qspi_Flash_download_onthefly/OTA_Qspi_Flash_download_onthefly.ino create mode 100644 examples/OTA_Qspi_Flash_download_onthefly/arduino_secrets.h diff --git a/examples/OTA_Qspi_Flash_download_onthefly/OTA_Qspi_Flash_download_onthefly.ino b/examples/OTA_Qspi_Flash_download_onthefly/OTA_Qspi_Flash_download_onthefly.ino new file mode 100644 index 0000000..3944029 --- /dev/null +++ b/examples/OTA_Qspi_Flash_download_onthefly/OTA_Qspi_Flash_download_onthefly.ino @@ -0,0 +1,137 @@ +/* + * This example demonstrates how to use to update the firmware of the Arduino Portenta H7 using + * a firmware image stored on the QSPI. + * + * Steps: + * 1) Create a sketch for the Portenta H7 and verify + * that it both compiles and works on a board. + * 2) In the IDE select: Sketch -> Export compiled Binary. + * 3) Create an OTA update file utilising the tools 'lzss.py' and 'bin2ota.py' stored in + * https://github.com/arduino-libraries/ArduinoIoTCloud/tree/master/extras/tools . + * A) ./lzss.py --encode SKETCH.bin SKETCH.lzss + * B) ./bin2ota.py PORTENTA_H7_M7 SKETCH.lzss SKETCH.ota + * 4) Upload the OTA file to a network reachable location, e.g. OTA_Usage_Portenta.ino.PORTENTA_H7_M7.ota + * has been uploaded to: http://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.PORTENTA_H7_M7.ota + * 5) Perform an OTA update via steps outlined below. + */ + +/****************************************************************************** + * INCLUDE + ******************************************************************************/ + +#include + +#include + +#include "arduino_secrets.h" + +/****************************************************************************** + * CONSTANT + ******************************************************************************/ + +/* Please enter your sensitive data in the Secret tab/arduino_secrets.h */ +static char const SSID[] = SECRET_SSID; /* your network SSID (name) */ +static char const PASS[] = SECRET_PASS; /* your network password (use for WPA, or use as key for WEP) */ + +#if defined(ARDUINO_NICLA_VISION) +static char const OTA_FILE_LOCATION[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.NICLA_VISION.ota"; +#elif defined(ARDUINO_PORTENTA_H7_M7) +static char const OTA_FILE_LOCATION[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.PORTENTA_H7_M7.ota"; +#elif defined(ARDUINO_OPTA) +static char const OTA_FILE_LOCATION[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.OPTA.ota"; +#elif defined(ARDUINO_GIGA) +static char const OTA_FILE_LOCATION[] = "https://downloads.arduino.cc/ota/OTA_Usage_Portenta.ino.GIGA.ota"; +#else +#error "Board not supported" +#endif + +/****************************************************************************** + * SETUP/LOOP + ******************************************************************************/ + +void setup() +{ + Serial.begin(115200); + while (!Serial) {} + + if (WiFi.status() == WL_NO_SHIELD) + { + Serial.println("Communication with WiFi module failed!"); + return; + } + + int status = WL_IDLE_STATUS; + while (status != WL_CONNECTED) + { + Serial.print ("Attempting to connect to '"); + Serial.print (SSID); + Serial.println("'"); + status = WiFi.begin(SSID, PASS); + if(status != WL_CONNECTED) { + delay(10000); + } + } + Serial.print ("You're connected to '"); + Serial.print (WiFi.SSID()); + Serial.println("'"); + + Arduino_Portenta_OTA_QSPI ota(QSPI_FLASH_FATFS_MBR, 2); + Arduino_Portenta_OTA::Error ota_err = Arduino_Portenta_OTA::Error::None; + + if (!ota.isOtaCapable()) + { + Serial.println("Higher version bootloader required to perform OTA."); + Serial.println("Please update the bootloader."); + Serial.println("File -> Examples -> Portenta_System -> PortentaH7_updateBootloader"); + return; + } + + Serial.println("Initializing OTA storage"); + if ((ota_err = ota.begin()) != Arduino_Portenta_OTA::Error::None) + { + Serial.print ("Arduino_Portenta_OTA::begin() failed with error code "); + Serial.println((int)ota_err); + return; + } + + uint32_t start = millis(); + float elapsed, speed; + + Serial.println("Starting download to QSPI ..."); + int const ota_download = ota.downloadAndDecompress(OTA_FILE_LOCATION, true /* is_https */); + if (ota_download <= 0) + { + Serial.print ("Arduino_Portenta_OTA_QSPI::download failed with error code "); + Serial.println(ota_download); + return; + } + Serial.print (ota_download); + Serial.println(" bytes stored."); + + elapsed = (millis()-start)/1000.0; // elapsed expressed in seconds + speed = (ota_download/elapsed)/1024; + + Serial.print("download elapsed "); + Serial.print(elapsed); + Serial.print("s speed: "); + Serial.print(speed); + Serial.println("KBps"); + + Serial.println("Storing parameters for firmware update in bootloader accessible non-volatile memory ..."); + if ((ota_err = ota.update()) != Arduino_Portenta_OTA::Error::None) + { + Serial.print ("ota.update() failed with error code "); + Serial.println((int)ota_err); + return; + } + + Serial.println("Performing a reset after which the bootloader will update the firmware."); + Serial.println("Hint: Portenta H7 LED will blink Red-Blue-Green."); + delay(1000); /* Make sure the serial message gets out before the reset. */ + ota.reset(); +} + +void loop() +{ + +} diff --git a/examples/OTA_Qspi_Flash_download_onthefly/arduino_secrets.h b/examples/OTA_Qspi_Flash_download_onthefly/arduino_secrets.h new file mode 100644 index 0000000..0c9fdd5 --- /dev/null +++ b/examples/OTA_Qspi_Flash_download_onthefly/arduino_secrets.h @@ -0,0 +1,2 @@ +#define SECRET_SSID "" +#define SECRET_PASS "" From f3db4bc6d7179570c5de3ede561ed88ea25a804f Mon Sep 17 00:00:00 2001 From: Andrea Gilardoni Date: Wed, 6 Dec 2023 08:33:13 +0100 Subject: [PATCH 7/7] adding more examples to the ci --- .github/workflows/compile-examples.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/compile-examples.yml b/.github/workflows/compile-examples.yml index 6673c41..c8ba840 100644 --- a/.github/workflows/compile-examples.yml +++ b/.github/workflows/compile-examples.yml @@ -42,6 +42,8 @@ jobs: - examples/OTA_Qspi_Flash_Ethernet - examples/OTA_SD_Portenta - examples/OTA_Usage_Portenta + - examples/LZSS + - examples/OTA_Qspi_Flash_download_onthefly - fqbn: arduino:mbed_nicla:nicla_vision platforms: | - name: arduino:mbed_nicla @@ -50,6 +52,8 @@ jobs: sketch-paths: | - examples/OTA_Qspi_Flash - examples/OTA_Usage_Portenta + - examples/LZSS + - examples/OTA_Qspi_Flash_download_onthefly - fqbn: arduino:mbed_opta:opta platforms: | - name: arduino:mbed_opta @@ -59,6 +63,8 @@ jobs: - examples/OTA_Qspi_Flash - examples/OTA_Qspi_Flash_Ethernet - examples/OTA_Usage_Portenta + - examples/LZSS + - examples/OTA_Qspi_Flash_download_onthefly - fqbn: arduino:mbed_giga:giga platforms: | - name: arduino:mbed_giga @@ -67,6 +73,8 @@ jobs: sketch-paths: | - examples/OTA_Qspi_Flash - examples/OTA_Usage_Portenta + - examples/LZSS + - examples/OTA_Qspi_Flash_download_onthefly steps: - name: Checkout