Use esptool.py to handle sketch upload, make python available on Windows, too

.gitignore

@@ -1,7 +1,6 @@
 .DS_Store
 tools/dist/
 tools/xtensa-lx106-elf/
-tools/esptool/
 tools/mkspiffs/
 package/versions/
 exclude.txt

.gitmodules

@@ -7,3 +7,9 @@
 [submodule "libraries/SoftwareSerial"]
 	path = libraries/SoftwareSerial
 	url = https://github.com/plerup/espsoftwareserial.git
+[submodule "tools/pyserial"]
+	path = tools/pyserial
+	url = https://github.com/pyserial/pyserial.git
+[submodule "tools/esptool"]
+	path = tools/esptool
+	url = https://github.com/espressif/esptool.git

package/build_boards_manager_package.sh

@@ -78,7 +78,6 @@ $SED 's/runtime.tools.esptool.path={runtime.platform.path}\/tools\/esptool//g' |
 $SED 's/tools.esptool.path={runtime.platform.path}\/tools\/esptool/tools.esptool.path=\{runtime.tools.esptool.path\}/g' | \
 $SED 's/tools.mkspiffs.path={runtime.platform.path}\/tools\/mkspiffs/tools.mkspiffs.path=\{runtime.tools.mkspiffs.path\}/g' |\
 $SED 's/recipe.hooks.core.prebuild.2.pattern.*//g' |\
-$SED 's/recipe.hooks.core.prebuild.3.pattern.*//g' |\
 $SED "s/version=.*/version=$ver/g" |\
 $SED -E "s/name=([a-zA-Z0-9\ -]+).*/name=\1($ver)/g"\
  > $outdir/platform.txt

package/package_esp8266com_index.template.json

@@ -1,11 +1,13 @@
 {
    "packages": [
       {
+         "name": "esp8266",
          "maintainer": "ESP8266 Community",
+         "websiteURL": "https://github.com/esp8266/Arduino",
+         "email": "[email protected]",
          "help": {
             "online": "http://esp8266.com/arduino"
          },
-         "websiteURL": "https://github.com/esp8266/Arduino",
          "platforms": [
             {
                "category": "ESP8266",
@@ -110,17 +112,17 @@
                   {
                      "packager": "esp8266",
                      "version": "2.5.0-3-20ed2b9",
-                     "name": "esptool"
+                     "name": "xtensa-lx106-elf-gcc"
                   },
                   {
                      "packager": "esp8266",
                      "version": "2.5.0-3-20ed2b9",
-                     "name": "xtensa-lx106-elf-gcc"
+                     "name": "mkspiffs"
                   },
                   {
                      "packager": "esp8266",
-                     "version": "2.5.0-3-20ed2b9",
-                     "name": "mkspiffs"
+                     "version": "3.7.2-post1",
+                     "name": "python"
                   }
                ],
                "help": {
@@ -130,57 +132,22 @@
          ],
          "tools": [
             {
-               "version": "2.5.0-3-20ed2b9",
-               "name": "esptool",
+               "version": "3.7.2-post1",
+               "name": "python",
                "systems": [
                   {
-                     "host": "aarch64-linux-gnu",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/aarch64-linux-gnu.esptool-f80ae31.tar.gz",
-                     "archiveFileName": "aarch64-linux-gnu.esptool-f80ae31.tar.gz",
-                     "checksum": "SHA-256:888425ff1e33a97ea155b6f128de6b578c34468895ba9b4acd1e4f28608d917a",
-                     "size": "14681"
-                  },
-                  {
-                     "host": "arm-linux-gnueabihf",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/arm-linux-gnueabihf.esptool-f80ae31.tar.gz",
-                     "archiveFileName": "arm-linux-gnueabihf.esptool-f80ae31.tar.gz",
-                     "checksum": "SHA-256:71c878ac6a21ee69dcd615cd28f2dccd29a87079e0b3069eba625089d89e5058",
-                     "size": "13873"
+                     "host": "x86_64-mingw32",
+                     "url": "https://www.python.org/ftp/python/3.7.2/python-3.7.2.post1-embed-win32.zip",
+                     "archiveFileName": "python-3.7.2.post1-embed-win32.zip",
+                     "checksum": "SHA-256:ceb06a5244e93e7e7523e26e1354447b79a9e6fd8c45891d81df9c7da1e02d05",
+                     "size": "6533256"
                   },
                   {
                      "host": "i686-mingw32",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/i686-w64-mingw32.esptool-f80ae31.zip",
-                     "archiveFileName": "i686-w64-mingw32.esptool-f80ae31.zip",
-                     "checksum": "SHA-256:e30f25a19a78635000401b083b479e111d591bac20cfd89b1bfdf36a60e9ee20",
-                     "size": "16466"
-                  },
-                  {
-                     "host": "i686-pc-linux-gnu",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/i686-linux-gnu.esptool-f80ae31.tar.gz",
-                     "archiveFileName": "i686-linux-gnu.esptool-f80ae31.tar.gz",
-                     "checksum": "SHA-256:fe632f4602d02b6a9425c5bf95074095cb6d3c57912168a0f6b796fddd8ce991",
-                     "size": "16543"
-                  },
-                  {
-                     "host": "x86_64-apple-darwin",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/x86_64-apple-darwin14.esptool-f80ae31.tar.gz",
-                     "archiveFileName": "x86_64-apple-darwin14.esptool-f80ae31.tar.gz",
-                     "checksum": "SHA-256:0f51e487706a476b0b87299a769282ad65623774770655168a79d1748d2506e7",
-                     "size": "15003"
-                  },
-                  {
-                     "host": "x86_64-pc-linux-gnu",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/x86_64-linux-gnu.esptool-f80ae31.tar.gz",
-                     "archiveFileName": "x86_64-linux-gnu.esptool-f80ae31.tar.gz",
-                     "checksum": "SHA-256:bded1dca953377838b6086a9bcd40a1dc5286ba5f69f2372c22a1d1819baad24",
-                     "size": "16526"
-                  },
-                  {
-                     "host": "x86_64-mingw32",
-                     "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/x86_64-w64-mingw32.esptool-f80ae31.zip",
-                     "archiveFileName": "x86_64-w64-mingw32.esptool-f80ae31.zip",
-                     "checksum": "SHA-256:d6d5976fde82d07e93d5a01f38bbb4f84a7796187ff0541ee62650041791d0e8",
-                     "size": "19724"
+                     "url": "https://www.python.org/ftp/python/3.7.2/python-3.7.2.post1-embed-win32.zip",
+                     "archiveFileName": "python-3.7.2.post1-embed-win32.zip",
+                     "checksum": "SHA-256:ceb06a5244e93e7e7523e26e1354447b79a9e6fd8c45891d81df9c7da1e02d05",
+                     "size": "6533256"
                   }
                ]
             },
@@ -294,9 +261,7 @@
                   }
                ]
             }
-         ],
-         "email": "[email protected]",
-         "name": "esp8266"
+         ]
       }
    ]
 }

platform.txt

@@ -11,6 +11,11 @@ version=2.6.0-dev
 runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf
 runtime.tools.esptool.path={runtime.platform.path}/tools/esptool
 runtime.tools.signing={runtime.platform.path}/tools/signing.py
+runtime.tools.elf2bin={runtime.platform.path}/tools/elf2bin.py
+runtime.tools.makecorever={runtime.platform.path}/tools/makecorever.py
+runtime.tools.eboot={runtime.platform.path}/bootloaders/eboot/eboot.elf
+runtime.tools.python=python
+runtime.tools.python.windows={runtime.platform.path}/tools/python/python.exe
 
 compiler.warning_flags=-w
 compiler.warning_flags.none=-w
@@ -76,14 +81,8 @@ compiler.elf2hex.extra_flags=
 
 ## generate file with git version number
 ## needs bash, git, and echo
-recipe.hooks.core.prebuild.1.pattern=python "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h"
-recipe.hooks.core.prebuild.2.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_VER 0x`git --git-dir {runtime.platform.path}/.git rev-parse --short=8 HEAD 2>/dev/null || echo ffffffff` >{build.path}/core/core_version.h"
-recipe.hooks.core.prebuild.3.pattern=bash -c "mkdir -p {build.path}/core && echo \#define ARDUINO_ESP8266_GIT_DESC `cd "{runtime.platform.path}"; git describe --tags 2>/dev/null || echo unix-{version}` >>{build.path}/core/core_version.h"
-
-## windows-compatible version without git
-recipe.hooks.core.prebuild.1.pattern.windows=cmd.exe /c rem cannot sign on windows
-recipe.hooks.core.prebuild.2.pattern.windows=cmd.exe /c mkdir {build.path}\core & (echo #define ARDUINO_ESP8266_GIT_VER 0x00000000 & echo #define ARDUINO_ESP8266_GIT_DESC win-{version} ) > {build.path}\core\core_version.h
-recipe.hooks.core.prebuild.3.pattern.windows=cmd.exe /c if exist {build.source.path}\public.key echo #error Cannot automatically build signed binaries on Windows > {build.path}\core\Updater_Signing.h
+recipe.hooks.core.prebuild.1.pattern="{runtime.tools.python}" "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h"
+recipe.hooks.core.prebuild.2.pattern="{runtime.tools.python}" "{runtime.tools.makecorever}" --build_path "{build.path}' --platform_path "{runtime.platform.path}" --version "unix-{version}"
 
 ## Build the app.ld linker file
 recipe.hooks.linking.prelink.1.pattern="{compiler.path}{compiler.c.cmd}" -CC -E -P {build.vtable_flags} "{runtime.platform.path}/tools/sdk/ld/eagle.app.v6.common.ld.h" -o "{build.path}/local.eagle.app.v6.common.ld"
@@ -107,14 +106,8 @@ recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {build.exception_
 recipe.objcopy.eep.pattern=
 
 ## Create hex
-#recipe.objcopy.hex.pattern="{compiler.path}{compiler.elf2hex.cmd}" {compiler.elf2hex.flags} {compiler.elf2hex.extra_flags} "{build.path}/{build.project_name}.elf" "{build.path}/{build.project_name}.hex"
-
-recipe.objcopy.hex.1.pattern="{runtime.tools.esptool.path}/{compiler.esptool.cmd}" -eo "{runtime.platform.path}/bootloaders/eboot/eboot.elf" -bo "{build.path}/{build.project_name}.bin" -bm {build.flash_mode} -bf {build.flash_freq} -bz {build.flash_size} -bs .text -bp 4096 -ec -eo "{build.path}/{build.project_name}.elf" -bs .irom0.text -bs .text -bs .data -bs .rodata -bc -ec
-recipe.objcopy.hex.2.pattern=python "{runtime.tools.signing}" --mode sign --privatekey "{build.source.path}/private.key" --bin "{build.path}/{build.project_name}.bin" --out "{build.path}/{build.project_name}.bin.signed"
-
-# No signing on Windows
-recipe.objcopy.hex.1.pattern.windows="{runtime.tools.esptool.path}/{compiler.esptool.cmd}" -eo "{runtime.platform.path}/bootloaders/eboot/eboot.elf" -bo "{build.path}/{build.project_name}.bin" -bm {build.flash_mode} -bf {build.flash_freq} -bz {build.flash_size} -bs .text -bp 4096 -ec -eo "{build.path}/{build.project_name}.elf" -bs .irom0.text -bs .text -bs .data -bs .rodata -bc -ec
-recipe.objcopy.hex.2.pattern.windows=
+recipe.objcopy.hex.1.pattern="{runtime.tools.python}" "{runtime.tools.elf2bin}" --eboot "{runtime.tools.eboot}" --app "{build.path}/{build.project_name}.elf" --flash_mode {build.flash_mode} --flash_freq {build.flash_freq} --flash_size {build.flash_size} --path "{runtime.tools.xtensa-lx106-elf-gcc.path}/bin" --out "{build.path}/{build.project_name}.bin"
+recipe.objcopy.hex.2.pattern="{runtime.tools.python}" "{runtime.tools.signing}" --mode sign --privatekey "{build.source.path}/private.key" --bin "{build.path}/{build.project_name}.bin" --out "{build.path}/{build.project_name}.bin.signed"
 
 ## Save hex
 recipe.output.tmp_file={build.project_name}.bin
@@ -128,16 +121,23 @@ recipe.size.regex.data=^(?:\.data|\.rodata|\.bss)\s+([0-9]+).*
 
 # ------------------------------
 
-tools.esptool.cmd=esptool
-tools.esptool.cmd.windows=esptool.exe
-tools.esptool.path={runtime.platform.path}/tools/esptool
+# Need to duplicate the python path because Arduino does not replace properly in this stage
+tools.esptool.cmd=python
+tools.esptool.cmd.windows={runtime.platform.path}/tools/python/python.exe
+tools.esptool.path=
 tools.esptool.network_cmd=python
-tools.esptool.network_cmd.windows=python.exe
+tools.esptool.network_cmd.windows={runtime.platform.path}/tools/python/python.exe
 
 tools.esptool.upload.protocol=esp
-tools.esptool.upload.params.verbose=-vv
+tools.esptool.upload.params.verbose=--trace
 tools.esptool.upload.params.quiet=
-tools.esptool.upload.pattern="{path}/{cmd}" {upload.verbose} -cd {upload.resetmethod} -cb {upload.speed} -cp "{serial.port}" {upload.erase_cmd} -ca 0x00000 -cf "{build.path}/{build.project_name}.bin" 
+
+# First, potentially perform an erase or nothing
+# Next, do the binary upload
+# Combined in one rule because Arduino doesn't suport upload.1.pattern/upload.3.pattern
+tools.esptool.upload.pattern="{cmd}" "{runtime.platform.path}/tools/upload.py" "{runtime.platform.path}/tools/pyserial" "{runtime.platform.path}/tools/esptool/esptool.py" --chip esp8266 --port "{serial.port}" --baud "{upload.speed}" "{upload.verbose}" {upload.erase_cmd} --end  --chip esp8266 --port "{serial.port}" --baud "{upload.speed}" "{upload.verbose}" write_flash 0x0 "{build.path}/{build.project_name}.bin" --end
+
+
 tools.esptool.upload.network_pattern="{network_cmd}" "{runtime.platform.path}/tools/espota.py" -i "{serial.port}" -p "{network.port}" "--auth={network.password}" -f "{build.path}/{build.project_name}.bin"
 
 tools.mkspiffs.cmd=mkspiffs

tools/boards.txt.py

@@ -851,7 +851,7 @@
         ( '.upload.tool', 'esptool' ),
         ( '.upload.maximum_data_size', '81920' ),
         ( '.upload.wait_for_upload_port', 'true' ),
-        ( '.upload.erase_cmd', ''),
+        ( '.upload.erase_cmd', 'version'),
         ( '.serial.disableDTR', 'true' ),
         ( '.serial.disableRTS', 'true' ),
         ( '.build.mcu', 'esp8266' ),
@@ -1066,11 +1066,11 @@
 
     'flash_erase_menu': collections.OrderedDict([
         ( '.menu.wipe.none', 'Only Sketch' ),
-        ( '.menu.wipe.none.upload.erase_cmd', '' ),
+        ( '.menu.wipe.none.upload.erase_cmd', 'version' ),
         ( '.menu.wipe.sdk', 'Sketch + WiFi Settings' ),
-        ( '.menu.wipe.sdk.upload.erase_cmd', '-ca "{build.rfcal_addr}" -cz 0x4000' ),
+        ( '.menu.wipe.sdk.upload.erase_cmd', 'erase_region "{build.rfcal_addr}" 0x4000' ),
         ( '.menu.wipe.all', 'All Flash Contents' ),
-        ( '.menu.wipe.all.upload.erase_cmd', '-ca 0x0 -cz "{build.flash_size_bytes}"' ),
+        ( '.menu.wipe.all.upload.erase_cmd', 'erase_flash' ),
         ]),
 
     }

tools/elf2bin.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+
+# Generate an Arduino compatible BIN file from bootloader and sketch ELF
+# Replaces esptool-ck.exe and emulates its behavior.
+#
+# Copyright (C) 2019 - Earle F. Philhower, III
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import re
+import os
+import subprocess
+import sys
+import tempfile
+
+fmodeb = { 'dout': 3, 'dio': 2, 'quot': 1, 'qio': 0 }
+ffreqb = { '40': 0, '26': 1, '20': 2, '80': 15 }
+fsizeb = { '512K': 0, '256K': 1, '1M': 2, '2M': 3, '4M': 4, '8M': 8, '16M': 9 }
+
+def get_elf_entry(elf, path):
+    p = subprocess.Popen([path + "/xtensa-lx106-elf-readelf", '-h', elf], stdout=subprocess.PIPE, universal_newlines=True )
+    lines = p.stdout.readlines()
+    for line in lines:
+        if 'Entry point address' in line:
+            words = re.split('\s+', line)
+            entry_point = words[-2]
+            return int(entry_point, 16)
+    raise Exception('Unable to find entry point in file "' + elf + '"')
+
+def get_segment_size_addr(elf, segment, path):
+    p = subprocess.Popen([path + '/xtensa-lx106-elf-objdump', '-h', '-j', segment,  elf], stdout=subprocess.PIPE, universal_newlines=True )
+    lines = p.stdout.readlines()
+    for line in lines:
+        if segment in line:
+            words = re.split('\s+', line)
+            size = int(words[3], 16)
+            addr = int(words[4], 16)
+            return [ size, addr ]
+    raise Exception('Unable to find size and start point in file "' + elf + '" for "' + segment + '"')
+
+def read_segment(elf, segment, path):
+    tmpfile, dumpfile = tempfile.mkstemp()
+    os.close(tmpfile)
+    p = subprocess.check_call([path + "/xtensa-lx106-elf-objcopy", '-O', 'binary', '--only-section=' + segment, elf, dumpfile], stdout=subprocess.PIPE)
+    binfile = open(dumpfile, "rb")
+    raw = binfile.read()
+    binfile.close()
+    return raw
+
+def write_bin(out, elf, segments, to_addr, flash_mode, flash_size, flash_freq, path):
+    entry = int(get_elf_entry( elf, path ))
+    header = [ 0xe9, len(segments), fmodeb[flash_mode], ffreqb[flash_freq] + 16 * fsizeb[flash_size],
+               entry & 255, (entry>>8) & 255, (entry>>16) & 255, (entry>>24) & 255 ]
+    out.write(bytearray(header))
+    total_size = 8
+    checksum = 0xef
+    for segment in segments:
+        [size, addr] = get_segment_size_addr(elf, segment, path)
+        seghdr = [ addr & 255, (addr>>8) & 255, (addr>>16) & 255, (addr>>24) & 255,
+                   size & 255, (size>>8) & 255, (size>>16) & 255, (size>>24) & 255]
+        out.write(bytearray(seghdr));
+        total_size += 8;
+        raw = read_segment(elf, segment, path)
+        if len(raw) != size:
+            raise Exception('Segment size doesn\'t match read data for "' + segment + '" in "' + elf + '"')
+        out.write(raw)
+        total_size += len(raw)
+        try:
+            for data in raw:
+                checksum = checksum ^ ord(data)
+        except:
+            for data in raw:
+                checksum = checksum ^ data
+    total_size += 1
+    while total_size & 15:
+        total_size += 1
+        out.write(bytearray([0]))
+    out.write(bytearray([checksum]))
+    if to_addr != 0:
+        while total_size < to_addr:
+            out.write(bytearray([0xaa]))
+            total_size += 1
+
+def main():
+    parser = argparse.ArgumentParser(description='Create a BIN file from eboot.elf and Arduino sketch.elf for upload by esptool.py')
+    parser.add_argument('-e', '--eboot', action='store', required=True, help='Path to the Arduino eboot.elf bootloader')
+    parser.add_argument('-a', '--app', action='store', required=True, help='Path to the Arduino sketch ELF')
+    parser.add_argument('-m', '--flash_mode', action='store', required=True, choices=['dout', 'dio', 'qout', 'qio'], help='SPI flash mode')
+    parser.add_argument('-f', '--flash_freq', action='store', required=True, choices=['20', '26', '40', '80'], help='SPI flash speed')
+    parser.add_argument('-s', '--flash_size', action='store', required=True, choices=['256K', '512K', '1M', '2M', '4M', '8M', '16M'], help='SPI flash size')
+    parser.add_argument('-o', '--out', action='store', required=True, help='Output BIN filename')
+    parser.add_argument('-p', '--path', action='store', required=True, help='Path to Xtensa toolchain binaries')
+
+    args = parser.parse_args()
+
+    print('Creating BIN file "' + args.out + '" using "' + args.app + '"')
+
+    out = open(args.out, "wb")
+    write_bin(out, args.eboot, ['.text'], 4096, args.flash_mode, args.flash_size, args.flash_freq, args.path)
+    write_bin(out, args.app, ['.irom0.text', '.text', '.data', '.rodata'], 0, args.flash_mode, args.flash_size, args.flash_freq, args.path)
+    out.close()
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())