From 79c3adc49de598ac848c6e069a71fc6fb7171bea Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 23 Aug 2022 03:47:29 +0300 Subject: [PATCH 1/8] stack decoder --- tools/decoder.py | 179 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tools/decoder.py diff --git a/tools/decoder.py b/tools/decoder.py new file mode 100644 index 0000000000..21fe7f2a22 --- /dev/null +++ b/tools/decoder.py @@ -0,0 +1,179 @@ +# Baseline code from https://github.com/me-no-dev/EspExceptionDecoder by Hristo Gochkov (@me-no-dev) +# - https://github.com/me-no-dev/EspExceptionDecoder/blob/master/src/EspExceptionDecoder.java +# Stack line detection from https://github.com/platformio/platform-espressif8266/ monitor exception filter by Vojtěch Boček (@Tasssadar) +# - https://github.com/platformio/platform-espressif8266/commits?author=Tasssadar + +import os +import argparse +import sys +import re +import subprocess + +# https://github.com/me-no-dev/EspExceptionDecoder/blob/349d17e4c9896306e2c00b4932be3ba510cad208/src/EspExceptionDecoder.java#L59-L90 +EXCEPTION_CODES = ( + "Illegal instruction", + "SYSCALL instruction", + "InstructionFetchError: Processor internal physical address or data error during " + "instruction fetch", + "LoadStoreError: Processor internal physical address or data error during load or store", + "Level1Interrupt: Level-1 interrupt as indicated by set level-1 bits in " + "the INTERRUPT register", + "Alloca: MOVSP instruction, if caller's registers are not in the register file", + "IntegerDivideByZero: QUOS, QUOU, REMS, or REMU divisor operand is zero", + "reserved", + "Privileged: Attempt to execute a privileged operation when CRING ? 0", + "LoadStoreAlignmentCause: Load or store to an unaligned address", + "reserved", + "reserved", + "InstrPIFDataError: PIF data error during instruction fetch", + "LoadStorePIFDataError: Synchronous PIF data error during LoadStore access", + "InstrPIFAddrError: PIF address error during instruction fetch", + "LoadStorePIFAddrError: Synchronous PIF address error during LoadStore access", + "InstTLBMiss: Error during Instruction TLB refill", + "InstTLBMultiHit: Multiple instruction TLB entries matched", + "InstFetchPrivilege: An instruction fetch referenced a virtual address at a ring level " + "less than CRING", + "reserved", + "InstFetchProhibited: An instruction fetch referenced a page mapped with an attribute " + "that does not permit instruction fetch", + "reserved", + "reserved", + "reserved", + "LoadStoreTLBMiss: Error during TLB refill for a load or store", + "LoadStoreTLBMultiHit: Multiple TLB entries matched for a load or store", + "LoadStorePrivilege: A load or store referenced a virtual address at a ring level " + "less than CRING", + "reserved", + "LoadProhibited: A load referenced a page mapped with an attribute that does not " + "permit loads", + "StoreProhibited: A store referenced a page mapped with an attribute that does not " + "permit stores", +) + +# similar to java version, which used `list` and re-formatted it +# instead, simply use an already short-format `info line` +# TODO `info symbol`? revert to `list`? +def addresses_gdb(gdb, elf, addresses): + cmd = [gdb, "--batch"] + for address in addresses: + if not address.startswith("0x"): + address = f"0x{address}" + cmd.extend(["--ex", f"info line *{address}"]) + cmd.append(elf) + + with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc: + for line in proc.stdout.readlines(): + if "No line number" in line: + continue + yield line.strip() + + +# original approach using addr2line, which is pretty enough already +def addresses_addr2line(addr2line, elf, addresses): + cmd = [ + addr2line, + "--addresses", + "--inlines", + "--functions", + "--pretty-print", + "--demangle", + "--exe", + elf, + ] + + for address in addresses: + if not address.startswith("0x"): + address = f"0x{address}" + cmd.append(address) + + with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc: + for line in proc.stdout.readlines(): + if "??:0" in line: + continue + yield line.strip() + + +def decode_lines(format_addresses, elf, lines): + STACK_RE = re.compile(r"^[0-9a-f]{8}:\s+([0-9a-f]{8} ?)+ *$") + + EXCEPTION_STRING = "Exception (" + EPC_STRING = "epc1=" + + # either print everything as-is, or cache current string and dump after stack contents end + stack_addresses = [] + in_stack = False + + def print_all_addresses(addresses): + if addresses: + for formatted in format_addresses(elf, addresses): + print(formatted) + return list() + + for line in lines: + # ctx could happen multiple times. for the 2nd one, reset list + # ctx: bearssl *or* ctx: cont *or* ctx: sys *or* ctx: whatever + if in_stack and "ctx:" in line: + stack_addresses = print_all_addresses(stack_addresses) + # sp: 3ffffdf0 end: 3fffffc0 offset: 0000 + elif in_stack and "sp:" in line: + continue + # 3fffffb0: feefeffe feefeffe 3ffe85d8 401004ed + elif in_stack and STACK_RE.match(line): + stack, addrs = line.split(":") + addrs = addrs.strip() + addrs = addrs.split(" ") + for addr in addrs: + stack_addresses.append(addr) + # epc1=0xfffefefe epc2=0xfefefefe epc3=0xefefefef excvaddr=0xfefefefe depc=0xfefefefe + elif EPC_STRING in line: + pairs = line.split() + for pair in pairs: + name, addr = pair.split("=") + if name in ["epc1", "excvaddr"]: + output = "\n".join(format_addresses(elf, [addr])) + if output: + print(f"{name}={output}") + # Exception (123): + # Other reasons coming before the guard shown as-is + elif EXCEPTION_STRING in line: + number = line.strip()[len(EXCEPTION_STRING) : -2] + print(f"Exception ({number}) - {EXCEPTION_CODES[int(number)]}") + # postmortem guards our actual stack dump values with these + elif ">>>stack>>>" in line: + in_stack = True + elif "<< Date: Tue, 23 Aug 2022 04:08:55 +0300 Subject: [PATCH 2/8] +x --- tools/decoder.py | 2 ++ 1 file changed, 2 insertions(+) mode change 100644 => 100755 tools/decoder.py diff --git a/tools/decoder.py b/tools/decoder.py old mode 100644 new mode 100755 index 21fe7f2a22..a874d4c2da --- a/tools/decoder.py +++ b/tools/decoder.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # Baseline code from https://github.com/me-no-dev/EspExceptionDecoder by Hristo Gochkov (@me-no-dev) # - https://github.com/me-no-dev/EspExceptionDecoder/blob/master/src/EspExceptionDecoder.java # Stack line detection from https://github.com/platformio/platform-espressif8266/ monitor exception filter by Vojtěch Boček (@Tasssadar) From 701b060603a997852131893093c32764b59cb9cd Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 23 Aug 2022 04:31:23 +0300 Subject: [PATCH 3/8] cut here --- tools/decoder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/decoder.py b/tools/decoder.py index a874d4c2da..15d305e788 100755 --- a/tools/decoder.py +++ b/tools/decoder.py @@ -98,6 +98,7 @@ def addresses_addr2line(addr2line, elf, addresses): def decode_lines(format_addresses, elf, lines): STACK_RE = re.compile(r"^[0-9a-f]{8}:\s+([0-9a-f]{8} ?)+ *$") + CUT_HERE_STRING = "CUT HERE FOR EXCEPTION DECODER" EXCEPTION_STRING = "Exception (" EPC_STRING = "epc1=" @@ -145,6 +146,9 @@ def print_all_addresses(addresses): in_stack = True elif "<< Date: Tue, 23 Aug 2022 14:53:47 +0300 Subject: [PATCH 4/8] last alloc explain, line breaks --- tools/decoder.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tools/decoder.py b/tools/decoder.py index 15d305e788..a9887ad45b 100755 --- a/tools/decoder.py +++ b/tools/decoder.py @@ -98,6 +98,9 @@ def addresses_addr2line(addr2line, elf, addresses): def decode_lines(format_addresses, elf, lines): STACK_RE = re.compile(r"^[0-9a-f]{8}:\s+([0-9a-f]{8} ?)+ *$") + LAST_ALLOC_RE = re.compile(r"last failed alloc call: ([0-9a-fA-F]{8})\(([0-9]+)\).*") + LAST_ALLOC = "last failed alloc" + CUT_HERE_STRING = "CUT HERE FOR EXCEPTION DECODER" EXCEPTION_STRING = "Exception (" EPC_STRING = "epc1=" @@ -141,17 +144,28 @@ def print_all_addresses(addresses): elif EXCEPTION_STRING in line: number = line.strip()[len(EXCEPTION_STRING) : -2] print(f"Exception ({number}) - {EXCEPTION_CODES[int(number)]}") + # last failed alloc call: ()[@] + elif LAST_ALLOC in line: + values = LAST_ALLOC_RE.match(line) + if values: + addr, size = values.groups() + output = "\n".join(format_addresses(elf, [addr])) + print() + print(f"Allocation of {size} bytes failed: {output}") # postmortem guards our actual stack dump values with these elif ">>>stack>>>" in line: in_stack = True - elif "<< Date: Tue, 23 Aug 2022 14:56:28 +0300 Subject: [PATCH 5/8] capture --- tools/decoder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/decoder.py b/tools/decoder.py index a9887ad45b..d18a272ce5 100755 --- a/tools/decoder.py +++ b/tools/decoder.py @@ -115,6 +115,9 @@ def print_all_addresses(addresses): print(formatted) return list() + def format_address(address): + return "\n".join(format_addresses(elf, [address])) + for line in lines: # ctx could happen multiple times. for the 2nd one, reset list # ctx: bearssl *or* ctx: cont *or* ctx: sys *or* ctx: whatever @@ -136,7 +139,7 @@ def print_all_addresses(addresses): for pair in pairs: name, addr = pair.split("=") if name in ["epc1", "excvaddr"]: - output = "\n".join(format_addresses(elf, [addr])) + output = format_address(addr) if output: print(f"{name}={output}") # Exception (123): @@ -149,9 +152,8 @@ def print_all_addresses(addresses): values = LAST_ALLOC_RE.match(line) if values: addr, size = values.groups() - output = "\n".join(format_addresses(elf, [addr])) print() - print(f"Allocation of {size} bytes failed: {output}") + print(f"Allocation of {size} bytes failed: {format_address(addr)}") # postmortem guards our actual stack dump values with these elif ">>>stack>>>" in line: in_stack = True From b61155add27cb4d6242eb8a6ed9b71e51095c302 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 23 Aug 2022 18:57:57 +0300 Subject: [PATCH 6/8] print ctx line --- tools/decoder.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/decoder.py b/tools/decoder.py index d18a272ce5..6c9117adb0 100755 --- a/tools/decoder.py +++ b/tools/decoder.py @@ -106,14 +106,18 @@ def decode_lines(format_addresses, elf, lines): EPC_STRING = "epc1=" # either print everything as-is, or cache current string and dump after stack contents end - stack_addresses = [] + last_stack = None + stack_addresses = {} + in_stack = False def print_all_addresses(addresses): - if addresses: - for formatted in format_addresses(elf, addresses): + for ctx, addrs in addresses.items(): + print() + print(ctx) + for formatted in format_addresses(elf, addrs): print(formatted) - return list() + return dict() def format_address(address): return "\n".join(format_addresses(elf, [address])) @@ -123,6 +127,7 @@ def format_address(address): # ctx: bearssl *or* ctx: cont *or* ctx: sys *or* ctx: whatever if in_stack and "ctx:" in line: stack_addresses = print_all_addresses(stack_addresses) + last_stack = line.strip() # sp: 3ffffdf0 end: 3fffffc0 offset: 0000 elif in_stack and "sp:" in line: continue @@ -131,8 +136,9 @@ def format_address(address): stack, addrs = line.split(":") addrs = addrs.strip() addrs = addrs.split(" ") + stack_addresses.setdefault(last_stack, []) for addr in addrs: - stack_addresses.append(addr) + stack_addresses[last_stack].append(addr) # epc1=0xfffefefe epc2=0xfefefefe epc3=0xefefefef excvaddr=0xfefefefe depc=0xfefefefe elif EPC_STRING in line: pairs = line.split() @@ -167,7 +173,6 @@ def format_address(address): if line: print(line) - print() print_all_addresses(stack_addresses) From 894c80f74825467a169f865c9a8527aa5da41a41 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 23 Aug 2022 19:02:04 +0300 Subject: [PATCH 7/8] ...and dont ignore sp --- tools/decoder.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/decoder.py b/tools/decoder.py index 6c9117adb0..cc55ae82d3 100755 --- a/tools/decoder.py +++ b/tools/decoder.py @@ -128,9 +128,6 @@ def format_address(address): if in_stack and "ctx:" in line: stack_addresses = print_all_addresses(stack_addresses) last_stack = line.strip() - # sp: 3ffffdf0 end: 3fffffc0 offset: 0000 - elif in_stack and "sp:" in line: - continue # 3fffffb0: feefeffe feefeffe 3ffe85d8 401004ed elif in_stack and STACK_RE.match(line): stack, addrs = line.split(":") From 79eea27638607115bbc0499fd2013f2b41e9d136 Mon Sep 17 00:00:00 2001 From: Maxim Prokhorov Date: Tue, 23 Aug 2022 19:14:52 +0300 Subject: [PATCH 8/8] non-hyphenated arg for elf, toolchain path to bin/ either for when tools are already in PATH or, using `pio pkg exec --package toolchain-xtensa python decoder.py ...` (where package is a full version spec for pio registry) --- tools/decoder.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/decoder.py b/tools/decoder.py index cc55ae82d3..1ed3dcf8ed 100755 --- a/tools/decoder.py +++ b/tools/decoder.py @@ -98,7 +98,9 @@ def addresses_addr2line(addr2line, elf, addresses): def decode_lines(format_addresses, elf, lines): STACK_RE = re.compile(r"^[0-9a-f]{8}:\s+([0-9a-f]{8} ?)+ *$") - LAST_ALLOC_RE = re.compile(r"last failed alloc call: ([0-9a-fA-F]{8})\(([0-9]+)\).*") + LAST_ALLOC_RE = re.compile( + r"last failed alloc call: ([0-9a-fA-F]{8})\(([0-9]+)\).*" + ) LAST_ALLOC = "last failed alloc" CUT_HERE_STRING = "CUT HERE FOR EXCEPTION DECODER" @@ -176,8 +178,10 @@ def format_address(address): TOOLS = {"gdb": addresses_gdb, "addr2line": addresses_addr2line} -def select_tool(toolchain_path, tool): - path = os.path.join(toolchain_path, "bin", f"xtensa-lx106-elf-{tool}") +def select_tool(toolchain_path, tool, func): + path = f"xtensa-lx106-elf-{tool}" + if toolchain_path: + path = os.path.join(toolchain_path, path) def formatter(func, path): def wrapper(elf, addresses): @@ -185,19 +189,24 @@ def wrapper(elf, addresses): return wrapper - return formatter(TOOLS[tool], path) + return formatter(func, path) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--elf-path", required=True) - parser.add_argument("--toolchain-path", required=True) parser.add_argument("--tool", choices=TOOLS, default="addr2line") + parser.add_argument( + "--toolchain-path", help="Sets path to Xtensa tools, when they are not in PATH" + ) + + parser.add_argument("firmware_elf") parser.add_argument( "postmortem", nargs="?", type=argparse.FileType("r"), default=sys.stdin ) args = parser.parse_args() decode_lines( - select_tool(args.toolchain_path, args.tool), args.elf_path, args.postmortem + select_tool(args.toolchain_path, args.tool, TOOLS[args.tool]), + args.firmware_elf, + args.postmortem, )