Skip to content

Commit e6e2a4b

Browse files
mocleirimocleiri
committed
Merge branch 'feature/59-integrate-esp32-camera-acquisition'
2 parents b1c4dc8 + f468325 commit e6e2a4b

23 files changed

+1409
-1
lines changed

‎.github/workflows/build_esp32.yml

+2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ jobs:
2525
git submodule update --init lib/axtls
2626
git submodule update --init lib/berkeley-db-1.xx
2727
cd ..
28+
cd tflm_esp_kernels
29+
git submodule update --init examples/person_detection/esp32-camera
2830
- name: Get Cache Keys
2931
# later get this like this: git ls-remote --heads https://github.com/espressif/esp-idf
3032
# this commit is hard-coded in micropython/tools/ci.sh

‎.gitmodules

+4
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,7 @@
1010
path = micropython-ulab
1111
url = https://github.com/v923z/micropython-ulab.git
1212
branch = master
13+
[submodule "tflm_esp_kernels"]
14+
path = tflm_esp_kernels
15+
url = https://github.com/espressif/tflite-micro-esp-examples.git
16+
branch = master

‎.vscode/c_cpp_properties.json

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"MICROPY_MODULE_FROZEN_MPY",
3131
"MICROPY_ROM_TEXT_COMPRESSION=1",
3232
"CMSIS_NN",
33+
"MODULE_CAMERA_ENABLED",
3334
"__ARM_FEATURE_DSP=1"
3435
],
3536
"compilerPath": "/opt/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc",

‎boards/esp32/MICROLITE_SPIRAM_CAM/CMakeLists.txt

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Top-level cmake file for building MicroPython on ESP32.
2+
# Nothing in here needs to be customised, it will work for any board.
3+
4+
cmake_minimum_required(VERSION 3.12)
5+
6+
# Set the location of MicroPython, the esp32 port, and the board directory.
7+
get_filename_component(PROJECT_DIR "../../.." ABSOLUTE)
8+
set(MICROPY_PORT_DIR ${PROJECT_DIR}/micropython/ports/esp32)
9+
get_filename_component(MICROPY_BOARD_DIR "." ABSOLUTE)
10+
11+
set(EXTRA_COMPONENT_DIRS ../../../tflm_esp_kernels/examples/person_detection/esp32-camera)
12+
13+
message (STATUS "EXTRA_COMPONENT_DIRS=${EXTRA_COMPONENT_DIRS}")
14+
message (STATUS "PROJECT_DIR=${PROJECT_DIR}")
15+
message (STATUS "MICROPY_PORT_DIR=${MICROPY_PORT_DIR}")
16+
message (STATUS "MICROPY_BOARD_DIR=${MICROPY_BOARD_DIR}")
17+
18+
# Define the output sdkconfig so it goes in the build directory.
19+
set(SDKCONFIG ${CMAKE_BINARY_DIR}/sdkconfig)
20+
21+
# Include board config; this is expected to set SDKCONFIG_DEFAULTS (among other options).
22+
include(${MICROPY_BOARD_DIR}/mpconfigboard.cmake)
23+
24+
# Concatenate all sdkconfig files into a combined one for the IDF to use.
25+
file(WRITE ${CMAKE_BINARY_DIR}/sdkconfig.combined.in "")
26+
foreach(SDKCONFIG_DEFAULT ${SDKCONFIG_DEFAULTS})
27+
file(READ ${SDKCONFIG_DEFAULT} CONTENTS)
28+
file(APPEND ${CMAKE_BINARY_DIR}/sdkconfig.combined.in "${CONTENTS}")
29+
endforeach()
30+
configure_file(${CMAKE_BINARY_DIR}/sdkconfig.combined.in ${CMAKE_BINARY_DIR}/sdkconfig.combined COPYONLY)
31+
set(SDKCONFIG_DEFAULTS ${CMAKE_BINARY_DIR}/sdkconfig.combined)
32+
33+
# Include main IDF cmake file and define the project.
34+
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
35+
project(micropython)

‎boards/esp32/MICROLITE_SPIRAM_CAM/custom-partitions.csv

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Notes: the offset of the partition table itself is set in
2+
# $ESPIDF/components/partition_table/Kconfig.projbuild and the
3+
# offset of the factory/ota_0 partition is set in makeimg.py
4+
# I needed to increase the size of the app partition to fit the tensorflow microlite library
5+
# There is 1/2 as much data partition as with standard micropython on esp32 4MiB.
6+
# Name, Type, SubType, Offset, Size, Flags
7+
nvs, data, nvs, 0x9000, 0x6000,
8+
phy_init, data, phy, 0xf000, 0x1000,
9+
factory, app, factory, 0x10000, 0x280000,
10+
vfs, data, fat, 0x300000, 0x100000,

‎boards/esp32/MICROLITE_SPIRAM_CAM/main/CMakeLists.txt

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Include MicroPython ESP32 component.
2+
3+
get_filename_component(CURRENT_DIR "." ABSOLUTE)
4+
5+
message(STATUS "microlite/main/cmake: CURRENT_DIR=${CURRENT_DIR}")
6+
7+
get_filename_component(MICROPY_DIR "../../../../micropython" ABSOLUTE)
8+
9+
10+
message (STATUS "microlite/main/cmake: MICROPY_DIR=${MICROPY_DIR}")
11+
12+
set(PROJECT_DIR ${MICROPY_DIR}/ports/esp32)
13+
include(${PROJECT_DIR}/main/CMakeLists.txt)

‎boards/esp32/MICROLITE_SPIRAM_CAM/micropython.patch

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
From d235a3f7b097a0b64fd8ef0f679a88699f6b5f71 Mon Sep 17 00:00:00 2001
2+
From: Michael O'Cleirigh <[email protected]>
3+
Date: Fri, 28 Jan 2022 00:35:53 -0500
4+
Subject: [PATCH] esp32/main: Support SPRAM allocated using malloc.
5+
6+
This change allows the same heap allocation rules to be used when using malloc regardless if the board has
7+
SPRAM or normal RAM.
8+
9+
Integrating with the esp32-camera for example requires that ESP32 SPRAM be allocatable using the esp-idf
10+
capabilities aware allocation functions. In the case of esp32-camera its for the framebuffer.
11+
12+
Detect when CONFIG_SPIRAM_USE_MALLOC is in use and use the standard automatic configuration of leaving 1/2
13+
of the SPRAM available to other FreeRTOS tasks.
14+
---
15+
ports/esp32/main.c | 5 ++++-
16+
1 file changed, 4 insertions(+), 1 deletion(-)
17+
18+
diff --git a/ports/esp32/main.c b/ports/esp32/main.c
19+
index e25e6fdd1..6fe82e40a 100644
20+
--- a/ports/esp32/main.c
21+
+++ b/ports/esp32/main.c
22+
@@ -100,7 +100,10 @@ void mp_task(void *pvParameter) {
23+
size_t mp_task_heap_size;
24+
void *mp_task_heap = NULL;
25+
26+
- #if CONFIG_ESP32_SPIRAM_SUPPORT
27+
+ #if CONFIG_SPIRAM_USE_MALLOC
28+
+ // SPRAM is issued using MALLOC, fallback to normal allocation rules
29+
+ mp_task_heap = NULL;
30+
+ #elif CONFIG_ESP32_SPIRAM_SUPPORT
31+
// Try to use the entire external SPIRAM directly for the heap
32+
mp_task_heap = (void *)SOC_EXTRAM_DATA_LOW;
33+
switch (esp_spiram_get_chip_size()) {
34+
--
35+
2.20.1
36+

‎boards/esp32/MICROLITE_SPIRAM_CAM/mpconfigboard.cmake

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
set (IDF_TARGET esp32)
2+
3+
# this option causes the camera module to build
4+
set(CAMERA_TYPE esp32)
5+
6+
set(SDKCONFIG_DEFAULTS
7+
${MICROPY_PORT_DIR}/boards/sdkconfig.base
8+
${MICROPY_PORT_DIR}/boards/sdkconfig.ble
9+
${MICROPY_BOARD_DIR}/sdkconfig.esp32cam
10+
${MICROPY_BOARD_DIR}/sdkconfig.partition
11+
)
12+
13+
message (STATUS "mpconfigboard.cmake: PROJECT_DIR=${PROJECT_DIR}")
14+
15+
set(USER_C_MODULES
16+
${PROJECT_DIR}/micropython-modules/micropython.cmake
17+
)
18+
19+
20+
if(NOT MICROPY_FROZEN_MANIFEST)
21+
set(MICROPY_FROZEN_MANIFEST ${MICROPY_PORT_DIR}/boards/manifest.py)
22+
endif()

‎boards/esp32/MICROLITE_SPIRAM_CAM/mpconfigboard.h

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#define MICROPY_HW_BOARD_NAME "ESP32-cam module (microlite)"
2+
#define MICROPY_HW_MCU_NAME "ESP32"
3+
4+
#define MICROPY_PY_BLUETOOTH (0)
5+
#define MODULE_CAMERA_ENABLED (1)
6+
7+
// The offset only has an effect if a board has psram
8+
// it allows the start of the range allocated to
9+
#define MICROPY_ALLOCATE_HEAP_USING_MALLOC (1)
10+
#define MICROPY_HEAP_SIZE_REDUCTION (512 * 1024)

‎boards/esp32/MICROLITE_SPIRAM_CAM/sdkconfig.esp32cam

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# MicroPython on ESP32, ESP IDF configuration with SPIRAM support
2+
# The following options override the defaults
3+
4+
# Bootloader config
5+
# CONFIG_LOG_BOOTLOADER_LEVEL_WARN=y
6+
7+
# ESP32-specific
8+
CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
9+
CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU0=n
10+
CONFIG_ESP32_XTAL_FREQ_AUTO=y
11+
CONFIG_ESP32_SPIRAM_SUPPORT=y
12+
13+
# verbose logging for debugging
14+
# CONFIG_LOG_DEFAULT_LEVEL_NONE=
15+
# CONFIG_LOG_DEFAULT_LEVEL_ERROR=
16+
# CONFIG_LOG_DEFAULT_LEVEL_WARN=
17+
# CONFIG_LOG_DEFAULT_LEVEL_INFO=
18+
# CONFIG_LOG_DEFAULT_LEVEL_DEBUG=
19+
# CONFIG_LOG_DEFAULT_LEVEL_VERBOSE=y
20+
# CONFIG_LOG_DEFAULT_LEVEL=5
21+
22+
#
23+
# SPI RAM config
24+
#
25+
CONFIG_SPIRAM_TYPE_AUTO=y
26+
# CONFIG_SPIRAM_TYPE_ESPPSRAM16 is not set
27+
# CONFIG_SPIRAM_TYPE_ESPPSRAM32 is not set
28+
# CONFIG_SPIRAM_TYPE_ESPPSRAM64 is not set
29+
CONFIG_SPIRAM_SIZE=-1
30+
# CONFIG_SPIRAM_SPEED_40M is not set
31+
CONFIG_SPIRAM_SPEED_80M=y
32+
CONFIG_SPIRAM=y
33+
CONFIG_SPIRAM_BOOT_INIT=y
34+
# CONFIG_SPIRAM_IGNORE_NOTFOUND is not set
35+
# CONFIG_SPIRAM_USE_MEMMAP is not set
36+
# CONFIG_SPIRAM_USE_CAPS_ALLOC is not set
37+
38+
# Micropython default is to use CONFIG_SPIRAM_USE_MEMMAP where
39+
# the Micropython Heap is established based on memory address.
40+
# For examples using esp32-camera the code underneath needs to be able to use the
41+
# malloc with capabilities as it needs to be able to get dma memory
42+
# as well as allocate the framebuffer
43+
CONFIG_SPIRAM_USE_MALLOC=y
44+
CONFIG_SPIRAM_MEMTEST=y
45+
# CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384
46+
# CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP is not set
47+
# CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=32768
48+
# CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY is not set
49+
# CONFIG_SPIRAM_ALLOW_NOINIT_SEG_EXTERNAL_MEMORY is not set
50+
CONFIG_SPIRAM_CACHE_WORKAROUND=y
51+
52+
# FreeRTOS
53+
#CONFIG_FREERTOS_UNICORE=y
54+
CONFIG_SUPPORT_STATIC_ALLOCATION=y
55+
CONFIG_ENABLE_STATIC_TASK_CLEAN_UP_HOOK=y
56+
57+
# UDP
58+
CONFIG_PPP_SUPPORT=y
59+
CONFIG_PPP_PAP_SUPPORT=y
60+
CONFIG_PPP_CHAP_SUPPORT=y
61+
62+
# ESP32-CAMERA
63+
CONFIG_OV2640_SUPPORT=y
64+
CONFIG_OV3660_SUPPORT=y

‎boards/esp32/MICROLITE_SPIRAM_CAM/sdkconfig.partition

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#CONFIG_ESPTOOLPY_FLASHMODE_DIO=y
2+
#CONFIG_ESPTOOLPY_FLASHFREQ_40M=y
3+
#CONFIG_ESPTOOLPY_FLASHSIZE_2MB=y
4+
CONFIG_PARTITION_TABLE_CUSTOM=y
5+
# move back up from micropython/ports/esp32 to the main project source code
6+
CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="../../../boards/esp32/MICROLITE/custom-partitions.csv"

‎examples/person_detection/README.md

+16-1
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,14 @@ I was able to restore the tflite model from the person_detection_int8 archive.
2525

2626
I was able to create this (96,96) binary file from the corresponding .cc file from the person_detection_int8 archive.
2727

28+
![person image](images/person.png)
29+
2830
## no_person_image_data.dat
2931

3032
I was able to create this (96,96) binary file from the corresponding .cc file from the person_detection_int8 archive.
3133

34+
![not a person](images/not_a_person.png)
35+
3236
## show-test-images.py
3337

3438
A script that runs in python3 to load and display the two test images.
@@ -55,7 +59,7 @@ setup 9612 bytes on the inputTensor.
5559
'not a person' = -113, 'person' = 113
5660
```
5761

58-
# Running on ESP32 port
62+
# Running on ESP32 port without a camera
5963

6064
**NOTE: The person detection model is 300 kb so you need to use a board with SPI RAM**
6165

@@ -76,3 +80,14 @@ After flashing upload these files from here onto the board:
7680

7781
Then import person_detection to run the example:
7882
![ESP32 Running Person Detection](../../images/person-detection-esp32-running.png)
83+
84+
# Running on an ESP32 With a Camera
85+
86+
This has been tested using an ESP32-CAM-MB and an M5 Timer Camera using the
87+
[MICROLITE_SPIRAM_CAM](../../boards/esp32/MICROLITE_SPIRAM_CAM) Firmware.
88+
89+
Flash that firmware and then copy the esp32-cam/person_detection_cam.py and person_detect_model.tflite
90+
model to the camera.
91+
92+
Run import person_detection_cam to activate the demo program. The led will illuminate
93+
when it thinks a person is in frame.

‎examples/person_detection/esp32-cam/imageTools/getImage.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#!/bin/sh
2+
rshell cp /pyboard/camImage.raw .

‎examples/person_detection/esp32-cam/imageTools/readImage.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Reads a 96x96 pixel gray scale image from the camera in raw mode
2+
# This image can directly be passed into the input tensor of the
3+
# person detection model
4+
# The program is part of a course on AI and edge computing at the
5+
# University of Cape Coast, Ghana
6+
# Copyright (c) U. Raich [2022]
7+
# The program is released under the MIT License
8+
9+
import sys
10+
import camera
11+
from utime import sleep_ms
12+
13+
try:
14+
# this is for the esp32-cam-mb using ov2640 sensor
15+
camera.init(0,format=camera.GRAYSCALE,framesize=camera.FRAME_96X96)
16+
# switch to this for the m5 timer camera with ov3660 sensor
17+
# camera.init(0,format=camera.GRAYSCALE,framesize=camera.FRAME_96X96,
18+
# sioc=23,siod=25,xclk=27,vsync=22,href=26,pclk=21,
19+
# d0=32,d1=35,d2=34,d3=5,d4=39,d5=18,d6=36,d7=19,
20+
# reset=15)
21+
except:
22+
print("Error when initializing the camera")
23+
sys.exit()
24+
25+
buf=camera.capture()
26+
print("type: ", type(buf), " Length: ",len(buf))
27+
# save the raw image to a file
28+
print("Writing the data to camImage.raw")
29+
if len(buf) == 96*96:
30+
f = open("camImage.raw","w+b")
31+
f.write(buf)
32+
f.close()
33+
34+
camera.deinit()

‎examples/person_detection/esp32-cam/imageTools/showGrayScaleImg.py

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/python3
2+
from PIL import Image
3+
import io,sys
4+
5+
if len(sys.argv) != 2:
6+
print("Usage %s filename"%sys.argv[0])
7+
sys.exit()
8+
9+
image_data_file = open (sys.argv[1], 'rb')
10+
image_data = bytearray(9612)
11+
image_data_file.readinto(image_data)
12+
image_data_file.close()
13+
image = Image.frombytes ('L', (96,96), bytes(image_data) ,'raw')
14+
image.show()
15+
16+

0 commit comments

Comments
 (0)