From 00c8c713c0fbce90a53290b01c2a1fb2d5ba6009 Mon Sep 17 00:00:00 2001 From: Vladimir Smitka Date: Sat, 12 Sep 2026 17:05:58 +0000 Subject: [PATCH] storage: map_file - read a file straight from memory-mapped flash storage.map_file(f) returns a tuple of read-only memoryviews over the flash bytes of an open file on the internal CIRCUITPY drive, one per contiguous cluster run, in file order and 0 copy. Anything that takes a buffer can then use the file without reading it into RAM: a synthio.MidiTrack, a RawSample, a wavetable, a ulab array, a bitmap. Assets stay ordinary files on the drive. Opening a file for reading already builds its FatFs cluster-link map, so the function only reads that map. The supervisor maps a FatFs sector to a flash address through a port hook that also reports how far the mapping stays contiguous, so a run is split where it is not. raspberrypi returns the execute-in-place address (the drive is XIP on every RP2 board); espressif esp_partition_mmap's each drive partition on first use and reports the seam of an extended drive. The function is always present; on a port whose drive is not mapped (CIRCUITPY_STORAGE_MAP_FILE off) it raises NotImplementedError. 600 B of text on pajenicko_picopad, 496 B on adafruit_feather_esp32s3_tft. --- ports/espressif/mpconfigport.mk | 3 + ports/espressif/supervisor/internal_flash.c | 37 ++++++++++ ports/raspberrypi/mpconfigport.mk | 2 + ports/raspberrypi/supervisor/internal_flash.c | 8 +++ py/circuitpy_mpconfig.mk | 5 ++ shared-bindings/storage/__init__.c | 24 +++++++ shared-bindings/storage/__init__.h | 2 + shared-module/storage/__init__.c | 68 +++++++++++++++++++ supervisor/flash.h | 6 ++ supervisor/shared/flash.c | 18 +++++ supervisor/shared/internal_flash.h | 6 ++ 11 files changed, 179 insertions(+) diff --git a/ports/espressif/mpconfigport.mk b/ports/espressif/mpconfigport.mk index ee28bba9918..524b0995d54 100644 --- a/ports/espressif/mpconfigport.mk +++ b/ports/espressif/mpconfigport.mk @@ -466,3 +466,6 @@ endif # Usually lots of flash space available CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1 + +# The CIRCUITPY partition is mapped into the data address space on first use +CIRCUITPY_STORAGE_MAP_FILE ?= 1 diff --git a/ports/espressif/supervisor/internal_flash.c b/ports/espressif/supervisor/internal_flash.c index a18d3fe0a0f..cebeb4f89bb 100644 --- a/ports/espressif/supervisor/internal_flash.c +++ b/ports/espressif/supervisor/internal_flash.c @@ -80,6 +80,43 @@ uint32_t supervisor_flash_get_block_count(void) { void port_internal_flash_flush(void) { } +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: each drive partition mapped into the data address space on first use. With +// extended storage the drive spans two partitions that need not be adjacent in the mapping, so +// *contiguous stops at the seam and the caller splits a cluster run there. +static const uint8_t *map_partition(size_t i) { + static const uint8_t *base[2]; + if (base[i] == NULL) { + const void *p; + esp_partition_mmap_handle_t handle; // stays mapped for the session + if (esp_partition_mmap(_partition[i], 0, _partition[i]->size, ESP_PARTITION_MMAP_DATA, + &p, &handle) != ESP_OK) { + return NULL; + } + base[i] = p; + } + return base[i]; +} + +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + size_t i = 0; + uint32_t blocks = _partition[0]->size / FILESYSTEM_BLOCK_SIZE; + #if CIRCUITPY_STORAGE_EXTEND + if (storage_extended && block >= blocks) { + block -= blocks; + blocks = _partition[1]->size / FILESYSTEM_BLOCK_SIZE; + i = 1; + } + #endif + const uint8_t *base = map_partition(i); + if (base == NULL) { + return NULL; + } + *contiguous = blocks - block; + return base + block * FILESYSTEM_BLOCK_SIZE; +} +#endif + static void single_partition_rw(const esp_partition_t *partition, uint8_t *data, const uint32_t offset, const uint32_t size_total, const bool op) { if (op == OP_READ) { diff --git a/ports/raspberrypi/mpconfigport.mk b/ports/raspberrypi/mpconfigport.mk index 551ab00ab47..851e2398b2b 100644 --- a/ports/raspberrypi/mpconfigport.mk +++ b/ports/raspberrypi/mpconfigport.mk @@ -26,6 +26,8 @@ CIRCUITPY_PWMIO ?= 1 CIRCUITPY_RGBMATRIX ?= $(CIRCUITPY_DISPLAYIO) CIRCUITPY_ROTARYIO ?= 1 CIRCUITPY_ROTARYIO_SOFTENCODER = 1 +# The CIRCUITPY drive is execute-in-place on every RP2 board +CIRCUITPY_STORAGE_MAP_FILE ?= 1 CIRCUITPY_SYNTHIO_MAX_CHANNELS = 24 CIRCUITPY_USB_HOST ?= 1 CIRCUITPY_USB_VIDEO ?= 1 diff --git a/ports/raspberrypi/supervisor/internal_flash.c b/ports/raspberrypi/supervisor/internal_flash.c index 07b15564c60..e9da8a39a14 100644 --- a/ports/raspberrypi/supervisor/internal_flash.c +++ b/ports/raspberrypi/supervisor/internal_flash.c @@ -149,3 +149,11 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32 void supervisor_flash_release_cache(void) { } + +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: the CIRCUITPY drive is execute-in-place on every RP2 board. +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + *contiguous = supervisor_flash_get_block_count() - block; + return (const uint8_t *)(XIP_BASE + CIRCUITPY_CIRCUITPY_DRIVE_START_ADDR + block * FILESYSTEM_BLOCK_SIZE); +} +#endif diff --git a/py/circuitpy_mpconfig.mk b/py/circuitpy_mpconfig.mk index 178f424d5c0..8df24c26cd4 100755 --- a/py/circuitpy_mpconfig.mk +++ b/py/circuitpy_mpconfig.mk @@ -625,6 +625,11 @@ CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE) CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK) CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND) +# storage.map_file(): read a file straight out of memory-mapped flash. Ports whose CIRCUITPY drive +# is memory-mapped implement port_internal_flash_xip_address() and turn it on. +CIRCUITPY_STORAGE_MAP_FILE ?= 0 +CFLAGS += -DCIRCUITPY_STORAGE_MAP_FILE=$(CIRCUITPY_STORAGE_MAP_FILE) + CIRCUITPY_STRUCT ?= 1 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT) diff --git a/shared-bindings/storage/__init__.c b/shared-bindings/storage/__init__.c index b5f19529be1..b0e36dc27f3 100644 --- a/shared-bindings/storage/__init__.c +++ b/shared-bindings/storage/__init__.c @@ -300,6 +300,29 @@ static mp_obj_t storage_enable_usb_drive(void) { } MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive); + +//| def map_file(file: typing.BinaryIO) -> Tuple[memoryview, ...]: +//| """Map an open file on the internal CIRCUITPY drive straight from flash, without copying it +//| into RAM. Returns one read-only `memoryview` per contiguous run of the file's clusters, in +//| file order: a contiguous file gives a 1-tuple, an empty file an empty tuple. Run boundaries +//| fall on cluster boundaries. The views stay valid after the file is closed, and anything that +//| takes a buffer can use them: a `synthio.MidiTrack`, an `audiocore.RawSample`, a wavetable. +//| +//| Treat the file as read-only while a view is in use: rewriting it shows the new bytes, and +//| mid-write it shows a torn file. +//| +//| :param typing.BinaryIO file: A file on the CIRCUITPY drive open for binary reading (``"rb"``) +//| :raises OSError: ``EINVAL`` if the file is closed or not open for reading only, +//| ``EOPNOTSUPP`` if it is on another mount or this build cannot map the drive, +//| ``EIO`` if its cluster chain is corrupt +//| :raises ~builtins.MemoryError: if ``open()`` could not allocate the file's cluster map +//| :raises NotImplementedError: on a port whose drive is not memory-mapped""" +//| ... +static mp_obj_t storage_map_file(mp_obj_t file_in) { + return common_hal_storage_map_file(file_in); +} +MP_DEFINE_CONST_FUN_OBJ_1(storage_map_file_obj, storage_map_file); + static const mp_rom_map_elem_t storage_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_storage) }, @@ -307,6 +330,7 @@ static const mp_rom_map_elem_t storage_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&storage_umount_obj) }, { MP_ROM_QSTR(MP_QSTR_remount), MP_ROM_PTR(&storage_remount_obj) }, { MP_ROM_QSTR(MP_QSTR_getmount), MP_ROM_PTR(&storage_getmount_obj) }, + { MP_ROM_QSTR(MP_QSTR_map_file), MP_ROM_PTR(&storage_map_file_obj) }, { MP_ROM_QSTR(MP_QSTR_erase_filesystem), MP_ROM_PTR(&storage_erase_filesystem_obj) }, { MP_ROM_QSTR(MP_QSTR_disable_usb_drive), MP_ROM_PTR(&storage_disable_usb_drive_obj) }, { MP_ROM_QSTR(MP_QSTR_enable_usb_drive), MP_ROM_PTR(&storage_enable_usb_drive_obj) }, diff --git a/shared-bindings/storage/__init__.h b/shared-bindings/storage/__init__.h index 0e53c78b153..12d13c2ccfc 100644 --- a/shared-bindings/storage/__init__.h +++ b/shared-bindings/storage/__init__.h @@ -21,3 +21,5 @@ MP_NORETURN void common_hal_storage_erase_filesystem(bool extended); bool common_hal_storage_disable_usb_drive(void); bool common_hal_storage_unsafe_disable_usb_drive(void); bool common_hal_storage_enable_usb_drive(void); + +mp_obj_t common_hal_storage_map_file(mp_obj_t file); diff --git a/shared-module/storage/__init__.c b/shared-module/storage/__init__.c index 1522136332a..8fe286dfb34 100644 --- a/shared-module/storage/__init__.c +++ b/shared-module/storage/__init__.c @@ -18,6 +18,12 @@ #include "shared-bindings/storage/__init__.h" #include "supervisor/filesystem.h" #include "supervisor/flash.h" +#if CIRCUITPY_STORAGE_MAP_FILE +#include "extmod/vfs_fat.h" +#include "lib/oofatfs/ff.h" +#include "py/objarray.h" +#include "py/objlist.h" +#endif #if CIRCUITPY_USB_DEVICE #include "supervisor/usb.h" @@ -233,3 +239,65 @@ void common_hal_storage_erase_filesystem(bool extended) { common_hal_mcu_reset(); // We won't actually get here, since we're resetting. } + +mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) { + #if CIRCUITPY_STORAGE_MAP_FILE + pyb_file_obj_t *file = MP_OBJ_TO_PTR(mp_arg_validate_type(file_in, &mp_type_vfs_fat_fileio, MP_QSTR_file)); + FATFS *fatfs = file->fp.obj.fs; + if (fatfs == NULL || (file->fp.flag & FA_WRITE)) { + mp_raise_OSError(MP_EINVAL); // closed, or not open for reading only + } + fs_user_mount_t *drive = filesystem_circuitpy(); + if (drive == NULL || fatfs != &drive->fatfs) { + mp_raise_OSError(MP_EOPNOTSUPP); // another mount (an SD card): not memory-mapped + } + if (file->fp.err) { + mp_raise_OSError(fresult_to_errno_table[file->fp.err]); // open() hit a bad cluster chain + } + DWORD *tbl = file->fp.cltbl; + if (tbl == NULL) { + mp_raise_type(&mp_type_MemoryError); // open() could not allocate the cluster map + } + FSIZE_t size = f_size(&file->fp); + if (size == 0) { + return mp_const_empty_tuple; + } + supervisor_flash_flush(); // write back any RAM sector cache + // Walk the cluster runs of the link map open() built and split a run wherever the port's + // mapping is not contiguous, e.g. at the seam of a drive that spans two flash partitions. + mp_obj_t views = mp_obj_new_list(0, NULL); + size_t nruns = (size_t)((tbl[0] - 2) / 2); + FSIZE_t left = size; + for (size_t i = 0; i < nruns && left > 0; i++) { + DWORD sector = fatfs->database + (tbl[2 + 2 * i] - 2) * fatfs->csize; + FSIZE_t span = (FSIZE_t)tbl[1 + 2 * i] * fatfs->csize * FF_MIN_SS; + if (span > left) { + span = left; // the last run is clipped to the dir-entry size + } + while (span > 0) { + uint32_t contiguous; + const uint8_t *addr = supervisor_flash_xip_address(sector, &contiguous); + if (addr == NULL) { + mp_raise_OSError(MP_EOPNOTSUPP); // this build cannot map the drive here + } + FSIZE_t piece = (FSIZE_t)contiguous * FF_MIN_SS; + if (piece > span) { + piece = span; + } + // read-only: a stray write raises instead of silently hitting the flash window + mp_obj_list_append(views, mp_obj_new_memoryview('B', (size_t)piece, (void *)addr)); + span -= piece; + left -= piece; + sector += contiguous; + } + } + if (left != 0) { + mp_raise_OSError(MP_EIO); // chain shorter than the dir-entry size: corrupt + } + mp_obj_list_t *list = MP_OBJ_TO_PTR(views); + return mp_obj_new_tuple(list->len, list->items); + #else + (void)file_in; + mp_raise_type(&mp_type_NotImplementedError); // this port does not map its drive + #endif +} diff --git a/supervisor/flash.h b/supervisor/flash.h index ca3e42735ff..fc37bf5f319 100644 --- a/supervisor/flash.h +++ b/supervisor/flash.h @@ -29,6 +29,12 @@ void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs); void supervisor_flash_flush(void); void supervisor_flash_release_cache(void); +#if CIRCUITPY_STORAGE_MAP_FILE +// storage.map_file: flash address of a CIRCUITPY FatFs sector, NULL if the port cannot map the +// drive there; *contiguous receives how many sectors from it are contiguous in the mapping. +const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous); +#endif + void supervisor_flash_set_extended(bool extended); bool supervisor_flash_get_extended(void); void supervisor_flash_update_extended(void); diff --git a/supervisor/shared/flash.c b/supervisor/shared/flash.c index ccb60efe321..135ae923c24 100644 --- a/supervisor/shared/flash.c +++ b/supervisor/shared/flash.c @@ -149,6 +149,24 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo return supervisor_flash_read_blocks(dest, block_num, num_blocks); } +#if CIRCUITPY_STORAGE_MAP_FILE +MP_WEAK const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) { + (void)block; + (void)contiguous; + return NULL; // this port does not map its drive +} + +// The sector -> block translation of flash_read_blocks(); FatFs has already validated the +// cluster chain, so every sector it hands over lies inside the volume. +const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous) { + uint32_t block = fatfs_sector - PART1_START_BLOCK; + #if CIRCUITPY_SAVES_PARTITION_SIZE > 0 + block += CIRCUITPY_SAVES_PARTITION_SIZE / FILESYSTEM_BLOCK_SIZE; + #endif + return port_internal_flash_xip_address(block, contiguous); +} +#endif + static volatile bool filesystem_dirty = false; static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32_t block_num, uint32_t num_blocks) { diff --git a/supervisor/shared/internal_flash.h b/supervisor/shared/internal_flash.h index 139e86cf49b..dd18f0100f1 100644 --- a/supervisor/shared/internal_flash.h +++ b/supervisor/shared/internal_flash.h @@ -8,3 +8,9 @@ #include "supervisor/internal_flash.h" // This is per-port. void port_internal_flash_flush(void); + +#if CIRCUITPY_STORAGE_MAP_FILE +// The memory-mapped address of a drive block, NULL if the port cannot map it; *contiguous +// receives how many blocks from it are contiguous in the mapping. Weak NULL default. +const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous); +#endif