storage: map_file - read a file straight from memory-mapped flash

ports/espressif/mpconfigport.mk

@@ -466,3 +466,6 @@ endif
 
 # Usually lots of flash space available
 CIRCUITPY_MESSAGE_COMPRESSION_LEVEL ?= 1
+
+# The CIRCUITPY partition is mapped into the data address space on first use
+CIRCUITPY_STORAGE_MAP_FILE ?= 1

ports/espressif/supervisor/internal_flash.c

@@ -80,6 +80,43 @@ uint32_t supervisor_flash_get_block_count(void) {
 void port_internal_flash_flush(void) {
 }
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+// storage.map_file: each drive partition mapped into the data address space on first use. With
+// extended storage the drive spans two partitions that need not be adjacent in the mapping, so
+// *contiguous stops at the seam and the caller splits a cluster run there.
+static const uint8_t *map_partition(size_t i) {
+    static const uint8_t *base[2];
+    if (base[i] == NULL) {
+        const void *p;
+        esp_partition_mmap_handle_t handle;                  // stays mapped for the session
+        if (esp_partition_mmap(_partition[i], 0, _partition[i]->size, ESP_PARTITION_MMAP_DATA,
+            &p, &handle) != ESP_OK) {
+            return NULL;
+        }
+        base[i] = p;
+    }
+    return base[i];
+}
+
+const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) {
+    size_t i = 0;
+    uint32_t blocks = _partition[0]->size / FILESYSTEM_BLOCK_SIZE;
+    #if CIRCUITPY_STORAGE_EXTEND
+    if (storage_extended && block >= blocks) {
+        block -= blocks;
+        blocks = _partition[1]->size / FILESYSTEM_BLOCK_SIZE;
+        i = 1;
+    }
+    #endif
+    const uint8_t *base = map_partition(i);
+    if (base == NULL) {
+        return NULL;
+    }
+    *contiguous = blocks - block;
+    return base + block * FILESYSTEM_BLOCK_SIZE;
+}
+#endif
+
 static void single_partition_rw(const esp_partition_t *partition, uint8_t *data,
     const uint32_t offset, const uint32_t size_total, const bool op) {
     if (op == OP_READ) {

ports/raspberrypi/mpconfigport.mk

@@ -26,6 +26,8 @@ CIRCUITPY_PWMIO ?= 1
 CIRCUITPY_RGBMATRIX ?= $(CIRCUITPY_DISPLAYIO)
 CIRCUITPY_ROTARYIO ?= 1
 CIRCUITPY_ROTARYIO_SOFTENCODER = 1
+# The CIRCUITPY drive is execute-in-place on every RP2 board
+CIRCUITPY_STORAGE_MAP_FILE ?= 1
 CIRCUITPY_SYNTHIO_MAX_CHANNELS = 24
 CIRCUITPY_USB_HOST ?= 1
 CIRCUITPY_USB_VIDEO ?= 1

ports/raspberrypi/supervisor/internal_flash.c

@@ -149,3 +149,11 @@ mp_uint_t supervisor_flash_write_blocks(const uint8_t *src, uint32_t lba, uint32
 
 void supervisor_flash_release_cache(void) {
 }
+
+#if CIRCUITPY_STORAGE_MAP_FILE
+// storage.map_file: the CIRCUITPY drive is execute-in-place on every RP2 board.
+const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) {
+    *contiguous = supervisor_flash_get_block_count() - block;
+    return (const uint8_t *)(XIP_BASE + CIRCUITPY_CIRCUITPY_DRIVE_START_ADDR + block * FILESYSTEM_BLOCK_SIZE);
+}
+#endif

py/circuitpy_mpconfig.mk

@@ -625,6 +625,11 @@ CFLAGS += -DCIRCUITPY_STORAGE=$(CIRCUITPY_STORAGE)
 CIRCUITPY_STORAGE_EXTEND ?= $(CIRCUITPY_DUALBANK)
 CFLAGS += -DCIRCUITPY_STORAGE_EXTEND=$(CIRCUITPY_STORAGE_EXTEND)
 
+# storage.map_file(): read a file straight out of memory-mapped flash. Ports whose CIRCUITPY drive
+# is memory-mapped implement port_internal_flash_xip_address() and turn it on.
+CIRCUITPY_STORAGE_MAP_FILE ?= 0
+CFLAGS += -DCIRCUITPY_STORAGE_MAP_FILE=$(CIRCUITPY_STORAGE_MAP_FILE)
+
 CIRCUITPY_STRUCT ?= 1
 CFLAGS += -DCIRCUITPY_STRUCT=$(CIRCUITPY_STRUCT)
 

shared-bindings/storage/__init__.c

@@ -300,13 +300,37 @@ static mp_obj_t storage_enable_usb_drive(void) {
 }
 MP_DEFINE_CONST_FUN_OBJ_0(storage_enable_usb_drive_obj, storage_enable_usb_drive);
 
+
+//| def map_file(file: typing.BinaryIO) -> Tuple[memoryview, ...]:
+//|     """Map an open file on the internal CIRCUITPY drive straight from flash, without copying it
+//|     into RAM. Returns one read-only `memoryview` per contiguous run of the file's clusters, in
+//|     file order: a contiguous file gives a 1-tuple, an empty file an empty tuple. Run boundaries
+//|     fall on cluster boundaries. The views stay valid after the file is closed, and anything that
+//|     takes a buffer can use them: a `synthio.MidiTrack`, an `audiocore.RawSample`, a wavetable.
+//|
+//|     Treat the file as read-only while a view is in use: rewriting it shows the new bytes, and
+//|     mid-write it shows a torn file.
+//|
+//|     :param typing.BinaryIO file: A file on the CIRCUITPY drive open for binary reading (``"rb"``)
+//|     :raises OSError: ``EINVAL`` if the file is closed or not open for reading only,
+//|       ``EOPNOTSUPP`` if it is on another mount or this build cannot map the drive,
+//|       ``EIO`` if its cluster chain is corrupt
+//|     :raises ~builtins.MemoryError: if ``open()`` could not allocate the file's cluster map
+//|     :raises NotImplementedError: on a port whose drive is not memory-mapped"""
+//|     ...
+static mp_obj_t storage_map_file(mp_obj_t file_in) {
+    return common_hal_storage_map_file(file_in);
+}
+MP_DEFINE_CONST_FUN_OBJ_1(storage_map_file_obj, storage_map_file);
+
 static const mp_rom_map_elem_t storage_module_globals_table[] = {
     { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_storage) },
 
     { MP_ROM_QSTR(MP_QSTR_mount),                    MP_ROM_PTR(&storage_mount_obj) },
     { MP_ROM_QSTR(MP_QSTR_umount),                   MP_ROM_PTR(&storage_umount_obj) },
     { MP_ROM_QSTR(MP_QSTR_remount),                  MP_ROM_PTR(&storage_remount_obj) },
     { MP_ROM_QSTR(MP_QSTR_getmount),                 MP_ROM_PTR(&storage_getmount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_map_file),                 MP_ROM_PTR(&storage_map_file_obj) },
     { MP_ROM_QSTR(MP_QSTR_erase_filesystem),         MP_ROM_PTR(&storage_erase_filesystem_obj) },
     { MP_ROM_QSTR(MP_QSTR_disable_usb_drive),        MP_ROM_PTR(&storage_disable_usb_drive_obj) },
     { MP_ROM_QSTR(MP_QSTR_enable_usb_drive),         MP_ROM_PTR(&storage_enable_usb_drive_obj) },

shared-bindings/storage/__init__.h

@@ -21,3 +21,5 @@ MP_NORETURN void common_hal_storage_erase_filesystem(bool extended);
 bool common_hal_storage_disable_usb_drive(void);
 bool common_hal_storage_unsafe_disable_usb_drive(void);
 bool common_hal_storage_enable_usb_drive(void);
+
+mp_obj_t common_hal_storage_map_file(mp_obj_t file);

shared-module/storage/__init__.c

@@ -18,6 +18,12 @@
 #include "shared-bindings/storage/__init__.h"
 #include "supervisor/filesystem.h"
 #include "supervisor/flash.h"
+#if CIRCUITPY_STORAGE_MAP_FILE
+#include "extmod/vfs_fat.h"
+#include "lib/oofatfs/ff.h"
+#include "py/objarray.h"
+#include "py/objlist.h"
+#endif
 
 #if CIRCUITPY_USB_DEVICE
 #include "supervisor/usb.h"
@@ -233,3 +239,65 @@ void common_hal_storage_erase_filesystem(bool extended) {
     common_hal_mcu_reset();
     // We won't actually get here, since we're resetting.
 }
+
+mp_obj_t common_hal_storage_map_file(mp_obj_t file_in) {
+    #if CIRCUITPY_STORAGE_MAP_FILE
+    pyb_file_obj_t *file = MP_OBJ_TO_PTR(mp_arg_validate_type(file_in, &mp_type_vfs_fat_fileio, MP_QSTR_file));
+    FATFS *fatfs = file->fp.obj.fs;
+    if (fatfs == NULL || (file->fp.flag & FA_WRITE)) {
+        mp_raise_OSError(MP_EINVAL);            // closed, or not open for reading only
+    }
+    fs_user_mount_t *drive = filesystem_circuitpy();
+    if (drive == NULL || fatfs != &drive->fatfs) {
+        mp_raise_OSError(MP_EOPNOTSUPP);        // another mount (an SD card): not memory-mapped
+    }
+    if (file->fp.err) {
+        mp_raise_OSError(fresult_to_errno_table[file->fp.err]);   // open() hit a bad cluster chain
+    }
+    DWORD *tbl = file->fp.cltbl;
+    if (tbl == NULL) {
+        mp_raise_type(&mp_type_MemoryError);    // open() could not allocate the cluster map
+    }
+    FSIZE_t size = f_size(&file->fp);
+    if (size == 0) {
+        return mp_const_empty_tuple;
+    }
+    supervisor_flash_flush();                   // write back any RAM sector cache
+    // Walk the cluster runs of the link map open() built and split a run wherever the port's
+    // mapping is not contiguous, e.g. at the seam of a drive that spans two flash partitions.
+    mp_obj_t views = mp_obj_new_list(0, NULL);
+    size_t nruns = (size_t)((tbl[0] - 2) / 2);
+    FSIZE_t left = size;
+    for (size_t i = 0; i < nruns && left > 0; i++) {
+        DWORD sector = fatfs->database + (tbl[2 + 2 * i] - 2) * fatfs->csize;
+        FSIZE_t span = (FSIZE_t)tbl[1 + 2 * i] * fatfs->csize * FF_MIN_SS;
+        if (span > left) {
+            span = left;                        // the last run is clipped to the dir-entry size
+        }
+        while (span > 0) {
+            uint32_t contiguous;
+            const uint8_t *addr = supervisor_flash_xip_address(sector, &contiguous);
+            if (addr == NULL) {
+                mp_raise_OSError(MP_EOPNOTSUPP);    // this build cannot map the drive here
+            }
+            FSIZE_t piece = (FSIZE_t)contiguous * FF_MIN_SS;
+            if (piece > span) {
+                piece = span;
+            }
+            // read-only: a stray write raises instead of silently hitting the flash window
+            mp_obj_list_append(views, mp_obj_new_memoryview('B', (size_t)piece, (void *)addr));
+            span -= piece;
+            left -= piece;
+            sector += contiguous;
+        }
+    }
+    if (left != 0) {
+        mp_raise_OSError(MP_EIO);               // chain shorter than the dir-entry size: corrupt
+    }
+    mp_obj_list_t *list = MP_OBJ_TO_PTR(views);
+    return mp_obj_new_tuple(list->len, list->items);
+    #else
+    (void)file_in;
+    mp_raise_type(&mp_type_NotImplementedError);    // this port does not map its drive
+    #endif
+}

supervisor/flash.h

@@ -29,6 +29,12 @@ void supervisor_flash_init_vfs(struct _fs_user_mount_t *vfs);
 void supervisor_flash_flush(void);
 void supervisor_flash_release_cache(void);
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+// storage.map_file: flash address of a CIRCUITPY FatFs sector, NULL if the port cannot map the
+// drive there; *contiguous receives how many sectors from it are contiguous in the mapping.
+const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous);
+#endif
+
 void supervisor_flash_set_extended(bool extended);
 bool supervisor_flash_get_extended(void);
 void supervisor_flash_update_extended(void);

supervisor/shared/flash.c

@@ -149,6 +149,24 @@ static mp_uint_t flash_read_blocks(mp_obj_t self_in, uint8_t *dest, uint32_t blo
     return supervisor_flash_read_blocks(dest, block_num, num_blocks);
 }
 
+#if CIRCUITPY_STORAGE_MAP_FILE
+MP_WEAK const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous) {
+    (void)block;
+    (void)contiguous;
+    return NULL;                                // this port does not map its drive
+}
+
+// The sector -> block translation of flash_read_blocks(); FatFs has already validated the
+// cluster chain, so every sector it hands over lies inside the volume.
+const uint8_t *supervisor_flash_xip_address(uint32_t fatfs_sector, uint32_t *contiguous) {
+    uint32_t block = fatfs_sector - PART1_START_BLOCK;
+    #if CIRCUITPY_SAVES_PARTITION_SIZE > 0
+    block += CIRCUITPY_SAVES_PARTITION_SIZE / FILESYSTEM_BLOCK_SIZE;
+    #endif
+    return port_internal_flash_xip_address(block, contiguous);
+}
+#endif
+
 static volatile bool filesystem_dirty = false;
 
 static mp_uint_t flash_write_blocks(mp_obj_t self_in, const uint8_t *src, uint32_t block_num, uint32_t num_blocks) {

supervisor/shared/internal_flash.h

@@ -8,3 +8,9 @@
 #include "supervisor/internal_flash.h" // This is per-port.
 
 void port_internal_flash_flush(void);
+
+#if CIRCUITPY_STORAGE_MAP_FILE
+// The memory-mapped address of a drive block, NULL if the port cannot map it; *contiguous
+// receives how many blocks from it are contiguous in the mapping. Weak NULL default.
+const uint8_t *port_internal_flash_xip_address(uint32_t block, uint32_t *contiguous);
+#endif