[userspace LL] vregion related syscalls

posix/include/rtos/alloc.h

@@ -126,6 +126,8 @@ struct vregion;
 struct mod_alloc_ctx {
 	struct k_heap *heap;
 	struct vregion *vreg;
+	uintptr_t vreg_start;
+	size_t vreg_size;
 };
 
 /**

src/audio/buffers/comp_buffer.c

@@ -11,6 +11,7 @@
 #include <sof/audio/sink_api.h>
 #include <sof/audio/source_api.h>
 #include <sof/audio/sink_source_utils.h>
+#include <sof/audio/module_adapter/module/generic.h>
 #include <rtos/userspace_helper.h>
 #include <sof/common.h>
 #include <rtos/interrupt.h>
@@ -165,8 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer)
 
 	if (alloc && alloc->vreg) {
 		vregion_free(alloc->vreg, buffer);
-		if (!vregion_put(alloc->vreg))
-			rfree(alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(alloc ? alloc->heap : NULL, buffer);
 	}

src/audio/module_adapter/module_adapter.c

@@ -58,14 +58,79 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv,
 #define PAGE_SZ HOST_PAGE_SIZE
 #endif
 
-static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config,
-						  size_t *heap_size)
+struct vregion *z_impl_module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size)
 {
 	/* src-lite with 8 channels has been seen allocating 14k in one go */
 	/* FIXME: the size will be derived from configuration */
 	const size_t buf_size = 28 * 1024;
+	struct vregion *vr = vregion_create(buf_size);
 
-	return vregion_create(buf_size);
+	if (!vr)
+		return NULL;
+
+#ifdef CONFIG_SOF_USERSPACE_LL
+	vregion_mem_info(vr, vreg_size, vreg_start);
+
+	/*
+	 * In the userspace LL case allocations are also performed by the
+	 * userspace IPC thread, which is also the one, executing this syscall
+	 */
+	struct k_mem_partition part = {
+		.start = *vreg_start,
+		.size = *vreg_size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB,
+	};
+	int ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part);
+
+	if (ret < 0) {
+		vregion_put(vr);
+		return NULL;
+	}
+
+	part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start);
+	part.attr = K_MEM_PARTITION_P_RW_U_RW;
+
+	ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &part);
+	if (ret < 0) {
+		vregion_put(vr);
+		return NULL;
+	}
+#else
+	(void)vreg_start;
+	(void)vreg_size;
+#endif
+
+	return vr;
+}
+
+void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc)
+{
+#ifdef CONFIG_SOF_USERSPACE_LL
+	struct k_mem_partition part = {
+		.start = alloc->vreg_start,
+		.size = alloc->vreg_size,
+		.attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB,
+	};
+
+	k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part);
+
+	part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start);
+	part.attr = K_MEM_PARTITION_P_RW_U_RW;
+
+	k_mem_domain_remove_partition(zephyr_ll_mem_domain(), &part);
+#else
+	(void)alloc;
+#endif
+}
+
+void module_adapter_vreg_free(struct mod_alloc_ctx *alloc)
+{
+	if (vregion_put(alloc->vreg))
+		return;
+
+	module_adapter_vreg_unmap(alloc);
+
+	sof_heap_free(alloc->heap, alloc);
 }
 
 static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv,
@@ -84,11 +149,12 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	 */
 	uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ?
 		SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER;
-	size_t heap_size;
+	size_t vreg_size;
+	uintptr_t vreg_start;
 
 	if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) &&
 	    IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) {
-		mod_vreg = module_adapter_dp_heap_new(config, &heap_size);
+		mod_vreg = module_adapter_vreg_new(&vreg_start, &vreg_size);
 		if (!mod_vreg) {
 			comp_cl_err(drv, "Failed to allocate DP module heap / vregion");
 			return NULL;
@@ -105,7 +171,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 #else
 		mod_heap = drv->user_heap;
 #endif
-		heap_size = 0;
+		vreg_size = 0;
+		vreg_start = 0;
 		mod_vreg = NULL;
 	}
 
@@ -129,6 +196,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	memset(mod, 0, sizeof(*mod));
 	alloc->heap = mod_heap;
 	alloc->vreg = mod_vreg;
+	alloc->vreg_start = vreg_start;
+	alloc->vreg_size = vreg_size;
 	mod->priv.resources.alloc = alloc;
 	mod_resource_init(mod);
 
@@ -169,6 +238,23 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv
 	return NULL;
 }
 
+#ifdef CONFIG_USERSPACE
+#include <zephyr/internal/syscall_handler.h>
+struct vregion *z_vrfy_module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size)
+{
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start)));
+	K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size)));
+	return z_impl_module_adapter_vreg_new(vreg_start, vreg_size);
+}
+#include <zephyr/syscalls/module_adapter_vreg_new_mrsh.c>
+void z_vrfy_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc)
+{
+	K_OOPS(K_SYSCALL_MEMORY_READ(alloc, sizeof(*alloc)));
+	z_impl_module_adapter_vreg_unmap(alloc);
+}
+#include <zephyr/syscalls/module_adapter_vreg_unmap_mrsh.c>
+#endif
+
 static void module_adapter_mem_free(struct processing_module *mod)
 {
 	struct mod_alloc_ctx *alloc = mod->priv.resources.alloc;
@@ -186,8 +272,7 @@ static void module_adapter_mem_free(struct processing_module *mod)
 
 		vregion_free(mod_vreg, mod->dev);
 		vregion_free(mod_vreg, mod);
-		if (!vregion_put(mod_vreg))
-			sof_heap_free(alloc->heap, alloc);
+		module_adapter_vreg_free(alloc);
 	} else {
 		sof_heap_free(mod_heap, mod->dev);
 		sof_heap_free(mod_heap, mod);

src/include/sof/audio/module_adapter/module/generic.h

@@ -192,16 +192,23 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t
 #endif
 void mod_resource_init(struct processing_module *mod);
 void mod_heap_info(struct processing_module *mod, size_t *size, uintptr_t *start);
+void module_adapter_vreg_free(struct mod_alloc_ctx *alloc);
 #if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION)
+__syscall struct vregion *module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size);
+__syscall void module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc);
 __syscall void *mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
 			      size_t alignment);
 __syscall int mod_free(struct processing_module *mod, const void *ptr);
 __syscall void mod_free_all(struct processing_module *mod);
 #else
+struct vregion *z_impl_module_adapter_vreg_new(uintptr_t *vreg_start, size_t *vreg_size);
+void z_impl_module_adapter_vreg_unmap(const struct mod_alloc_ctx *alloc);
 void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t size,
 			   size_t alignment);
 int z_impl_mod_free(struct processing_module *mod, const void *ptr);
 void z_impl_mod_free_all(struct processing_module *mod);
+#define module_adapter_vreg_new z_impl_module_adapter_vreg_new
+#define module_adapter_vreg_unmap z_impl_module_adapter_vreg_unmap
 #define mod_alloc_ext z_impl_mod_alloc_ext
 #define mod_free z_impl_mod_free
 #define mod_free_all z_impl_mod_free_all

src/include/sof/lib/vregion.h

@@ -6,6 +6,8 @@
 #define __SOF_LIB_VREGION_H__
 
 #include <stddef.h>
+#include <stdint.h>
+#include <sof/compiler_attributes.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -49,7 +51,7 @@ struct vregion *vregion_create(size_t memsize);
  *
  * @param[in] vr Pointer to the virtual region instance.
  */
-void vregion_set_interim(struct vregion *vr);
+__syscall void vregion_set_interim(struct vregion *vr);
 
 /**
  * @brief Increment virtual region's user count.
@@ -60,7 +62,7 @@ void vregion_set_interim(struct vregion *vr);
  * @param[in] vr Pointer to the virtual region instance to release.
  * @return struct vregion* Pointer to the virtual region instance.
  */
-struct vregion *vregion_get(struct vregion *vr);
+__syscall struct vregion *vregion_get(struct vregion *vr);
 
 /**
  * @brief Decrement virtual region's user count or destroy it.
@@ -71,7 +73,7 @@ struct vregion *vregion_get(struct vregion *vr);
  * @param[in] vr Pointer to the virtual region instance to release.
  * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed.
  */
-struct vregion *vregion_put(struct vregion *vr);
+__syscall struct vregion *vregion_put(struct vregion *vr);
 
 /**
  * @brief Allocate memory from the specified virtual region.
@@ -80,12 +82,16 @@ struct vregion *vregion_put(struct vregion *vr);
  * @param[in] size Size of memory to allocate in bytes.
  * @return void* Pointer to the allocated memory, or NULL on failure.
  */
-void *vregion_alloc(struct vregion *vr, size_t size);
+__syscall void *vregion_alloc(struct vregion *vr, size_t size);
+
+void *z_impl_vregion_alloc(struct vregion *vr, size_t size);
 
 /**
  * @brief like vregion_alloc() but allocates coherent memory
  */
-void *vregion_alloc_coherent(struct vregion *vr, size_t size);
+__syscall void *vregion_alloc_coherent(struct vregion *vr, size_t size);
+
+void *z_impl_vregion_alloc_coherent(struct vregion *vr, size_t size);
 
 /**
  * @brief Allocate aligned memory from the specified virtual region.
@@ -98,12 +104,16 @@ void *vregion_alloc_coherent(struct vregion *vr, size_t size);
  * @param[in] alignment Alignment of memory to allocate in bytes.
  * @return void* Pointer to the allocated memory, or NULL on failure.
  */
-void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment);
+__syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment);
+
+void *z_impl_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment);
 
 /**
  * @brief like vregion_alloc_align() but allocates coherent memory
  */
-void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment);
+__syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment);
+
+void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment);
 
 /**
  * @brief Free memory allocated from the specified virtual region.
@@ -113,7 +123,9 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align
  * @param[in] vr Pointer to the virtual region instance.
  * @param[in] ptr Pointer to the memory to free.
  */
-void vregion_free(struct vregion *vr, void *ptr);
+__syscall void vregion_free(struct vregion *vr, void *ptr);
+
+void z_impl_vregion_free(struct vregion *vr, void *ptr);
 
 /**
  * @brief Log virtual region memory usage.
@@ -131,6 +143,8 @@ void vregion_info(struct vregion *vr);
  */
 void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start);
 
+#include <zephyr/syscalls/vregion.h>
+
 #else /* CONFIG_SOF_VREGIONS */
 
 struct vregion {

test/cmocka/src/audio/volume/CMakeLists.txt

@@ -42,6 +42,7 @@ add_library(audio_for_volume STATIC
 	${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-stream.c
 	${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-xrun.c
 	${PROJECT_SOURCE_DIR}/src/audio/component.c
+	${PROJECT_SOURCE_DIR}/src/audio/data_blob.c
 	${PROJECT_SOURCE_DIR}/src/math/numbers.c
 )
 sof_append_relative_path_definitions(audio_for_volume)

test/cmocka/src/common_mocks.c

@@ -141,6 +141,11 @@ void WEAK sof_heap_free(struct k_heap *heap, void *addr)
 	free(addr);
 }
 
+void WEAK module_adapter_vreg_free(struct mod_alloc_ctx *alloc)
+{
+	(void)alloc;
+}
+
 int WEAK memcpy_s(void *dest, size_t dest_size,
 		  const void *src, size_t count)
 {

zephyr/CMakeLists.txt

@@ -629,6 +629,8 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h)
 zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h)
 zephyr_syscall_header(include/rtos/alloc.h)
 zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c)
+zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h)
+zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/vregion.c)
 zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h)
 zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c)
 

zephyr/Kconfig

@@ -36,11 +36,20 @@ config SOF_USERSPACE_INTERFACE_ALLOC
 	  Allow user-space threads to use sof_heap_alloc/sof_heap_free
 	  as Zephyr system calls.
 
+config SOF_USERSPACE_INTERFACE_VREGION
+	bool "Enable SOF vregion interface to userspace threads"
+	depends on USERSPACE
+	depends on SOF_VREGIONS
+	help
+	  Allow user-space threads to use vregion_alloc/vregion_free
+	  and their variants as Zephyr system calls.
+
 config SOF_USERSPACE_LL
 	bool "Run Low-Latency pipelines in userspace threads"
 	depends on USERSPACE
 	select SOF_USERSPACE_INTERFACE_ALLOC
 	select SOF_USERSPACE_INTERFACE_DMA
+	select SOF_USERSPACE_INTERFACE_VREGION if SOF_VREGIONS
 	help
 	  Run Low-Latency (LL) pipelines in userspace threads. This adds
 	  memory protection between operating system resources and

zephyr/include/rtos/alloc.h

@@ -167,6 +167,8 @@ size_t get_shared_buffer_heap_size(void);
 struct mod_alloc_ctx {
 	struct k_heap *heap;
 	struct vregion *vreg;
+	uintptr_t vreg_start;
+	size_t vreg_size;
 };
 
 /**