diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index e7207e636ee8..34106c57cc83 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -61,13 +63,13 @@ struct chain_dma_data { /* local host DMA config */ struct sof_dma *dma_host; - struct dma_chan_data *chan_host; + int chan_host_index; struct dma_config z_config_host; struct dma_block_config dma_block_cfg_host; /* local link DMA config */ struct sof_dma *dma_link; - struct dma_chan_data *chan_link; + int chan_link_index; struct dma_config z_config_link; struct dma_block_config dma_block_cfg_link; @@ -79,18 +81,18 @@ static int chain_host_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - if (!cd->chan_host || !cd->chan_host->dma) { + if (cd->chan_host_index < 0 || !cd->dma_host) { comp_err(dev, "incomplete initialization detected, aborting host %p", - cd->chan_host); + cd->dma_host); return -ENODEV; } - err = dma_start(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_start(cd->dma_host, cd->chan_host_index); if (err < 0) return err; - comp_info(dev, "dma_start() host chan_index = %u", - cd->chan_host->index); + comp_info(dev, "dma_start() host chan_index = %d", + cd->chan_host_index); return 0; } @@ -99,12 +101,18 @@ static int chain_link_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_start(cd->chan_link->dma->z_dev, cd->chan_link->index); + if (cd->chan_link_index < 0 || !cd->dma_link) { + comp_err(dev, "incomplete initialization detected, aborting link %p", + cd->dma_link); + return -ENODEV; + } + + err = sof_dma_start(cd->dma_link, cd->chan_link_index); if (err < 0) return err; - comp_info(dev, "dma_start() link chan_index = %u", - cd->chan_link->index); + comp_info(dev, "dma_start() link chan_index = %d", + cd->chan_link_index); return 0; } @@ -113,12 +121,18 @@ static int chain_link_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_link->dma->z_dev, cd->chan_link->index); + if (cd->chan_link_index < 0 || !cd->dma_link) { + comp_err(dev, "incomplete initialization detected, aborting link %p", + cd->dma_link); + return -ENODEV; + } + + err = sof_dma_stop(cd->dma_link, cd->chan_link_index); if (err < 0) return err; - comp_info(dev, "dma_stop() link chan_index = %u", - cd->chan_link->index); + comp_info(dev, "dma_stop() link chan_index = %d", + cd->chan_link_index); return 0; } @@ -128,12 +142,18 @@ static int chain_host_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_host->dma->z_dev, cd->chan_host->index); + if (cd->chan_host_index < 0 || !cd->dma_host) { + comp_err(dev, "incomplete initialization detected, aborting host %p", + cd->dma_host); + return -ENODEV; + } + + err = sof_dma_stop(cd->dma_host, cd->chan_host_index); if (err < 0) return err; - comp_info(dev, "dma_stop() host chan_index = %u", - cd->chan_host->index); + comp_info(dev, "dma_stop() host chan_index = %d", + cd->chan_host_index); return 0; } @@ -171,7 +191,7 @@ static enum task_state chain_task_run(void *data) /* Link DMA can return -EPIPE and current status if xrun occurs, then it is not critical * and flow shall continue. Other error values will be treated as critical. */ - ret = dma_get_status(cd->chan_link->dma->z_dev, cd->chan_link->index, &stat); + ret = sof_dma_get_status(cd->dma_link, cd->chan_link_index, &stat); switch (ret) { case 0: #if CONFIG_XRUN_NOTIFICATIONS_ENABLE @@ -195,7 +215,7 @@ static enum task_state chain_task_run(void *data) link_read_pos = stat.read_position; /* Host DMA does not report xruns. All error values will be treated as critical. */ - ret = dma_get_status(cd->chan_host->dma->z_dev, cd->chan_host->index, &stat); + ret = sof_dma_get_status(cd->dma_host, cd->chan_host_index, &stat); if (ret < 0) { tr_err(&chain_dma_tr, "dma_get_status() error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; @@ -213,14 +233,14 @@ static enum task_state chain_task_run(void *data) */ const size_t increment = MIN(host_free_bytes, link_avail_bytes); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; } - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -236,9 +256,8 @@ static enum task_state chain_task_run(void *data) const size_t half_buff_size = buff_size / 2; if (!cd->first_data_received && host_avail_bytes > half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, - cd->chan_link->index, 0, 0, - MIN(host_avail_bytes, link_free_bytes)); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + MIN(host_avail_bytes, link_free_bytes)); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -252,8 +271,8 @@ static enum task_state chain_task_run(void *data) host_read_pos, buff_size); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, - 0, 0, transferred); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, + transferred); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); @@ -262,8 +281,8 @@ static enum task_state chain_task_run(void *data) if (host_avail_bytes >= half_buff_size && link_free_bytes >= half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, - 0, 0, half_buff_size); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + half_buff_size); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -373,10 +392,21 @@ __cold static void chain_release(struct comp_dev *dev) assert_can_be_cold(); - dma_release_channel(cd->chan_host->dma->z_dev, cd->chan_host->index); - sof_dma_put(cd->dma_host); - dma_release_channel(cd->chan_link->dma->z_dev, cd->chan_link->index); - sof_dma_put(cd->dma_link); + if (cd->dma_host) { + if (cd->chan_host_index >= 0) + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); + sof_dma_put(cd->dma_host); + cd->dma_host = NULL; + cd->chan_host_index = -EINVAL; + } + + if (cd->dma_link) { + if (cd->chan_link_index >= 0) + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); + sof_dma_put(cd->dma_link); + cd->dma_link = NULL; + cd->chan_link_index = -EINVAL; + } if (cd->dma_buffer) { buffer_free(cd->dma_buffer); @@ -463,16 +493,16 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get host DMA channel */ channel = cd->host_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_host->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_host, channel); if (channel < 0) { comp_err(dev, "host dma_request_channel() failed for %u", cd->host_connector_node_id.f.v_index); return channel; } - cd->chan_host = &cd->dma_host->chan[channel]; + cd->chan_host_index = channel; - err = dma_config(cd->dma_host->z_dev, cd->chan_host->index, dma_cfg_host); + err = sof_dma_config(cd->dma_host, cd->chan_host_index, dma_cfg_host); if (err < 0) { comp_err(dev, "host dma_config() failed for %d", channel); goto error_host; @@ -480,7 +510,7 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get link DMA channel */ channel = cd->link_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_link->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_link, channel); if (channel < 0) { comp_err(dev, "link dma_request_channel() failed for %u", cd->link_connector_node_id.f.v_index); @@ -488,9 +518,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) goto error_host; } - cd->chan_link = &cd->dma_link->chan[channel]; + cd->chan_link_index = channel; - err = dma_config(cd->dma_link->z_dev, cd->chan_link->index, dma_cfg_link); + err = sof_dma_config(cd->dma_link, cd->chan_link_index, dma_cfg_link); if (err < 0) { comp_err(dev, "link dma_config() failed for %d", channel); goto error_link; @@ -498,11 +528,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) return 0; error_link: - dma_release_channel(cd->dma_link->z_dev, cd->chan_link->index); - cd->chan_link = NULL; + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); error_host: - dma_release_channel(cd->dma_host->z_dev, cd->chan_host->index); - cd->chan_host = NULL; + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); return err; } @@ -510,6 +538,7 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin uint32_t fifo_size) { struct chain_dma_data *cd = comp_get_drvdata(dev); + struct mod_alloc_ctx *alloc_ctx = NULL; uint32_t addr_align; size_t buff_size; void *buff_addr; @@ -559,8 +588,8 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } /* retrieve DMA buffer address alignment */ - ret = dma_get_attribute(cd->dma_host->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); + ret = sof_dma_get_attribute(cd->dma_host, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, + &addr_align); if (ret < 0) { comp_err(dev, "could not get dma buffer address alignment, err = %d", ret); @@ -588,8 +617,14 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align); + +#ifdef CONFIG_SOF_USERSPACE_LL + alloc_ctx = ipc_get()->ll_alloc; +#endif + /* allocate not shared buffer */ - cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + cd->dma_buffer = buffer_alloc(alloc_ctx, fifo_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); if (!cd->dma_buffer) { @@ -631,6 +666,70 @@ static int chain_task_trigger(struct comp_dev *dev, int cmd) } } +/* + * comp_dev and private data allocation helpers. For user-space LL both + * objects must live on the user heap so the (unprivileged) user LL thread + * can access them; otherwise the normal component/rmalloc paths are used. + */ +#ifdef CONFIG_SOF_USERSPACE_LL +__cold static struct comp_dev *chain_dev_alloc(const struct comp_driver *drv) +{ + struct comp_dev *dev; + + dev = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*dev), 0); + if (!dev) + return NULL; + + memset(dev, 0, sizeof(*dev)); + comp_init(drv, dev, sizeof(*dev)); + + return dev; +} + +__cold static struct chain_dma_data *chain_cd_alloc(void) +{ + struct chain_dma_data *cd; + + cd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, sizeof(*cd), 0); + if (cd) + memset(cd, 0, sizeof(*cd)); + + return cd; +} + +__cold static void chain_dev_free(struct comp_dev *dev) +{ + sof_heap_free(sof_sys_user_heap_get(), dev); +} + +__cold static void chain_cd_free(struct chain_dma_data *cd) +{ + sof_heap_free(sof_sys_user_heap_get(), cd); +} +#else +__cold static struct comp_dev *chain_dev_alloc(const struct comp_driver *drv) +{ + return comp_alloc(drv, sizeof(struct comp_dev)); +} + +__cold static struct chain_dma_data *chain_cd_alloc(void) +{ + return rzalloc(SOF_MEM_FLAG_USER, sizeof(struct chain_dma_data)); +} + +__cold static void chain_dev_free(struct comp_dev *dev) +{ + comp_free_device(dev); +} + +__cold static void chain_cd_free(struct chain_dma_data *cd) +{ + rfree(cd); +} +#endif + __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, const struct comp_ipc_config *ipc_config, const void *ipc_specific_config) @@ -649,17 +748,19 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (host_dma_id >= max_chain_number) return NULL; - dev = comp_alloc(drv, sizeof(*dev)); + dev = chain_dev_alloc(drv); if (!dev) return NULL; - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = chain_cd_alloc(); if (!cd) goto error; cd->first_data_received = false; cd->cs = scs ? 2 : 4; cd->chain_task.state = SOF_TASK_STATE_INIT; + cd->chan_host_index = -EINVAL; + cd->chan_link_index = -EINVAL; comp_set_drvdata(dev, cd); @@ -667,9 +768,9 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (!ret) return dev; - rfree(cd); + chain_cd_free(cd); error: - comp_free_device(dev); + chain_dev_free(dev); return NULL; } @@ -680,8 +781,8 @@ __cold static void chain_task_free(struct comp_dev *dev) assert_can_be_cold(); chain_release(dev); - rfree(cd); - comp_free_device(dev); + chain_cd_free(cd); + chain_dev_free(dev); } static const struct comp_driver comp_chain_dma = {