From 5c0958d80190822c4614ca00478d9acd9671bba6 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:37 -0600 Subject: [PATCH] ublk: add UBLK_F_IO_DESC_SIZE ublk passes the parameters of incoming I/O in memory shared between the kernel ublk driver and userspace ublk server in struct ublksrv_io_desc. The size of this struct is currently fixed to 24 bytes, which has been an obstacle to extending it with additional fields [1]. Additionally, with multiple ublk server threads handling I/Os from the same ublk queue (possible with UBLK_F_PER_IO_DAEMON or UBLK_F_BATCH_IO), false sharing results from adjacent io_descs sharing the same cache line. Add a ublk feature UBLK_F_IO_DESC_SIZE to allow a ublk server to override the size of each io_desc. The size must be at least 24 and a multiple of 8 to store a properly-aligned struct ublksrv_io_desc. It's also limited to a maximum of 256, though this bound could be lifted in the future. The struct ublksrv_io_desc is located at the beginning of each io_desc and the remainder is padding. The mmap() performed for each queue must have a length of queue_depth * io_desc_size rounded up to the page size. The mmap() offset must be q_id * UBLK_MAX_QUEUE_DEPTH * io_desc_size, also rounded up to the page size. [1]: https://lore.kernel.org/linux-block/aV8QfvaNO5P6vOs6@fedora/ Suggested-by: Ming Lei Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260803211441.2538144-6-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 38 ++++++++++++++++++++-------- include/uapi/linux/ublk_cmd.h | 5 +++- tools/testing/selftests/ublk/kublk.c | 1 + 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 369757f5af08..08d29c9ab898 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -89,7 +89,8 @@ | UBLK_F_SAFE_STOP_DEV \ | UBLK_F_BATCH_IO \ | UBLK_F_NO_AUTO_PART_SCAN \ - | UBLK_F_SHMEM_ZC) + | UBLK_F_SHMEM_ZC \ + | UBLK_F_IO_DESC_SIZE) #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ | UBLK_F_USER_RECOVERY_REISSUE \ @@ -107,6 +108,8 @@ UBLK_BATCH_F_HAS_BUF_ADDR | \ UBLK_BATCH_F_AUTO_BUF_REG_FALLBACK) +#define UBLK_MAX_IO_DESC_SIZE 256 + /* ublk batch fetch uring_cmd */ struct ublk_batch_fetch_cmd { struct list_head node; @@ -239,6 +242,7 @@ struct ublk_io { struct ublk_queue { u16 q_id; u16 q_depth; + u16 io_desc_size; unsigned long flags; struct ublksrv_io_desc *io_cmd_buf; @@ -405,7 +409,7 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q) static inline struct ublksrv_io_desc * ublk_get_iod(const struct ublk_queue *ubq, u16 tag) { - return &ubq->io_cmd_buf[tag]; + return (void *)ubq->io_cmd_buf + tag * (size_t)ubq->io_desc_size; } static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq) @@ -1248,19 +1252,20 @@ ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id) return ublk_get_queue(ub, q_id)->io_cmd_buf; } -static inline int __ublk_queue_cmd_buf_size(u16 depth) +static inline size_t __ublk_queue_cmd_buf_size(const struct ublk_device *ub, + u16 depth) { - return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE); + return round_up(depth * (size_t)ub->dev_info.io_desc_size, PAGE_SIZE); } -static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub) +static inline size_t ublk_queue_cmd_buf_size(const struct ublk_device *ub) { - return __ublk_queue_cmd_buf_size(ub->dev_info.queue_depth); + return __ublk_queue_cmd_buf_size(ub, ub->dev_info.queue_depth); } -static int ublk_max_cmd_buf_size(void) +static size_t ublk_max_cmd_buf_size(const struct ublk_device *ub) { - return __ublk_queue_cmd_buf_size(UBLK_MAX_QUEUE_DEPTH); + return __ublk_queue_cmd_buf_size(ub, UBLK_MAX_QUEUE_DEPTH); } /* @@ -2662,7 +2667,7 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) { struct ublk_device *ub = filp->private_data; size_t sz = vma->vm_end - vma->vm_start; - unsigned max_sz = ublk_max_cmd_buf_size(); + size_t max_sz = ublk_max_cmd_buf_size(ub); unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT; int ret = 0; u16 q_id; @@ -4188,7 +4193,7 @@ static const struct file_operations ublk_ch_batch_io_fops = { static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq) { - int size; + size_t size; u16 i; size = ublk_queue_cmd_buf_size(ub); @@ -4241,7 +4246,8 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id) struct ublk_queue *ubq; struct page *page; int numa_node; - int size, ret; + size_t size; + int ret; u16 i; /* Determine NUMA node based on queue's CPU affinity */ @@ -4266,6 +4272,7 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id) return -ENOMEM; } ubq->io_cmd_buf = page_address(page); + ubq->io_desc_size = ub->dev_info.io_desc_size; for (i = 0; i < ubq->q_depth; i++) spin_lock_init(&ubq->ios[i].lock); @@ -4750,6 +4757,15 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) if (info.flags & UBLK_F_INTEGRITY && !(info.flags & UBLK_F_USER_COPY)) return -EINVAL; + if (info.flags & UBLK_F_IO_DESC_SIZE) { + if (info.io_desc_size < sizeof(struct ublksrv_io_desc) || + info.io_desc_size % _Alignof(struct ublksrv_io_desc) || + info.io_desc_size > UBLK_MAX_IO_DESC_SIZE) + return -EINVAL; + } else { + info.io_desc_size = sizeof(struct ublksrv_io_desc); + } + /* the created device is always owned by current user */ ublk_store_owner_uid_gid(&info.owner_uid, &info.owner_gid); diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h index 6991370a72ce..33b25dd13965 100644 --- a/include/uapi/linux/ublk_cmd.h +++ b/include/uapi/linux/ublk_cmd.h @@ -417,6 +417,9 @@ struct ublk_shmem_buf_reg { */ #define UBLK_F_SHMEM_ZC (1ULL << 19) +/* ublksrv_io_desc size is specified by ublksrv_ctrl_dev_info's io_desc_size */ +#define UBLK_F_IO_DESC_SIZE (1ULL << 20) + /* device state */ #define UBLK_S_DEV_DEAD 0 #define UBLK_S_DEV_LIVE 1 @@ -452,7 +455,7 @@ struct ublksrv_ctrl_dev_info { __u16 nr_hw_queues; __u16 queue_depth; __u16 state; - __u16 pad0; + __u16 io_desc_size; __u32 max_io_buf_bytes; __u32 dev_id; diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 0b23c09daea5..5c4a1f18d0a3 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -1970,6 +1970,7 @@ static int cmd_dev_get_features(void) FEAT_NAME(UBLK_F_BATCH_IO), FEAT_NAME(UBLK_F_NO_AUTO_PART_SCAN), FEAT_NAME(UBLK_F_SHMEM_ZC), + FEAT_NAME(UBLK_F_IO_DESC_SIZE), }; struct ublk_dev *dev; __u64 features = 0;