diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c2ce83c7c1f1..79c40e0abf17 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2070,8 +2070,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) static void io_rings_free(struct io_ring_ctx *ctx) { - io_free_region(ctx->user, &ctx->sq_region); - io_free_region(ctx->user, &ctx->ring_region); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + io_free_region(NULL, &ctx->sq_region); + io_free_region(NULL, &ctx->ring_region); ctx->rings = NULL; RCU_INIT_POINTER(ctx->rings_rcu, NULL); ctx->sq_sqes = NULL; @@ -2735,7 +2736,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret; ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); @@ -2749,7 +2751,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &ctx->sq_region, &rd, IORING_OFF_SQES); + ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES); if (ret) { io_rings_free(ctx); return ret; diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 7c309173dd19..7c59ab9cfc8b 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -676,7 +676,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = reg.ring_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &bl->region, &rd, mmap_offset); + ret = io_create_region(ctx->user, &bl->region, &rd, mmap_offset); if (ret) goto fail; br = io_region_get_ptr(&bl->region); diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 48c0eb012412..d4d716946971 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -204,7 +204,7 @@ done: return 0; } -int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, +int io_create_region(struct user_struct *user, struct io_mapped_region *mr, struct io_uring_region_desc *reg, unsigned long mmap_offset) { @@ -230,8 +230,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, return -EOVERFLOW; nr_pages = reg->size >> PAGE_SHIFT; - if (ctx->user) { - ret = __io_account_mem(ctx->user, nr_pages); + if (user) { + ret = __io_account_mem(user, nr_pages); if (ret) return ret; } @@ -240,7 +240,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, if (reg->flags & IORING_MEM_REGION_TYPE_USER) ret = io_region_pin_pages(mr, reg); else - ret = io_region_allocate_pages(mr, reg, mmap_offset, ctx->user); + ret = io_region_allocate_pages(mr, reg, mmap_offset, user); if (ret) goto out_free; @@ -249,7 +249,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, goto out_free; return 0; out_free: - io_free_region(ctx->user, mr); + io_free_region(user, mr); return ret; } diff --git a/io_uring/memmap.h b/io_uring/memmap.h index f4cfbb6b9a1f..0714bb5a9616 100644 --- a/io_uring/memmap.h +++ b/io_uring/memmap.h @@ -18,7 +18,7 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, int io_uring_mmap(struct file *file, struct vm_area_struct *vma); void io_free_region(struct user_struct *user, struct io_mapped_region *mr); -int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, +int io_create_region(struct user_struct *user, struct io_mapped_region *mr, struct io_uring_region_desc *reg, unsigned long mmap_offset); diff --git a/io_uring/register.c b/io_uring/register.c index 02bc103bcc9d..d11a53d68431 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -480,8 +480,9 @@ struct io_ring_ctx_rings { static void io_register_free_rings(struct io_ring_ctx *ctx, struct io_ring_ctx_rings *r) { - io_free_region(ctx->user, &r->sq_region); - io_free_region(ctx->user, &r->ring_region); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + io_free_region(NULL, &r->sq_region); + io_free_region(NULL, &r->ring_region); } #define swap_old(ctx, o, n, field) \ @@ -529,7 +530,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING); + ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret; @@ -559,7 +560,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES); + ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES); if (ret) { io_register_free_rings(ctx, &n); return ret; @@ -730,7 +731,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) !(ctx->flags & IORING_SETUP_R_DISABLED)) return -EINVAL; - ret = io_create_region(ctx, ®ion, &rd, IORING_MAP_OFF_PARAM_REGION); + ret = io_create_region(ctx->user, ®ion, &rd, + IORING_MAP_OFF_PARAM_REGION); if (ret) return ret; if (copy_to_user(rd_uptr, &rd, sizeof(rd))) { diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index beb35077f3ce..939885985333 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -432,7 +432,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, mmap_offset = IORING_MAP_OFF_ZCRX_REGION; mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT; - ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset); + ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset); if (ret < 0) return ret;