From 5d6ed6b6a00b1f058780780fc8a840d141a5f323 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Mon, 27 Apr 2026 19:33:15 -0400 Subject: [PATCH] drm/amdgpu: Rework VMID reservation logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the VMID reservation logic by passing in the mask of VMIDs (for both GFXHUB and MMHUB) available for it to use. The VMID masks for both GFXHUB and MMHUB are initialized in the respective ASIC specific files. This also helps in reserving per-ASIC VMIDs. This is in preparation for reserving a VMID for NPA added in the next patch. Signed-off-by: Mukul Joshi Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c | 33 ++++++++++++++----------- drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h | 27 +++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 4 +++ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 5 ++++ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c | 14 +++++++---- drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 6 +++++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 6 +++++ 17 files changed, 95 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 8a2d64f0ebc3..8c32c6943764 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -625,12 +625,27 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev) for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[i]; - - for (j = 1; j < id_mgr->num_ids; ++j) + for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) amdgpu_vmid_reset(adev, i, j); } } +void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev, + unsigned long vmid_mask, bool for_mmhub) +{ + unsigned int i; + + BUILD_BUG_ON(AMDGPU_NUM_VMID > BITS_PER_LONG); + + for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) { + bool is_mmhub = AMDGPU_IS_MMHUB0(i) || AMDGPU_IS_MMHUB1(i); + + if (is_mmhub == for_mmhub) + bitmap_copy(adev->vm_manager.id_mgr[i].vmid_mask, + &vmid_mask, AMDGPU_NUM_VMID); + } +} + /** * amdgpu_vmid_mgr_init - init the VMID manager * @@ -649,19 +664,8 @@ void amdgpu_vmid_mgr_init(struct amdgpu_device *adev) mutex_init(&id_mgr->lock); INIT_LIST_HEAD(&id_mgr->ids_lru); - /* for GC <10, SDMA uses MMHUB so use first_kfd_vmid for both GC and MM */ - if (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 0, 0)) - /* manage only VMIDs not used by KFD */ - id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; - else if (AMDGPU_IS_MMHUB0(i) || - AMDGPU_IS_MMHUB1(i)) - id_mgr->num_ids = 16; - else - /* manage only VMIDs not used by KFD */ - id_mgr->num_ids = adev->vm_manager.first_kfd_vmid; - /* skip over VMID 0, since it is the system VM */ - for (j = 1; j < id_mgr->num_ids; ++j) { + for_each_set_bit(j, id_mgr->vmid_mask, AMDGPU_NUM_VMID) { amdgpu_vmid_reset(adev, i, j); amdgpu_sync_create(&id_mgr->ids[j].active); list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru); @@ -685,6 +689,7 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) &adev->vm_manager.id_mgr[i]; mutex_destroy(&id_mgr->lock); + for (j = 0; j < AMDGPU_NUM_VMID; ++j) { struct amdgpu_vmid *id = &id_mgr->ids[j]; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h index 4b55d0d9703a..46b2c2160126 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include "amdgpu_sync.h" @@ -33,6 +35,27 @@ /* maximum number of VMIDs */ #define AMDGPU_NUM_VMID 16 +/** + * for_each_vmid_and_zero - iterate over VMID 0 plus every VMID set in a hub's + * vmid_mask + * @vmid: loop cursor (int / unsigned int) + * @adev: struct amdgpu_device * + * @hub: vmhub index into adev->vm_manager.id_mgr[] + * + * Equivalent to OR-ing BIT(0) into id_mgr[hub].vmid_mask and walking the + * result with for_each_set_bit(). Used by the gfx_v*_constants_init() paths + * which must always touch VMID 0 (system) in addition to the regular VMIDs + * owned by the hub. + * + * Relies on AMDGPU_NUM_VMID (16) fitting in a single unsigned long, i.e. + * vmid_mask[] being a 1-element bitmap. + */ +#define for_each_vmid_and_zero(vmid, adev, hub) \ + for (unsigned long __vmid_mask = \ + (adev)->vm_manager.id_mgr[(hub)].vmid_mask[0] | BIT(0); \ + __vmid_mask; __vmid_mask = 0) \ + for_each_set_bit((vmid), &__vmid_mask, AMDGPU_NUM_VMID) + struct amdgpu_device; struct amdgpu_fpriv; struct amdgpu_vm; @@ -65,10 +88,10 @@ struct amdgpu_vmid { struct amdgpu_vmid_mgr { struct mutex lock; - unsigned num_ids; struct list_head ids_lru; struct amdgpu_vmid ids[AMDGPU_NUM_VMID]; bool reserved_vmid; + DECLARE_BITMAP(vmid_mask, AMDGPU_NUM_VMID); }; int amdgpu_pasid_alloc(unsigned int bits, struct amdgpu_fpriv *fpriv); @@ -95,5 +118,7 @@ void amdgpu_vmid_reset_all(struct amdgpu_device *adev); void amdgpu_vmid_mgr_init(struct amdgpu_device *adev); void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev); +void amdgpu_vmid_mgr_set_vmid_mask(struct amdgpu_device *adev, + unsigned long vmid_mask, bool for_mmhub); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 5033f85d3102..045b9c7d2a83 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -5368,7 +5368,7 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { nv_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 2c160ec9dc57..a7ed4923857b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -2217,7 +2217,7 @@ static void gfx_v11_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc21_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index 4d5f14a37be3..ec1e142549c5 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -1856,7 +1856,7 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc24_grbm_select(adev, 0, 0, 0, i); /* CP and shaders */ WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index c8a0ca2c3de0..f812d6d81e9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -1730,7 +1730,7 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev, /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ WREG32_SOC15(GC, GET_INST(GC, xcc_id), diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 755371b66574..549d1239e326 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -1938,7 +1938,7 @@ static void gfx_v7_0_constants_init(struct amdgpu_device *adev) WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { + for_each_vmid_and_zero(i, adev, 0) { if (i == 0) sh_mem_base = 0; else diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 63288382bcf4..950874ef8d04 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -3765,7 +3765,7 @@ static void gfx_v8_0_constants_init(struct amdgpu_device *adev) WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { + for_each_vmid_and_zero(i, adev, 0) { vi_srbm_select(adev, 0, 0, 0, i); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 0dae8ff9c762..3ba2584e48c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -2686,7 +2686,7 @@ static void gfx_v9_0_constants_init(struct amdgpu_device *adev) /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc15_grbm_select(adev, 0, 0, 0, i, 0); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index e1603563c49d..ce48da12edb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -1241,7 +1241,7 @@ static void gfx_v9_4_3_xcc_constants_init(struct amdgpu_device *adev, /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); - for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { + for_each_vmid_and_zero(i, adev, AMDGPU_GFXHUB(0)) { soc15_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); /* CP and shaders */ if (i == 0) { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 75a552782c97..9a8a7921ce98 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -869,6 +869,10 @@ static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); amdgpu_vm_manager_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 7fb97be16b7c..bee269a8877a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -855,6 +855,11 @@ static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block) */ adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); + amdgpu_vm_manager_init(adev); r = amdgpu_gmc_ras_sw_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index 18511e7995ef..3ae3ba42c6b0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -957,13 +957,17 @@ static int gmc_v12_0_sw_init(struct amdgpu_ip_block *ip_block) * number of VMs * VMID 0 is reserved for System * amdgpu graphics/compute will use VMIDs 1-7 - * amdkfd will use VMIDs 8-15 + * amdkfd will use VMIDs 8-15. + * On GFX 12.1, amdkfd will use VMIDs 3-15. */ adev->vm_manager.first_kfd_vmid = - amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ? - 3 : 8; - adev->vm_manager.first_kfd_vmid = - adev->gfx.disable_kq ? 1 : (adev->vm_manager.first_kfd_vmid); + adev->gfx.disable_kq ? 1 : + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) ? + 3 : 8); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); amdgpu_vm_manager_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c index ca7afb9b7d0a..cdd11587b12a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c @@ -864,6 +864,12 @@ static int gmc_v6_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index 1c1eccfe383f..c9cc55e18893 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -1045,6 +1045,12 @@ static int gmc_v7_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index bf089595e5da..8d8a0775878f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -1122,6 +1122,12 @@ static int gmc_v8_0_sw_init(struct amdgpu_ip_block *ip_block) * amdkfd will use VMIDs 8-15 */ adev->vm_manager.first_kfd_vmid = 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); /* base offset of vram pages */ diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index b46b87291c51..633772fa71fd 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -2038,6 +2038,12 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block) 3 : 8; + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + false); + amdgpu_vmid_mgr_set_vmid_mask(adev, + GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), + true); amdgpu_vm_manager_init(adev); gmc_v9_0_save_registers(adev);