diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index 69c9ef441fc1..add671363c82 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -637,6 +637,140 @@ static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) NUM_EXPECTED_SYNCS(2)); } +static void arm_smmu_v3_invs_test_verify(struct kunit *test, + struct arm_smmu_invs *invs, + int num_invs, const int num_trashes, + const int *ids, const int *users, + const int *ssids) +{ + KUNIT_EXPECT_EQ(test, invs->num_invs, num_invs); + KUNIT_EXPECT_EQ(test, invs->num_trashes, num_trashes); + while (num_invs--) { + KUNIT_EXPECT_EQ(test, invs->inv[num_invs].id, ids[num_invs]); + KUNIT_EXPECT_EQ(test, READ_ONCE(invs->inv[num_invs].users), + users[num_invs]); + KUNIT_EXPECT_EQ(test, invs->inv[num_invs].ssid, ssids[num_invs]); + } +} + +static struct arm_smmu_invs invs1 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, + { .type = INV_TYPE_S2_VMID_S1_CLEAR, .id = 1, }, + { .type = INV_TYPE_ATS, .id = 3, }, }, +}; + +static struct arm_smmu_invs invs2 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .type = INV_TYPE_ATS, .id = 4, }, + { .type = INV_TYPE_ATS, .id = 5, }, }, +}; + +static struct arm_smmu_invs invs3 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_S2_VMID, .id = 1, }, /* duplicated */ + { .type = INV_TYPE_ATS, .id = 5, }, /* recover a trash */ + { .type = INV_TYPE_ATS, .id = 6, }, }, +}; + +static struct arm_smmu_invs invs4 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 1 }, + { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, + { .type = INV_TYPE_ATS, .id = 12, .ssid = 1 }, }, +}; + +static struct arm_smmu_invs invs5 = { + .num_invs = 3, + .inv = { { .type = INV_TYPE_ATS, .id = 10, .ssid = 2 }, + { .type = INV_TYPE_ATS, .id = 10, .ssid = 3 }, /* duplicate */ + { .type = INV_TYPE_ATS, .id = 12, .ssid = 2 }, }, +}; + +static void arm_smmu_v3_invs_test(struct kunit *test) +{ + const int results1[3][3] = { { 1, 1, 3, }, { 1, 1, 1, }, { 0, 0, 0, } }; + const int results2[3][5] = { { 1, 1, 3, 4, 5, }, { 2, 1, 1, 1, 1, }, { 0, 0, 0, 0, 0, } }; + const int results3[3][3] = { { 1, 1, 3, }, { 1, 1, 1, }, { 0, 0, 0, } }; + const int results4[3][5] = { { 1, 1, 3, 5, 6, }, { 2, 1, 1, 1, 1, }, { 0, 0, 0, 0, 0, } }; + const int results5[3][5] = { { 1, 1, 3, 5, 6, }, { 1, 0, 0, 1, 1, }, { 0, 0, 0, 0, 0, } }; + const int results6[3][3] = { { 1, 5, 6, }, { 1, 1, 1, }, { 0, 0, 0, } }; + const int results7[3][3] = { { 10, 10, 12, }, { 1, 1, 1, }, { 1, 3, 1, } }; + const int results8[3][5] = { { 10, 10, 10, 12, 12, }, { 1, 1, 2, 1, 1, }, { 1, 2, 3, 1, 2, } }; + const int results9[3][4] = { { 10, 10, 10, 12, }, { 1, 0, 1, 1, }, { 1, 2, 3, 1, } }; + const int results10[3][3] = { { 10, 10, 12, }, { 1, 1, 1, }, { 1, 3, 1, } }; + struct arm_smmu_invs *test_a, *test_b; + + /* New array */ + test_a = arm_smmu_invs_alloc(0); + KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + + /* Test1: merge invs1 (new array) */ + test_b = arm_smmu_invs_merge(test_a, &invs1); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results1[0]), 0, + results1[0], results1[1], results1[2]); + + /* Test2: merge invs2 (new array) */ + test_a = arm_smmu_invs_merge(test_b, &invs2); + kfree(test_b); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results2[0]), 0, + results2[0], results2[1], results2[2]); + + /* Test3: unref invs2 (same array) */ + arm_smmu_invs_unref(test_a, &invs2); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results3[0]), 0, + results3[0], results3[1], results3[2]); + + /* Test4: merge invs3 (new array) */ + test_b = arm_smmu_invs_merge(test_a, &invs3); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results4[0]), 0, + results4[0], results4[1], results4[2]); + + /* Test5: unref invs1 (same array) */ + arm_smmu_invs_unref(test_b, &invs1); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results5[0]), 2, + results5[0], results5[1], results5[2]); + + /* Test6: purge test_b (new array) */ + test_a = arm_smmu_invs_purge(test_b); + kfree(test_b); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results6[0]), 0, + results6[0], results6[1], results6[2]); + + /* Test7: unref invs3 (same array) */ + arm_smmu_invs_unref(test_a, &invs3); + KUNIT_EXPECT_EQ(test, test_a->num_invs, 0); + KUNIT_EXPECT_EQ(test, test_a->num_trashes, 0); + + /* Test8: merge invs4 (new array) */ + test_b = arm_smmu_invs_merge(test_a, &invs4); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results7[0]), 0, + results7[0], results7[1], results7[2]); + + /* Test9: merge invs5 (new array) */ + test_a = arm_smmu_invs_merge(test_b, &invs5); + kfree(test_b); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results8[0]), 0, + results8[0], results8[1], results8[2]); + + /* Test10: unref invs5 (same array) */ + arm_smmu_invs_unref(test_a, &invs5); + arm_smmu_v3_invs_test_verify(test, test_a, ARRAY_SIZE(results9[0]), 1, + results9[0], results9[1], results9[2]); + + /* Test11: purge test_a (new array) */ + test_b = arm_smmu_invs_purge(test_a); + kfree(test_a); + arm_smmu_v3_invs_test_verify(test, test_b, ARRAY_SIZE(results10[0]), 0, + results10[0], results10[1], results10[2]); + + kfree(test_b); +} + static struct kunit_case arm_smmu_v3_test_cases[] = { KUNIT_CASE(arm_smmu_v3_write_ste_test_bypass_to_abort), KUNIT_CASE(arm_smmu_v3_write_ste_test_abort_to_bypass), @@ -662,6 +796,7 @@ static struct kunit_case arm_smmu_v3_test_cases[] = { KUNIT_CASE(arm_smmu_v3_write_ste_test_nested_s1bypass_to_s1dssbypass), KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_clear), KUNIT_CASE(arm_smmu_v3_write_cd_test_sva_release), + KUNIT_CASE(arm_smmu_v3_invs_test), {}, }; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 375f2143efc1..10050b1a6d94 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -1026,6 +1027,267 @@ static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused */ } +/* Invalidation array manipulation functions */ +static inline struct arm_smmu_inv * +arm_smmu_invs_iter_next(struct arm_smmu_invs *invs, size_t next, size_t *idx) +{ + while (true) { + if (next >= invs->num_invs) { + *idx = next; + return NULL; + } + if (!READ_ONCE(invs->inv[next].users)) { + next++; + continue; + } + *idx = next; + return &invs->inv[next]; + } +} + +/** + * arm_smmu_invs_for_each_entry - Iterate over all non-trash entries in invs + * @invs: the base invalidation array + * @idx: a stack variable of 'size_t', to store the array index + * @cur: a stack variable of 'struct arm_smmu_inv *' + */ +#define arm_smmu_invs_for_each_entry(invs, idx, cur) \ + for (cur = arm_smmu_invs_iter_next(invs, 0, &(idx)); cur; \ + cur = arm_smmu_invs_iter_next(invs, idx + 1, &(idx))) + +static int arm_smmu_inv_cmp(const struct arm_smmu_inv *inv_l, + const struct arm_smmu_inv *inv_r) +{ + if (inv_l->smmu != inv_r->smmu) + return cmp_int((uintptr_t)inv_l->smmu, (uintptr_t)inv_r->smmu); + if (inv_l->type != inv_r->type) + return cmp_int(inv_l->type, inv_r->type); + if (inv_l->id != inv_r->id) + return cmp_int(inv_l->id, inv_r->id); + if (arm_smmu_inv_is_ats(inv_l)) + return cmp_int(inv_l->ssid, inv_r->ssid); + return 0; +} + +static inline int arm_smmu_invs_iter_next_cmp(struct arm_smmu_invs *invs_l, + size_t next_l, size_t *idx_l, + struct arm_smmu_invs *invs_r, + size_t next_r, size_t *idx_r) +{ + struct arm_smmu_inv *cur_l = + arm_smmu_invs_iter_next(invs_l, next_l, idx_l); + + /* + * We have to update the idx_r manually, because the invs_r cannot call + * arm_smmu_invs_iter_next() as the invs_r never sets any users counter. + */ + *idx_r = next_r; + + /* + * Compare of two sorted arrays items. If one side is past the end of + * the array, return the other side to let it run out the iteration. + * + * If the left entry is empty, return 1 to pick the right entry. + * If the right entry is empty, return -1 to pick the left entry. + */ + if (!cur_l) + return 1; + if (next_r >= invs_r->num_invs) + return -1; + return arm_smmu_inv_cmp(cur_l, &invs_r->inv[next_r]); +} + +/** + * arm_smmu_invs_for_each_cmp - Iterate over two sorted arrays computing for + * arm_smmu_invs_merge() or arm_smmu_invs_unref() + * @invs_l: the base invalidation array + * @idx_l: a stack variable of 'size_t', to store the base array index + * @invs_r: the build_invs array as to_merge or to_unref + * @idx_r: a stack variable of 'size_t', to store the build_invs index + * @cmp: a stack variable of 'int', to store return value (-1, 0, or 1) + */ +#define arm_smmu_invs_for_each_cmp(invs_l, idx_l, invs_r, idx_r, cmp) \ + for (idx_l = idx_r = 0, \ + cmp = arm_smmu_invs_iter_next_cmp(invs_l, 0, &(idx_l), \ + invs_r, 0, &(idx_r)); \ + idx_l < invs_l->num_invs || idx_r < invs_r->num_invs; \ + cmp = arm_smmu_invs_iter_next_cmp( \ + invs_l, idx_l + (cmp <= 0 ? 1 : 0), &(idx_l), \ + invs_r, idx_r + (cmp >= 0 ? 1 : 0), &(idx_r))) + +/** + * arm_smmu_invs_merge() - Merge @to_merge into @invs and generate a new array + * @invs: the base invalidation array + * @to_merge: an array of invalidations to merge + * + * Return: a newly allocated array on success, or ERR_PTR + * + * This function must be locked and serialized with arm_smmu_invs_unref() and + * arm_smmu_invs_purge(), but do not lockdep on any lock for KUNIT test. + * + * Both @invs and @to_merge must be sorted, to ensure the returned array will be + * sorted as well. + * + * Caller is resposible for freeing the @invs and the returned new one. + * + * Entries marked as trash will be purged in the returned array. + */ +VISIBLE_IF_KUNIT +struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_merge) +{ + struct arm_smmu_invs *new_invs; + struct arm_smmu_inv *new; + size_t num_invs = 0; + size_t i, j; + int cmp; + + arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) + num_invs++; + + new_invs = arm_smmu_invs_alloc(num_invs); + if (!new_invs) + return ERR_PTR(-ENOMEM); + + new = new_invs->inv; + arm_smmu_invs_for_each_cmp(invs, i, to_merge, j, cmp) { + if (cmp < 0) { + *new = invs->inv[i]; + } else if (cmp == 0) { + *new = invs->inv[i]; + WRITE_ONCE(new->users, READ_ONCE(new->users) + 1); + } else { + *new = to_merge->inv[j]; + WRITE_ONCE(new->users, 1); + } + + /* + * Check that the new array is sorted. This also validates that + * to_merge is sorted. + */ + if (new != new_invs->inv) + WARN_ON_ONCE(arm_smmu_inv_cmp(new - 1, new) == 1); + if (arm_smmu_inv_is_ats(new)) + new_invs->has_ats = true; + new++; + } + + WARN_ON(new != new_invs->inv + new_invs->num_invs); + + return new_invs; +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_merge); + +/** + * arm_smmu_invs_unref() - Find in @invs for all entries in @to_unref, decrease + * the user counts without deletions + * @invs: the base invalidation array + * @to_unref: an array of invalidations to decrease their user counts + * + * Return: the number of trash entries in the array, for arm_smmu_invs_purge() + * + * This function will not fail. Any entry with users=0 will be marked as trash, + * and caller will be notified about the trashed entry via @to_unref by setting + * a users=0. + * + * All tailing trash entries in the array will be dropped. And the size of the + * array will be trimmed properly. All trash entries in-between will remain in + * the @invs until being completely deleted by the next arm_smmu_invs_merge() + * or an arm_smmu_invs_purge() function call. + * + * This function must be locked and serialized with arm_smmu_invs_merge() and + * arm_smmu_invs_purge(), but do not lockdep on any mutex for KUNIT test. + * + * Note that the final @invs->num_invs might not reflect the actual number of + * invalidations due to trash entries. Any reader should take the read lock to + * iterate each entry and check its users counter till the last entry. + */ +VISIBLE_IF_KUNIT +void arm_smmu_invs_unref(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_unref) +{ + unsigned long flags; + size_t num_invs = 0; + size_t i, j; + int cmp; + + arm_smmu_invs_for_each_cmp(invs, i, to_unref, j, cmp) { + if (cmp < 0) { + /* not found in to_unref, leave alone */ + num_invs = i + 1; + } else if (cmp == 0) { + int users = READ_ONCE(invs->inv[i].users) - 1; + + if (WARN_ON(users < 0)) + continue; + + /* same item */ + WRITE_ONCE(invs->inv[i].users, users); + if (users) { + WRITE_ONCE(to_unref->inv[j].users, 1); + num_invs = i + 1; + continue; + } + + /* Notify the caller about the trash entry */ + WRITE_ONCE(to_unref->inv[j].users, 0); + invs->num_trashes++; + } else { + /* item in to_unref is not in invs or already a trash */ + WARN_ON(true); + } + } + + /* Exclude any tailing trash */ + invs->num_trashes -= invs->num_invs - num_invs; + + /* The lock is required to fence concurrent ATS operations. */ + write_lock_irqsave(&invs->rwlock, flags); + WRITE_ONCE(invs->num_invs, num_invs); /* Remove tailing trash entries */ + write_unlock_irqrestore(&invs->rwlock, flags); +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_unref); + +/** + * arm_smmu_invs_purge() - Purge all the trash entries in the @invs + * @invs: the base invalidation array + * + * Return: a newly allocated array on success removing all the trash entries, or + * NULL if there is no trash entry in the array or if allocation failed + * + * This function must be locked and serialized with arm_smmu_invs_merge() and + * arm_smmu_invs_unref(), but do not lockdep on any lock for KUNIT test. + * + * Caller is resposible for freeing the @invs and the returned new one. + */ +VISIBLE_IF_KUNIT +struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs) +{ + struct arm_smmu_invs *new_invs; + struct arm_smmu_inv *inv; + size_t i, num_invs = 0; + + if (WARN_ON(invs->num_invs < invs->num_trashes)) + return NULL; + if (!invs->num_invs || !invs->num_trashes) + return NULL; + + new_invs = arm_smmu_invs_alloc(invs->num_invs - invs->num_trashes); + if (!new_invs) + return NULL; + + arm_smmu_invs_for_each_entry(invs, i, inv) { + new_invs->inv[num_invs] = *inv; + if (arm_smmu_inv_is_ats(inv)) + new_invs->has_ats = true; + num_invs++; + } + + WARN_ON(num_invs != new_invs->num_invs); + return new_invs; +} +EXPORT_SYMBOL_IF_KUNIT(arm_smmu_invs_purge); + /* Context descriptor manipulation functions */ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) { @@ -2529,13 +2791,21 @@ static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain) struct arm_smmu_domain *arm_smmu_domain_alloc(void) { struct arm_smmu_domain *smmu_domain; + struct arm_smmu_invs *new_invs; smmu_domain = kzalloc_obj(*smmu_domain); if (!smmu_domain) return ERR_PTR(-ENOMEM); + new_invs = arm_smmu_invs_alloc(0); + if (!new_invs) { + kfree(smmu_domain); + return ERR_PTR(-ENOMEM); + } + INIT_LIST_HEAD(&smmu_domain->devices); spin_lock_init(&smmu_domain->devices_lock); + rcu_assign_pointer(smmu_domain->invs, new_invs); return smmu_domain; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index cfbedb76c8ba..ed8820f12ba3 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -648,6 +648,93 @@ struct arm_smmu_cmdq_batch { int num; }; +/* + * The order here also determines the sequence in which commands are sent to the + * command queue. E.g. TLBI must be done before ATC_INV. + */ +enum arm_smmu_inv_type { + INV_TYPE_S1_ASID, + INV_TYPE_S2_VMID, + INV_TYPE_S2_VMID_S1_CLEAR, + INV_TYPE_ATS, + INV_TYPE_ATS_FULL, +}; + +struct arm_smmu_inv { + struct arm_smmu_device *smmu; + u8 type; + u8 size_opcode; + u8 nsize_opcode; + u32 id; /* ASID or VMID or SID */ + union { + size_t pgsize; /* ARM_SMMU_FEAT_RANGE_INV */ + u32 ssid; /* INV_TYPE_ATS */ + }; + + int users; /* users=0 to mark as a trash to be purged */ +}; + +static inline bool arm_smmu_inv_is_ats(const struct arm_smmu_inv *inv) +{ + return inv->type == INV_TYPE_ATS || inv->type == INV_TYPE_ATS_FULL; +} + +/** + * struct arm_smmu_invs - Per-domain invalidation array + * @max_invs: maximum capacity of the flexible array + * @num_invs: number of invalidations in the flexible array. May be smaller than + * @max_invs after a tailing trash entry is excluded, but must not be + * greater than @max_invs + * @num_trashes: number of trash entries in the array for arm_smmu_invs_purge(). + * Must not be greater than @num_invs + * @rwlock: optional rwlock to fench ATS operations + * @has_ats: flag if the array contains an INV_TYPE_ATS or INV_TYPE_ATS_FULL + * @rcu: rcu head for kfree_rcu() + * @inv: flexible invalidation array + * + * The arm_smmu_invs is an RCU data structure. During a ->attach_dev callback, + * arm_smmu_invs_merge(), arm_smmu_invs_unref() and arm_smmu_invs_purge() will + * be used to allocate a new copy of an old array for addition and deletion in + * the old domain's and new domain's invs arrays. + * + * The arm_smmu_invs_unref() mutates a given array, by internally reducing the + * users counts of some given entries. This exists to support a no-fail routine + * like attaching to an IOMMU_DOMAIN_BLOCKED. And it could pair with a followup + * arm_smmu_invs_purge() call to generate a new clean array. + * + * Concurrent invalidation thread will push every invalidation described in the + * array into the command queue for each invalidation event. It is designed like + * this to optimize the invalidation fast path by avoiding locks. + * + * A domain can be shared across SMMU instances. When an instance gets removed, + * it would delete all the entries that belong to that SMMU instance. Then, a + * synchronize_rcu() would have to be called to sync the array, to prevent any + * concurrent invalidation thread accessing the old array from issuing commands + * to the command queue of a removed SMMU instance. + */ +struct arm_smmu_invs { + size_t max_invs; + size_t num_invs; + size_t num_trashes; + rwlock_t rwlock; + bool has_ats; + struct rcu_head rcu; + struct arm_smmu_inv inv[] __counted_by(max_invs); +}; + +static inline struct arm_smmu_invs *arm_smmu_invs_alloc(size_t num_invs) +{ + struct arm_smmu_invs *new_invs; + + new_invs = kzalloc(struct_size(new_invs, inv, num_invs), GFP_KERNEL); + if (!new_invs) + return NULL; + new_invs->max_invs = num_invs; + new_invs->num_invs = num_invs; + rwlock_init(&new_invs->rwlock); + return new_invs; +} + struct arm_smmu_evtq { struct arm_smmu_queue q; struct iopf_queue *iopf; @@ -873,6 +960,8 @@ struct arm_smmu_domain { struct iommu_domain domain; + struct arm_smmu_invs __rcu *invs; + /* List of struct arm_smmu_master_domain */ struct list_head devices; spinlock_t devices_lock; @@ -925,6 +1014,12 @@ void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target, void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, struct arm_smmu_master *master, struct mm_struct *mm, u16 asid); + +struct arm_smmu_invs *arm_smmu_invs_merge(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_merge); +void arm_smmu_invs_unref(struct arm_smmu_invs *invs, + struct arm_smmu_invs *to_unref); +struct arm_smmu_invs *arm_smmu_invs_purge(struct arm_smmu_invs *invs); #endif struct arm_smmu_master_domain { @@ -958,6 +1053,8 @@ struct arm_smmu_domain *arm_smmu_domain_alloc(void); static inline void arm_smmu_domain_free(struct arm_smmu_domain *smmu_domain) { + /* No concurrency with invalidation is possible at this point */ + kfree(rcu_dereference_protected(smmu_domain->invs, true)); kfree(smmu_domain); }