2019-06-01 10:08:55 +02:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-01-22 18:20:36 +00:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2015-01-22 20:52:10 +00:00
|
|
|
|
2015-01-22 18:20:36 +00:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-04-22 18:48:04 +02:00
|
|
|
|
2015-01-22 18:20:36 +00:00
|
|
|
|
|
|
|
|
|
2016-05-31 14:49:01 +01:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2019-08-07 16:55:16 +01:00
|
|
|
|
2016-10-27 09:27:32 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2016-02-05 16:24:48 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2015-10-07 12:00:23 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2025-02-21 10:12:21 +05:30
|
|
|
|
|
|
|
|
|
2024-11-05 10:11:54 +05:30
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-11-21 13:51:32 +00:00
|
|
|
|
2020-03-16 16:50:53 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2019-11-27 09:51:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:19 +00:00
|
|
|
|
2020-02-03 17:36:38 -08:00
|
|
|
|
arm64: mm: dump: log span level
The page table dump code logs spans of entries at the same level
(pgd/pud/pmd/pte) which have the same attributes. While we log the
(decoded) attributes, we don't log the level, which leaves the output
ambiguous and/or confusing in some cases.
For example:
0xffff800800000000-0xffff800980000000 6G RW NX SHD AF BLK UXN MEM/NORMAL
If using 4K pages, this may describe a span of 6 1G block entries at the
PGD/PUD level, or 3072 2M block entries at the PMD level.
This patch adds the page table level to each output line, removing this
ambiguity. For the example above, this will produce:
0xffffffc800000000-0xffffffc980000000 6G PUD RW NX SHD AF BLK UXN MEM/NORMAL
When 3 level tables are in use, and we use the asm-generic/nopud.h
definitions, the dump code treats each entry in the PGD as a 1 element
table at the PUD level, and logs spans as being PUDs, which can be
confusing. To counteract this, the "PUD" mnemonic is replaced with "PGD"
when CONFIG_PGTABLE_LEVELS <= 3. Likewise for "PMD" when
CONFIG_PGTABLE_LEVELS <= 2.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-05-31 14:49:02 +01:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2024-02-14 13:29:24 +01:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-02-14 13:29:24 +01:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
arm64: mm: dump: log span level
The page table dump code logs spans of entries at the same level
(pgd/pud/pmd/pte) which have the same attributes. While we log the
(decoded) attributes, we don't log the level, which leaves the output
ambiguous and/or confusing in some cases.
For example:
0xffff800800000000-0xffff800980000000 6G RW NX SHD AF BLK UXN MEM/NORMAL
If using 4K pages, this may describe a span of 6 1G block entries at the
PGD/PUD level, or 3072 2M block entries at the PMD level.
This patch adds the page table level to each output line, removing this
ambiguity. For the example above, this will produce:
0xffffffc800000000-0xffffffc980000000 6G PUD RW NX SHD AF BLK UXN MEM/NORMAL
When 3 level tables are in use, and we use the asm-generic/nopud.h
definitions, the dump code treats each entry in the PGD as a 1 element
table at the PUD level, and logs spans as being PUDs, which can be
confusing. To counteract this, the "PUD" mnemonic is replaced with "PGD"
when CONFIG_PGTABLE_LEVELS <= 3. Likewise for "PMD" when
CONFIG_PGTABLE_LEVELS <= 2.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Huang Shijie <shijie.huang@arm.com>
Cc: Laura Abbott <labbott@fedoraproject.org>
Cc: Steve Capper <steve.capper@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2016-05-31 14:49:02 +01:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:32 -07:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2025-04-07 11:01:11 +05:30
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2024-09-09 12:47:19 +00:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2026-05-20 07:34:17 +01:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2024-02-14 13:29:24 +01:00
|
|
|
|
2024-09-09 12:47:20 +00:00
|
|
|
|
|
|
|
|
|
2024-02-14 13:29:24 +01:00
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2020-02-03 17:36:38 -08:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:32 -07:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:34 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:34 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2020-02-03 17:36:34 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:32 -07:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:32 -07:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2025-04-07 11:01:11 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
arm64: Enable vmalloc-huge with ptdump
Our goal is to move towards enabling vmalloc-huge by default on arm64 so
as to reduce TLB pressure. Therefore, we need a way to analyze the portion
of block mappings in vmalloc space we can get on a production system; this
can be done through ptdump, but currently we disable vmalloc-huge if
CONFIG_PTDUMP_DEBUGFS is on. The reason is that lazy freeing of kernel
pagetables via vmap_try_huge_pxd() may race with ptdump, so ptdump
may dereference a bogus address.
To solve this, we need to synchronize ptdump_walk() and ptdump_check_wx()
with pud_free_pmd_page() and pmd_free_pte_page().
Since this race is very unlikely to happen in practice, we do not want to
penalize the vmalloc pagetable tearing path by taking the init_mm
mmap_lock. Therefore, we use static keys. ptdump_walk() and
ptdump_check_wx() are the pagetable walkers; they will enable the static
key - upon observing that, the vmalloc pagetable tearing path will get
patched in with an mmap_read_lock/unlock sequence. A combination of the
patched-in mmap_read_lock/unlock, the acquire semantics of
static_branch_inc(), and the barriers in __flush_tlb_kernel_pgtable()
ensures that ptdump will never get a hold on the address of a freed PMD
or PTE table.
We can verify the correctness of the algorithm via the following litmus
test (thanks to James Houghton and Will Deacon):
AArch64 ptdump
Variant=Ifetch
{
uint64_t pud=0xa110c;
uint64_t pmd;
0:X0=label:"P1:L0"; 0:X1=instr:"NOP"; 0:X2=lock; 0:X3=pud; 0:X4=pmd;
1:X1=0xdead; 1:X2=lock; 1:X3=pud; 1:X4=pmd;
}
P0 | P1 ;
(* static_key_enable *) | (* pud_free_pmd_page *) ;
STR W1, [X0] | LDR X9, [X3] ;
DC CVAU,X0 | STR XZR, [X3] ;
DSB ISH | DSB ISH ;
IC IVAU,X0 | ISB ;
DSB ISH | ;
ISB | (* static key *) ;
| L0: ;
(* mmap_lock *) | B out1 ;
Lwlock: | ;
MOV W7, #1 | (* mmap_lock *) ;
SWPA W7, W8, [X2] | Lrlock: ;
| MOV W7, #1 ;
| SWPA W7, W8, [X2] ;
(* walk pgtable *) | ;
LDR X9, [X3] | (* mmap_unlock *) ;
CBZ X9, out0 | STLR WZR, [X2] ;
EOR X10, X9, X9 | ;
LDR X11, [X4, X10] | out1: ;
| EOR X10, X9, X9 ;
out0: | STR X1, [X4, X10] ;
exists (0:X8=0 /\ 1:X8=0 /\ (* Lock acquisitions succeed *)
0:X9=0xa110c /\ (* P0 sees the valid PUD ...*)
0:X11=0xdead) (* ... but the freed PMD *)
For an approximate written proof of why this algorithm works, please read
the code comment in [1], which is now removed for the sake of simplicity.
mm-selftests pass. No issues were observed while parallelly running
test_vmalloc.sh (which stresses the vmalloc subsystem),
and cat /sys/kernel/debug/{kernel_page_tables, check_wx_pages} in a loop.
Link: https://lore.kernel.org/all/20250723161827.15802-1-dev.jain@arm.com/ [1]
Reviewed-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Dev Jain <dev.jain@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2025-09-22 12:11:26 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2016-05-31 14:49:01 +01:00
|
|
|
|
2024-02-14 13:29:24 +01:00
|
|
|
|
2024-09-09 12:47:19 +00:00
|
|
|
|
2021-02-02 23:07:49 +08:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2025-04-07 11:01:11 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
arm64: Enable vmalloc-huge with ptdump
Our goal is to move towards enabling vmalloc-huge by default on arm64 so
as to reduce TLB pressure. Therefore, we need a way to analyze the portion
of block mappings in vmalloc space we can get on a production system; this
can be done through ptdump, but currently we disable vmalloc-huge if
CONFIG_PTDUMP_DEBUGFS is on. The reason is that lazy freeing of kernel
pagetables via vmap_try_huge_pxd() may race with ptdump, so ptdump
may dereference a bogus address.
To solve this, we need to synchronize ptdump_walk() and ptdump_check_wx()
with pud_free_pmd_page() and pmd_free_pte_page().
Since this race is very unlikely to happen in practice, we do not want to
penalize the vmalloc pagetable tearing path by taking the init_mm
mmap_lock. Therefore, we use static keys. ptdump_walk() and
ptdump_check_wx() are the pagetable walkers; they will enable the static
key - upon observing that, the vmalloc pagetable tearing path will get
patched in with an mmap_read_lock/unlock sequence. A combination of the
patched-in mmap_read_lock/unlock, the acquire semantics of
static_branch_inc(), and the barriers in __flush_tlb_kernel_pgtable()
ensures that ptdump will never get a hold on the address of a freed PMD
or PTE table.
We can verify the correctness of the algorithm via the following litmus
test (thanks to James Houghton and Will Deacon):
AArch64 ptdump
Variant=Ifetch
{
uint64_t pud=0xa110c;
uint64_t pmd;
0:X0=label:"P1:L0"; 0:X1=instr:"NOP"; 0:X2=lock; 0:X3=pud; 0:X4=pmd;
1:X1=0xdead; 1:X2=lock; 1:X3=pud; 1:X4=pmd;
}
P0 | P1 ;
(* static_key_enable *) | (* pud_free_pmd_page *) ;
STR W1, [X0] | LDR X9, [X3] ;
DC CVAU,X0 | STR XZR, [X3] ;
DSB ISH | DSB ISH ;
IC IVAU,X0 | ISB ;
DSB ISH | ;
ISB | (* static key *) ;
| L0: ;
(* mmap_lock *) | B out1 ;
Lwlock: | ;
MOV W7, #1 | (* mmap_lock *) ;
SWPA W7, W8, [X2] | Lrlock: ;
| MOV W7, #1 ;
| SWPA W7, W8, [X2] ;
(* walk pgtable *) | ;
LDR X9, [X3] | (* mmap_unlock *) ;
CBZ X9, out0 | STLR WZR, [X2] ;
EOR X10, X9, X9 | ;
LDR X11, [X4, X10] | out1: ;
| EOR X10, X9, X9 ;
out0: | STR X1, [X4, X10] ;
exists (0:X8=0 /\ 1:X8=0 /\ (* Lock acquisitions succeed *)
0:X9=0xa110c /\ (* P0 sees the valid PUD ...*)
0:X11=0xdead) (* ... but the freed PMD *)
For an approximate written proof of why this algorithm works, please read
the code comment in [1], which is now removed for the sake of simplicity.
mm-selftests pass. No issues were observed while parallelly running
test_vmalloc.sh (which stresses the vmalloc subsystem),
and cat /sys/kernel/debug/{kernel_page_tables, check_wx_pages} in a loop.
Link: https://lore.kernel.org/all/20250723161827.15802-1-dev.jain@arm.com/ [1]
Reviewed-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Dev Jain <dev.jain@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2025-09-22 12:11:26 +05:30
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
2021-03-30 13:54:49 +08:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:19 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|
2016-05-31 14:49:01 +01:00
|
|
|
|
2023-12-13 09:40:28 +01:00
|
|
|
|
2016-05-31 14:49:01 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-01-30 11:34:35 +01:00
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
2024-09-09 12:47:18 +00:00
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-09-09 12:47:19 +00:00
|
|
|
|
2020-02-03 17:36:38 -08:00
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2025-04-07 11:01:11 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2024-02-14 13:29:23 +01:00
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
arm64: Enable vmalloc-huge with ptdump
Our goal is to move towards enabling vmalloc-huge by default on arm64 so
as to reduce TLB pressure. Therefore, we need a way to analyze the portion
of block mappings in vmalloc space we can get on a production system; this
can be done through ptdump, but currently we disable vmalloc-huge if
CONFIG_PTDUMP_DEBUGFS is on. The reason is that lazy freeing of kernel
pagetables via vmap_try_huge_pxd() may race with ptdump, so ptdump
may dereference a bogus address.
To solve this, we need to synchronize ptdump_walk() and ptdump_check_wx()
with pud_free_pmd_page() and pmd_free_pte_page().
Since this race is very unlikely to happen in practice, we do not want to
penalize the vmalloc pagetable tearing path by taking the init_mm
mmap_lock. Therefore, we use static keys. ptdump_walk() and
ptdump_check_wx() are the pagetable walkers; they will enable the static
key - upon observing that, the vmalloc pagetable tearing path will get
patched in with an mmap_read_lock/unlock sequence. A combination of the
patched-in mmap_read_lock/unlock, the acquire semantics of
static_branch_inc(), and the barriers in __flush_tlb_kernel_pgtable()
ensures that ptdump will never get a hold on the address of a freed PMD
or PTE table.
We can verify the correctness of the algorithm via the following litmus
test (thanks to James Houghton and Will Deacon):
AArch64 ptdump
Variant=Ifetch
{
uint64_t pud=0xa110c;
uint64_t pmd;
0:X0=label:"P1:L0"; 0:X1=instr:"NOP"; 0:X2=lock; 0:X3=pud; 0:X4=pmd;
1:X1=0xdead; 1:X2=lock; 1:X3=pud; 1:X4=pmd;
}
P0 | P1 ;
(* static_key_enable *) | (* pud_free_pmd_page *) ;
STR W1, [X0] | LDR X9, [X3] ;
DC CVAU,X0 | STR XZR, [X3] ;
DSB ISH | DSB ISH ;
IC IVAU,X0 | ISB ;
DSB ISH | ;
ISB | (* static key *) ;
| L0: ;
(* mmap_lock *) | B out1 ;
Lwlock: | ;
MOV W7, #1 | (* mmap_lock *) ;
SWPA W7, W8, [X2] | Lrlock: ;
| MOV W7, #1 ;
| SWPA W7, W8, [X2] ;
(* walk pgtable *) | ;
LDR X9, [X3] | (* mmap_unlock *) ;
CBZ X9, out0 | STLR WZR, [X2] ;
EOR X10, X9, X9 | ;
LDR X11, [X4, X10] | out1: ;
| EOR X10, X9, X9 ;
out0: | STR X1, [X4, X10] ;
exists (0:X8=0 /\ 1:X8=0 /\ (* Lock acquisitions succeed *)
0:X9=0xa110c /\ (* P0 sees the valid PUD ...*)
0:X11=0xdead) (* ... but the freed PMD *)
For an approximate written proof of why this algorithm works, please read
the code comment in [1], which is now removed for the sake of simplicity.
mm-selftests pass. No issues were observed while parallelly running
test_vmalloc.sh (which stresses the vmalloc subsystem),
and cat /sys/kernel/debug/{kernel_page_tables, check_wx_pages} in a loop.
Link: https://lore.kernel.org/all/20250723161827.15802-1-dev.jain@arm.com/ [1]
Reviewed-by: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Dev Jain <dev.jain@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>
2025-09-22 12:11:26 +05:30
|
|
|
|
2020-02-03 17:36:29 -08:00
|
|
|
|
2024-01-30 11:34:35 +01:00
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
2024-01-30 11:34:35 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
2024-01-30 11:34:35 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2016-10-27 09:27:34 -07:00
|
|
|
|
|
|
|
|
|
2021-03-30 13:54:49 +08:00
|
|
|
|
2016-05-31 14:49:01 +01:00
|
|
|
|
2023-12-13 09:40:29 +01:00
|
|
|
|
|
|
|
|
|
2023-12-13 09:40:28 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-12-22 12:02:06 -08:00
|
|
|
|
2023-12-13 09:40:28 +01:00
|
|
|
|
|
|
|
|
|
2019-08-07 16:55:16 +01:00
|
|
|
|
2023-12-13 09:40:28 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2023-12-13 09:40:29 +01:00
|
|
|
|
2023-12-13 09:40:28 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2024-02-14 13:29:23 +01:00
|
|
|
|
2023-12-13 09:40:28 +01:00
|
|
|
|
2016-10-27 09:27:31 -07:00
|
|
|
|
2019-01-22 15:41:11 +01:00
|
|
|
|
|
|
|
|
|
2016-05-31 14:49:01 +01:00
|
|
|
|
2014-11-26 00:28:39 +00:00
|
|
|
|