Skip to content

Commit 84e6ffb

Browse files
rppttorvalds
rppt
authored and
torvalds
committed
arm: add support for folded p4d page tables
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate, and remove __ARCH_USE_5LEVEL_HACK. [[email protected]: fix kexec] Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Mike Rapoport <[email protected]> Signed-off-by: Andrew Morton <[email protected]> Tested-by: Marek Szyprowski <[email protected]> Cc: Arnd Bergmann <[email protected]> Cc: Benjamin Herrenschmidt <[email protected]> Cc: Brian Cain <[email protected]> Cc: Catalin Marinas <[email protected]> Cc: Christophe Leroy <[email protected]> Cc: Fenghua Yu <[email protected]> Cc: Geert Uytterhoeven <[email protected]> Cc: Guan Xuetao <[email protected]> Cc: James Morse <[email protected]> Cc: Jonas Bonn <[email protected]> Cc: Julien Thierry <[email protected]> Cc: Ley Foon Tan <[email protected]> Cc: Marc Zyngier <[email protected]> Cc: Michael Ellerman <[email protected]> Cc: Paul Mackerras <[email protected]> Cc: Rich Felker <[email protected]> Cc: Russell King <[email protected]> Cc: Stafford Horne <[email protected]> Cc: Stefan Kristiansson <[email protected]> Cc: Suzuki K Poulose <[email protected]> Cc: Tony Luck <[email protected]> Cc: Will Deacon <[email protected]> Cc: Yoshinori Sato <[email protected]> Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Linus Torvalds <[email protected]>
1 parent f426f4e commit 84e6ffb

File tree

12 files changed

+125
-37
lines changed

12 files changed

+125
-37
lines changed

‎arch/arm/include/asm/pgtable.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
#else
1919

20-
#define __ARCH_USE_5LEVEL_HACK
2120
#include <asm-generic/pgtable-nopud.h>
2221
#include <asm/memory.h>
2322
#include <asm/pgtable-hwdef.h>

‎arch/arm/lib/uaccess_with_memcpy.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
2424
{
2525
unsigned long addr = (unsigned long)_addr;
2626
pgd_t *pgd;
27+
p4d_t *p4d;
2728
pmd_t *pmd;
2829
pte_t *pte;
2930
pud_t *pud;
@@ -33,7 +34,11 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
3334
if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
3435
return 0;
3536

36-
pud = pud_offset(pgd, addr);
37+
p4d = p4d_offset(pgd, addr);
38+
if (unlikely(p4d_none(*p4d) || p4d_bad(*p4d)))
39+
return 0;
40+
41+
pud = pud_offset(p4d, addr);
3742
if (unlikely(pud_none(*pud) || pud_bad(*pud)))
3843
return 0;
3944

‎arch/arm/mach-sa1100/assabet.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -633,7 +633,7 @@ static void __init map_sa1100_gpio_regs( void )
633633
int prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_DOMAIN(DOMAIN_IO);
634634
pmd_t *pmd;
635635

636-
pmd = pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt);
636+
pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt);
637637
*pmd = __pmd(phys | prot);
638638
flush_pmd_entry(pmd);
639639
}

‎arch/arm/mm/dump.c

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ struct pg_level {
207207
static struct pg_level pg_level[] = {
208208
{
209209
}, { /* pgd */
210+
}, { /* p4d */
210211
}, { /* pud */
211212
}, { /* pmd */
212213
.bits = section_bits,
@@ -308,7 +309,7 @@ static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start,
308309

309310
for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
310311
addr = start + i * PAGE_SIZE;
311-
note_page(st, addr, 4, pte_val(*pte), domain);
312+
note_page(st, addr, 5, pte_val(*pte), domain);
312313
}
313314
}
314315

@@ -350,14 +351,14 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
350351
addr += SECTION_SIZE;
351352
pmd++;
352353
domain = get_domain_name(pmd);
353-
note_page(st, addr, 3, pmd_val(*pmd), domain);
354+
note_page(st, addr, 4, pmd_val(*pmd), domain);
354355
}
355356
}
356357
}
357358

358-
static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
359+
static void walk_pud(struct pg_state *st, p4d_t *p4d, unsigned long start)
359360
{
360-
pud_t *pud = pud_offset(pgd, 0);
361+
pud_t *pud = pud_offset(p4d, 0);
361362
unsigned long addr;
362363
unsigned i;
363364

@@ -366,7 +367,23 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
366367
if (!pud_none(*pud)) {
367368
walk_pmd(st, pud, addr);
368369
} else {
369-
note_page(st, addr, 2, pud_val(*pud), NULL);
370+
note_page(st, addr, 3, pud_val(*pud), NULL);
371+
}
372+
}
373+
}
374+
375+
static void walk_p4d(struct pg_state *st, pgd_t *pgd, unsigned long start)
376+
{
377+
p4d_t *p4d = p4d_offset(pgd, 0);
378+
unsigned long addr;
379+
unsigned i;
380+
381+
for (i = 0; i < PTRS_PER_P4D; i++, p4d++) {
382+
addr = start + i * P4D_SIZE;
383+
if (!p4d_none(*p4d)) {
384+
walk_pud(st, p4d, addr);
385+
} else {
386+
note_page(st, addr, 2, p4d_val(*p4d), NULL);
370387
}
371388
}
372389
}
@@ -381,7 +398,7 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm,
381398
for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
382399
addr = start + i * PGDIR_SIZE;
383400
if (!pgd_none(*pgd)) {
384-
walk_pud(st, pgd, addr);
401+
walk_p4d(st, pgd, addr);
385402
} else {
386403
note_page(st, addr, 1, pgd_val(*pgd), NULL);
387404
}

‎arch/arm/mm/fault-armv.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
9191
{
9292
spinlock_t *ptl;
9393
pgd_t *pgd;
94+
p4d_t *p4d;
9495
pud_t *pud;
9596
pmd_t *pmd;
9697
pte_t *pte;
@@ -100,7 +101,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
100101
if (pgd_none_or_clear_bad(pgd))
101102
return 0;
102103

103-
pud = pud_offset(pgd, address);
104+
p4d = p4d_offset(pgd, address);
105+
if (p4d_none_or_clear_bad(p4d))
106+
return 0;
107+
108+
pud = pud_offset(p4d, address);
104109
if (pud_none_or_clear_bad(pud))
105110
return 0;
106111

‎arch/arm/mm/fault.c

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,19 +43,21 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
4343
printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
4444

4545
do {
46+
p4d_t *p4d;
4647
pud_t *pud;
4748
pmd_t *pmd;
4849
pte_t *pte;
4950

50-
if (pgd_none(*pgd))
51+
p4d = p4d_offset(pgd, addr);
52+
if (p4d_none(*p4d))
5153
break;
5254

53-
if (pgd_bad(*pgd)) {
55+
if (p4d_bad(*p4d)) {
5456
pr_cont("(bad)");
5557
break;
5658
}
5759

58-
pud = pud_offset(pgd, addr);
60+
pud = pud_offset(p4d, addr);
5961
if (PTRS_PER_PUD != 1)
6062
pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
6163

@@ -405,6 +407,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
405407
{
406408
unsigned int index;
407409
pgd_t *pgd, *pgd_k;
410+
p4d_t *p4d, *p4d_k;
408411
pud_t *pud, *pud_k;
409412
pmd_t *pmd, *pmd_k;
410413

@@ -419,13 +422,16 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
419422
pgd = cpu_get_pgd() + index;
420423
pgd_k = init_mm.pgd + index;
421424

422-
if (pgd_none(*pgd_k))
425+
p4d = p4d_offset(pgd, addr);
426+
p4d_k = p4d_offset(pgd_k, addr);
427+
428+
if (p4d_none(*p4d_k))
423429
goto bad_area;
424-
if (!pgd_present(*pgd))
425-
set_pgd(pgd, *pgd_k);
430+
if (!p4d_present(*p4d))
431+
set_p4d(p4d, *p4d_k);
426432

427-
pud = pud_offset(pgd, addr);
428-
pud_k = pud_offset(pgd_k, addr);
433+
pud = pud_offset(p4d, addr);
434+
pud_k = pud_offset(p4d_k, addr);
429435

430436
if (pud_none(*pud_k))
431437
goto bad_area;

‎arch/arm/mm/idmap.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
6868
static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
6969
unsigned long prot)
7070
{
71-
pud_t *pud = pud_offset(pgd, addr);
71+
p4d_t *p4d = p4d_offset(pgd, addr);
72+
pud_t *pud = pud_offset(p4d, addr);
7273
unsigned long next;
7374

7475
do {

‎arch/arm/mm/init.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -519,7 +519,7 @@ static inline void section_update(unsigned long addr, pmdval_t mask,
519519
{
520520
pmd_t *pmd;
521521

522-
pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr);
522+
pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, addr), addr), addr), addr);
523523

524524
#ifdef CONFIG_ARM_LPAE
525525
pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);

‎arch/arm/mm/ioremap.c

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,12 +142,14 @@ static void unmap_area_sections(unsigned long virt, unsigned long size)
142142
{
143143
unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1));
144144
pgd_t *pgd;
145+
p4d_t *p4d;
145146
pud_t *pud;
146147
pmd_t *pmdp;
147148

148149
flush_cache_vunmap(addr, end);
149150
pgd = pgd_offset_k(addr);
150-
pud = pud_offset(pgd, addr);
151+
p4d = p4d_offset(pgd, addr);
152+
pud = pud_offset(p4d, addr);
151153
pmdp = pmd_offset(pud, addr);
152154
do {
153155
pmd_t pmd = *pmdp;
@@ -190,6 +192,7 @@ remap_area_sections(unsigned long virt, unsigned long pfn,
190192
{
191193
unsigned long addr = virt, end = virt + size;
192194
pgd_t *pgd;
195+
p4d_t *p4d;
193196
pud_t *pud;
194197
pmd_t *pmd;
195198

@@ -200,7 +203,8 @@ remap_area_sections(unsigned long virt, unsigned long pfn,
200203
unmap_area_sections(virt, size);
201204

202205
pgd = pgd_offset_k(addr);
203-
pud = pud_offset(pgd, addr);
206+
p4d = p4d_offset(pgd, addr);
207+
pud = pud_offset(p4d, addr);
204208
pmd = pmd_offset(pud, addr);
205209
do {
206210
pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
@@ -222,6 +226,7 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
222226
{
223227
unsigned long addr = virt, end = virt + size;
224228
pgd_t *pgd;
229+
p4d_t *p4d;
225230
pud_t *pud;
226231
pmd_t *pmd;
227232

@@ -232,7 +237,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
232237
unmap_area_sections(virt, size);
233238

234239
pgd = pgd_offset_k(virt);
235-
pud = pud_offset(pgd, addr);
240+
p4d = p4d_offset(pgd, addr);
241+
pud = pud_offset(p4d, addr);
236242
pmd = pmd_offset(pud, addr);
237243
do {
238244
unsigned long super_pmd_val, i;

‎arch/arm/mm/mm.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ static inline pte_t get_top_pte(unsigned long va)
3838

3939
static inline pmd_t *pmd_off_k(unsigned long virt)
4040
{
41-
return pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt);
41+
return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt);
4242
}
4343

4444
struct mem_type {

‎arch/arm/mm/mmu.c

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,8 @@ static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr)
357357
static inline pmd_t * __init fixmap_pmd(unsigned long addr)
358358
{
359359
pgd_t *pgd = pgd_offset_k(addr);
360-
pud_t *pud = pud_offset(pgd, addr);
360+
p4d_t *p4d = p4d_offset(pgd, addr);
361+
pud_t *pud = pud_offset(p4d, addr);
361362
pmd_t *pmd = pmd_offset(pud, addr);
362363

363364
return pmd;
@@ -801,12 +802,12 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
801802
} while (pmd++, addr = next, addr != end);
802803
}
803804

804-
static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
805+
static void __init alloc_init_pud(p4d_t *p4d, unsigned long addr,
805806
unsigned long end, phys_addr_t phys,
806807
const struct mem_type *type,
807808
void *(*alloc)(unsigned long sz), bool ng)
808809
{
809-
pud_t *pud = pud_offset(pgd, addr);
810+
pud_t *pud = pud_offset(p4d, addr);
810811
unsigned long next;
811812

812813
do {
@@ -816,6 +817,21 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
816817
} while (pud++, addr = next, addr != end);
817818
}
818819

820+
static void __init alloc_init_p4d(pgd_t *pgd, unsigned long addr,
821+
unsigned long end, phys_addr_t phys,
822+
const struct mem_type *type,
823+
void *(*alloc)(unsigned long sz), bool ng)
824+
{
825+
p4d_t *p4d = p4d_offset(pgd, addr);
826+
unsigned long next;
827+
828+
do {
829+
next = p4d_addr_end(addr, end);
830+
alloc_init_pud(p4d, addr, next, phys, type, alloc, ng);
831+
phys += next - addr;
832+
} while (p4d++, addr = next, addr != end);
833+
}
834+
819835
#ifndef CONFIG_ARM_LPAE
820836
static void __init create_36bit_mapping(struct mm_struct *mm,
821837
struct map_desc *md,
@@ -863,7 +879,8 @@ static void __init create_36bit_mapping(struct mm_struct *mm,
863879
pgd = pgd_offset(mm, addr);
864880
end = addr + length;
865881
do {
866-
pud_t *pud = pud_offset(pgd, addr);
882+
p4d_t *p4d = p4d_offset(pgd, addr);
883+
pud_t *pud = pud_offset(p4d, addr);
867884
pmd_t *pmd = pmd_offset(pud, addr);
868885
int i;
869886

@@ -914,7 +931,7 @@ static void __init __create_mapping(struct mm_struct *mm, struct map_desc *md,
914931
do {
915932
unsigned long next = pgd_addr_end(addr, end);
916933

917-
alloc_init_pud(pgd, addr, next, phys, type, alloc, ng);
934+
alloc_init_p4d(pgd, addr, next, phys, type, alloc, ng);
918935

919936
phys += next - addr;
920937
addr = next;
@@ -950,7 +967,13 @@ void __init create_mapping_late(struct mm_struct *mm, struct map_desc *md,
950967
bool ng)
951968
{
952969
#ifdef CONFIG_ARM_LPAE
953-
pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
970+
p4d_t *p4d;
971+
pud_t *pud;
972+
973+
p4d = p4d_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
974+
if (!WARN_ON(!p4d))
975+
return;
976+
pud = pud_alloc(mm, p4d, md->virtual);
954977
if (WARN_ON(!pud))
955978
return;
956979
pmd_alloc(mm, pud, 0);

0 commit comments

Comments
 (0)