@@ -318,6 +318,21 @@ extern struct cpu_tlb_fns cpu_tlb;
318
318
319
319
#define tlb_flag (f ) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f)))
320
320
321
+ #define __tlb_op (f , insnarg , arg ) \
322
+ do { \
323
+ if (always_tlb_flags & (f)) \
324
+ asm("mcr " insnarg \
325
+ : : "r" (arg) : "cc"); \
326
+ else if (possible_tlb_flags & (f)) \
327
+ asm("tst %1, %2\n\t" \
328
+ "mcrne " insnarg \
329
+ : : "r" (arg), "r" (__tlb_flag), "Ir" (f) \
330
+ : "cc"); \
331
+ } while (0)
332
+
333
+ #define tlb_op (f , regs , arg ) __tlb_op(f, "p15, 0, %0, " regs, arg)
334
+ #define tlb_l2_op (f , regs , arg ) __tlb_op(f, "p15, 1, %0, " regs, arg)
335
+
321
336
static inline void local_flush_tlb_all (void )
322
337
{
323
338
const int zero = 0 ;
@@ -326,16 +341,11 @@ static inline void local_flush_tlb_all(void)
326
341
if (tlb_flag (TLB_WB ))
327
342
dsb ();
328
343
329
- if (tlb_flag (TLB_V3_FULL ))
330
- asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero ) : "cc" );
331
- if (tlb_flag (TLB_V4_U_FULL | TLB_V6_U_FULL ))
332
- asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero ) : "cc" );
333
- if (tlb_flag (TLB_V4_D_FULL | TLB_V6_D_FULL ))
334
- asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero ) : "cc" );
335
- if (tlb_flag (TLB_V4_I_FULL | TLB_V6_I_FULL ))
336
- asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero ) : "cc" );
337
- if (tlb_flag (TLB_V7_UIS_FULL ))
338
- asm("mcr p15, 0, %0, c8, c3, 0" : : "r" (zero ) : "cc" );
344
+ tlb_op (TLB_V3_FULL , "c6, c0, 0" , zero );
345
+ tlb_op (TLB_V4_U_FULL | TLB_V6_U_FULL , "c8, c7, 0" , zero );
346
+ tlb_op (TLB_V4_D_FULL | TLB_V6_D_FULL , "c8, c6, 0" , zero );
347
+ tlb_op (TLB_V4_I_FULL | TLB_V6_I_FULL , "c8, c5, 0" , zero );
348
+ tlb_op (TLB_V7_UIS_FULL , "c8, c3, 0" , zero );
339
349
340
350
if (tlb_flag (TLB_BARRIER )) {
341
351
dsb ();
@@ -352,29 +362,23 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm)
352
362
if (tlb_flag (TLB_WB ))
353
363
dsb ();
354
364
355
- if (cpumask_test_cpu (get_cpu (), mm_cpumask (mm ))) {
356
- if (tlb_flag (TLB_V3_FULL ))
357
- asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero ) : "cc" );
358
- if (tlb_flag (TLB_V4_U_FULL ))
359
- asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero ) : "cc" );
360
- if (tlb_flag (TLB_V4_D_FULL ))
361
- asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero ) : "cc" );
362
- if (tlb_flag (TLB_V4_I_FULL ))
363
- asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero ) : "cc" );
365
+ if (possible_tlb_flags & (TLB_V3_FULL |TLB_V4_U_FULL |TLB_V4_D_FULL |TLB_V4_I_FULL )) {
366
+ if (cpumask_test_cpu (get_cpu (), mm_cpumask (mm ))) {
367
+ tlb_op (TLB_V3_FULL , "c6, c0, 0" , zero );
368
+ tlb_op (TLB_V4_U_FULL , "c8, c7, 0" , zero );
369
+ tlb_op (TLB_V4_D_FULL , "c8, c6, 0" , zero );
370
+ tlb_op (TLB_V4_I_FULL , "c8, c5, 0" , zero );
371
+ }
372
+ put_cpu ();
364
373
}
365
- put_cpu ();
366
-
367
- if (tlb_flag (TLB_V6_U_ASID ))
368
- asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid ) : "cc" );
369
- if (tlb_flag (TLB_V6_D_ASID ))
370
- asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid ) : "cc" );
371
- if (tlb_flag (TLB_V6_I_ASID ))
372
- asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid ) : "cc" );
373
- if (tlb_flag (TLB_V7_UIS_ASID ))
374
+
375
+ tlb_op (TLB_V6_U_ASID , "c8, c7, 2" , asid );
376
+ tlb_op (TLB_V6_D_ASID , "c8, c6, 2" , asid );
377
+ tlb_op (TLB_V6_I_ASID , "c8, c5, 2" , asid );
374
378
#ifdef CONFIG_ARM_ERRATA_720789
375
- asm( "mcr p15, 0, %0, c8, c3, 0" : : "r" ( zero ) : "cc" );
379
+ tlb_op ( TLB_V7_UIS_ASID , " c8, c3, 0", zero );
376
380
#else
377
- asm( "mcr p15, 0, %0, c8, c3, 2" : : "r" ( asid ) : "cc" );
381
+ tlb_op ( TLB_V7_UIS_ASID , " c8, c3, 2", asid );
378
382
#endif
379
383
380
384
if (tlb_flag (TLB_BARRIER ))
@@ -392,30 +396,23 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
392
396
if (tlb_flag (TLB_WB ))
393
397
dsb ();
394
398
395
- if (cpumask_test_cpu (smp_processor_id (), mm_cpumask (vma -> vm_mm ))) {
396
- if (tlb_flag (TLB_V3_PAGE ))
397
- asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr ) : "cc" );
398
- if (tlb_flag (TLB_V4_U_PAGE ))
399
- asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr ) : "cc" );
400
- if (tlb_flag (TLB_V4_D_PAGE ))
401
- asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr ) : "cc" );
402
- if (tlb_flag (TLB_V4_I_PAGE ))
403
- asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr ) : "cc" );
399
+ if (possible_tlb_flags & (TLB_V3_PAGE |TLB_V4_U_PAGE |TLB_V4_D_PAGE |TLB_V4_I_PAGE |TLB_V4_I_FULL ) &&
400
+ cpumask_test_cpu (smp_processor_id (), mm_cpumask (vma -> vm_mm ))) {
401
+ tlb_op (TLB_V3_PAGE , "c6, c0, 0" , uaddr );
402
+ tlb_op (TLB_V4_U_PAGE , "c8, c7, 1" , uaddr );
403
+ tlb_op (TLB_V4_D_PAGE , "c8, c6, 1" , uaddr );
404
+ tlb_op (TLB_V4_I_PAGE , "c8, c5, 1" , uaddr );
404
405
if (!tlb_flag (TLB_V4_I_PAGE ) && tlb_flag (TLB_V4_I_FULL ))
405
406
asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero ) : "cc" );
406
407
}
407
408
408
- if (tlb_flag (TLB_V6_U_PAGE ))
409
- asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr ) : "cc" );
410
- if (tlb_flag (TLB_V6_D_PAGE ))
411
- asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr ) : "cc" );
412
- if (tlb_flag (TLB_V6_I_PAGE ))
413
- asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr ) : "cc" );
414
- if (tlb_flag (TLB_V7_UIS_PAGE ))
409
+ tlb_op (TLB_V6_U_PAGE , "c8, c7, 1" , uaddr );
410
+ tlb_op (TLB_V6_D_PAGE , "c8, c6, 1" , uaddr );
411
+ tlb_op (TLB_V6_I_PAGE , "c8, c5, 1" , uaddr );
415
412
#ifdef CONFIG_ARM_ERRATA_720789
416
- asm( "mcr p15, 0, %0, c8, c3, 3" : : "r" ( uaddr & PAGE_MASK ) : "cc" );
413
+ tlb_op ( TLB_V7_UIS_PAGE , " c8, c3, 3", uaddr & PAGE_MASK );
417
414
#else
418
- asm( "mcr p15, 0, %0, c8, c3, 1" : : "r" ( uaddr ) : "cc" );
415
+ tlb_op ( TLB_V7_UIS_PAGE , " c8, c3, 1", uaddr );
419
416
#endif
420
417
421
418
if (tlb_flag (TLB_BARRIER ))
@@ -432,25 +429,17 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
432
429
if (tlb_flag (TLB_WB ))
433
430
dsb ();
434
431
435
- if (tlb_flag (TLB_V3_PAGE ))
436
- asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr ) : "cc" );
437
- if (tlb_flag (TLB_V4_U_PAGE ))
438
- asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr ) : "cc" );
439
- if (tlb_flag (TLB_V4_D_PAGE ))
440
- asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr ) : "cc" );
441
- if (tlb_flag (TLB_V4_I_PAGE ))
442
- asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr ) : "cc" );
432
+ tlb_op (TLB_V3_PAGE , "c6, c0, 0" , kaddr );
433
+ tlb_op (TLB_V4_U_PAGE , "c8, c7, 1" , kaddr );
434
+ tlb_op (TLB_V4_D_PAGE , "c8, c6, 1" , kaddr );
435
+ tlb_op (TLB_V4_I_PAGE , "c8, c5, 1" , kaddr );
443
436
if (!tlb_flag (TLB_V4_I_PAGE ) && tlb_flag (TLB_V4_I_FULL ))
444
437
asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero ) : "cc" );
445
438
446
- if (tlb_flag (TLB_V6_U_PAGE ))
447
- asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr ) : "cc" );
448
- if (tlb_flag (TLB_V6_D_PAGE ))
449
- asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr ) : "cc" );
450
- if (tlb_flag (TLB_V6_I_PAGE ))
451
- asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr ) : "cc" );
452
- if (tlb_flag (TLB_V7_UIS_PAGE ))
453
- asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (kaddr ) : "cc" );
439
+ tlb_op (TLB_V6_U_PAGE , "c8, c7, 1" , kaddr );
440
+ tlb_op (TLB_V6_D_PAGE , "c8, c6, 1" , kaddr );
441
+ tlb_op (TLB_V6_I_PAGE , "c8, c5, 1" , kaddr );
442
+ tlb_op (TLB_V7_UIS_PAGE , "c8, c3, 1" , kaddr );
454
443
455
444
if (tlb_flag (TLB_BARRIER )) {
456
445
dsb ();
@@ -475,13 +464,8 @@ static inline void flush_pmd_entry(void *pmd)
475
464
{
476
465
const unsigned int __tlb_flag = __cpu_tlb_flags ;
477
466
478
- if (tlb_flag (TLB_DCLEAN ))
479
- asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd"
480
- : : "r" (pmd ) : "cc" );
481
-
482
- if (tlb_flag (TLB_L2CLEAN_FR ))
483
- asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd"
484
- : : "r" (pmd ) : "cc" );
467
+ tlb_op (TLB_DCLEAN , "c7, c10, 1 @ flush_pmd" , pmd );
468
+ tlb_l2_op (TLB_L2CLEAN_FR , "c15, c9, 1 @ L2 flush_pmd" , pmd );
485
469
486
470
if (tlb_flag (TLB_WB ))
487
471
dsb ();
@@ -491,15 +475,11 @@ static inline void clean_pmd_entry(void *pmd)
491
475
{
492
476
const unsigned int __tlb_flag = __cpu_tlb_flags ;
493
477
494
- if (tlb_flag (TLB_DCLEAN ))
495
- asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd"
496
- : : "r" (pmd ) : "cc" );
497
-
498
- if (tlb_flag (TLB_L2CLEAN_FR ))
499
- asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd"
500
- : : "r" (pmd ) : "cc" );
478
+ tlb_op (TLB_DCLEAN , "c7, c10, 1 @ flush_pmd" , pmd );
479
+ tlb_l2_op (TLB_L2CLEAN_FR , "c15, c9, 1 @ L2 flush_pmd" , pmd );
501
480
}
502
481
482
+ #undef tlb_op
503
483
#undef tlb_flag
504
484
#undef always_tlb_flags
505
485
#undef possible_tlb_flags
0 commit comments