Skip to content

Commit 3d049bc

Browse files
pccpcc
committed
hwasan: Support for outlined checks in the Linux kernel.
Add support for match-all tags and GOT-free runtime calls, which are both required for the kernel to be able to support outlined checks. This requires extending the access info to let the backend know when to enable these features. To make the code easier to maintain introduce an enum with the bit field positions for the access info. Allow outlined checks to be enabled with -mllvm -hwasan-inline-all-checks=0. Kernels that contain runtime support for outlined checks may pass this flag. Kernels lacking runtime support will continue to link because they do not pass the flag. Old versions of LLVM will ignore the flag and continue to use inline checks. With a separate kernel patch [1] I measured the code size of defconfig + tag-based KASAN, as well as boot time (i.e. time to init launch) on a DragonBoard 845c with an Android arm64 GKI kernel. The results are below: code size boot time before 92824064 6.18s after 38822400 6.65s [1] https://linux-review.googlesource.com/id/I1a30036c70ab3c3ee78d75ed9b87ef7cdc3fdb76 Depends on D90425 Differential Revision: https://reviews.llvm.org/D90426
1 parent ee703eb commit 3d049bc

File tree

5 files changed

+140
-35
lines changed

5 files changed

+140
-35
lines changed

‎llvm/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,24 @@ class HWAddressSanitizerPass : public PassInfoMixin<HWAddressSanitizerPass> {
3737
FunctionPass *createHWAddressSanitizerLegacyPassPass(bool CompileKernel = false,
3838
bool Recover = false);
3939

40+
namespace HWASanAccessInfo {
41+
42+
// Bit field positions for the accessinfo parameter to
43+
// llvm.hwasan.check.memaccess. Shared between the pass and the backend. Bits
44+
// 0-15 are also used by the runtime.
45+
enum {
46+
AccessSizeShift = 0, // 4 bits
47+
IsWriteShift = 4,
48+
RecoverShift = 5,
49+
MatchAllShift = 16, // 8 bits
50+
HasMatchAllShift = 24,
51+
CompileKernelShift = 25,
52+
};
53+
54+
enum { RuntimeMask = 0xffff };
55+
56+
} // namespace HWASanAccessInfo
57+
4058
} // namespace llvm
4159

4260
#endif

‎llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

Lines changed: 63 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
#include "llvm/Support/TargetRegistry.h"
5656
#include "llvm/Support/raw_ostream.h"
5757
#include "llvm/Target/TargetMachine.h"
58+
#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
5859
#include <algorithm>
5960
#include <cassert>
6061
#include <cstdint>
@@ -338,6 +339,15 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
338339
IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
339340
MCSymbol *Sym = P.second;
340341

342+
bool HasMatchAllTag =
343+
(AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
344+
uint8_t MatchAllTag =
345+
(AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
346+
unsigned Size =
347+
1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
348+
bool CompileKernel =
349+
(AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
350+
341351
OutStreamer->SwitchSection(OutContext.getELFSection(
342352
".text.hot", ELF::SHT_PROGBITS,
343353
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
@@ -382,6 +392,26 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
382392
MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
383393
OutStreamer->emitLabel(HandleMismatchOrPartialSym);
384394

395+
if (HasMatchAllTag) {
396+
OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
397+
.addReg(AArch64::X16)
398+
.addReg(Reg)
399+
.addImm(56)
400+
.addImm(63),
401+
*STI);
402+
OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
403+
.addReg(AArch64::XZR)
404+
.addReg(AArch64::X16)
405+
.addImm(MatchAllTag)
406+
.addImm(0),
407+
*STI);
408+
OutStreamer->emitInstruction(
409+
MCInstBuilder(AArch64::Bcc)
410+
.addImm(AArch64CC::EQ)
411+
.addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
412+
*STI);
413+
}
414+
385415
if (IsShort) {
386416
OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
387417
.addReg(AArch64::WZR)
@@ -402,7 +432,6 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
402432
.addReg(Reg)
403433
.addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
404434
*STI);
405-
unsigned Size = 1 << (AccessInfo & 0xf);
406435
if (Size != 1)
407436
OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
408437
.addReg(AArch64::X17)
@@ -470,32 +499,41 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
470499
.addReg(Reg)
471500
.addImm(0),
472501
*STI);
473-
OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
474-
.addReg(AArch64::X1)
475-
.addImm(AccessInfo)
476-
.addImm(0),
477-
*STI);
478-
479-
// Intentionally load the GOT entry and branch to it, rather than possibly
480-
// late binding the function, which may clobber the registers before we have
481-
// a chance to save them.
482502
OutStreamer->emitInstruction(
483-
MCInstBuilder(AArch64::ADRP)
484-
.addReg(AArch64::X16)
485-
.addExpr(AArch64MCExpr::create(
486-
HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
487-
OutContext)),
488-
*STI);
489-
OutStreamer->emitInstruction(
490-
MCInstBuilder(AArch64::LDRXui)
491-
.addReg(AArch64::X16)
492-
.addReg(AArch64::X16)
493-
.addExpr(AArch64MCExpr::create(
494-
HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
495-
OutContext)),
503+
MCInstBuilder(AArch64::MOVZXi)
504+
.addReg(AArch64::X1)
505+
.addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
506+
.addImm(0),
496507
*STI);
497-
OutStreamer->emitInstruction(
498-
MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
508+
509+
if (CompileKernel) {
510+
// The Linux kernel's dynamic loader doesn't support GOT relative
511+
// relocations, but it doesn't support late binding either, so just call
512+
// the function directly.
513+
OutStreamer->emitInstruction(
514+
MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
515+
} else {
516+
// Intentionally load the GOT entry and branch to it, rather than possibly
517+
// late binding the function, which may clobber the registers before we
518+
// have a chance to save them.
519+
OutStreamer->emitInstruction(
520+
MCInstBuilder(AArch64::ADRP)
521+
.addReg(AArch64::X16)
522+
.addExpr(AArch64MCExpr::create(
523+
HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
524+
OutContext)),
525+
*STI);
526+
OutStreamer->emitInstruction(
527+
MCInstBuilder(AArch64::LDRXui)
528+
.addReg(AArch64::X16)
529+
.addReg(AArch64::X16)
530+
.addExpr(AArch64MCExpr::create(
531+
HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
532+
OutContext)),
533+
*STI);
534+
OutStreamer->emitInstruction(
535+
MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
536+
}
499537
}
500538
}
501539

‎llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -283,9 +283,13 @@ class HWAddressSanitizer {
283283

284284
bool CompileKernel;
285285
bool Recover;
286+
bool OutlinedChecks;
286287
bool UseShortGranules;
287288
bool InstrumentLandingPads;
288289

290+
bool HasMatchAllTag = false;
291+
uint8_t MatchAllTag = 0;
292+
289293
Function *HwasanCtorFunction;
290294

291295
FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
@@ -496,6 +500,19 @@ void HWAddressSanitizer::initializeModule() {
496500

497501
UseShortGranules =
498502
ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
503+
OutlinedChecks =
504+
TargetTriple.isAArch64() && TargetTriple.isOSBinFormatELF() &&
505+
(ClInlineAllChecks.getNumOccurrences() ? !ClInlineAllChecks : !Recover);
506+
507+
if (ClMatchAllTag.getNumOccurrences()) {
508+
if (ClMatchAllTag != -1) {
509+
HasMatchAllTag = true;
510+
MatchAllTag = ClMatchAllTag & 0xFF;
511+
}
512+
} else if (CompileKernel) {
513+
HasMatchAllTag = true;
514+
MatchAllTag = 0xFF;
515+
}
499516

500517
// If we don't have personality function support, fall back to landing pads.
501518
InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
@@ -706,11 +723,16 @@ Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
706723
void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
707724
unsigned AccessSizeIndex,
708725
Instruction *InsertBefore) {
709-
const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
726+
const int64_t AccessInfo =
727+
(CompileKernel << HWASanAccessInfo::CompileKernelShift) +
728+
(HasMatchAllTag << HWASanAccessInfo::HasMatchAllShift) +
729+
(MatchAllTag << HWASanAccessInfo::MatchAllShift) +
730+
(Recover << HWASanAccessInfo::RecoverShift) +
731+
(IsWrite << HWASanAccessInfo::IsWriteShift) +
732+
(AccessSizeIndex << HWASanAccessInfo::AccessSizeShift);
710733
IRBuilder<> IRB(InsertBefore);
711734

712-
if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
713-
TargetTriple.isOSBinFormatELF() && !Recover) {
735+
if (OutlinedChecks) {
714736
Module *M = IRB.GetInsertBlock()->getParent()->getParent();
715737
Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
716738
IRB.CreateCall(Intrinsic::getDeclaration(
@@ -729,11 +751,9 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
729751
Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
730752
Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
731753

732-
int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
733-
ClMatchAllTag : (CompileKernel ? 0xFF : -1);
734-
if (matchAllTag != -1) {
735-
Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag,
736-
ConstantInt::get(PtrTag->getType(), matchAllTag));
754+
if (HasMatchAllTag) {
755+
Value *TagNotIgnored = IRB.CreateICmpNE(
756+
PtrTag, ConstantInt::get(PtrTag->getType(), MatchAllTag));
737757
TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored);
738758
}
739759

@@ -773,7 +793,9 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
773793
// The signal handler will find the data address in rdi.
774794
Asm = InlineAsm::get(
775795
FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
776-
"int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)",
796+
"int3\nnopl " +
797+
itostr(0x40 + (AccessInfo & HWASanAccessInfo::RuntimeMask)) +
798+
"(%rax)",
777799
"{rdi}",
778800
/*hasSideEffects=*/true);
779801
break;
@@ -782,7 +804,8 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
782804
// The signal handler will find the data address in x0.
783805
Asm = InlineAsm::get(
784806
FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false),
785-
"brk #" + itostr(0x900 + AccessInfo),
807+
"brk #" +
808+
itostr(0x900 + (AccessInfo & HWASanAccessInfo::RuntimeMask)),
786809
"{x0}",
787810
/*hasSideEffects=*/true);
788811
break;

‎llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,12 @@ define i8* @f2(i8* %x0, i8* %x1) {
3030
ret i8* %x0
3131
}
3232

33+
define void @f3(i8* %x0, i8* %x1) {
34+
; 0x3ff0000 (kernel, match-all = 0xff)
35+
call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 67043328)
36+
ret void
37+
}
38+
3339
declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
3440
declare void @llvm.hwasan.check.memaccess.shortgranules(i8*, i8*, i32)
3541

@@ -83,3 +89,20 @@ declare void @llvm.hwasan.check.memaccess.shortgranules(i8*, i8*, i32)
8389
; CHECK-NEXT: adrp x16, :got:__hwasan_tag_mismatch
8490
; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch]
8591
; CHECK-NEXT: br x16
92+
93+
; CHECK: __hwasan_check_x1_67043328:
94+
; CHECK-NEXT: sbfx x16, x1, #4, #52
95+
; CHECK-NEXT: ldrb w16, [x9, x16]
96+
; CHECK-NEXT: cmp x16, x1, lsr #56
97+
; CHECK-NEXT: b.ne .Ltmp5
98+
; CHECK-NEXT: .Ltmp6:
99+
; CHECK-NEXT: ret
100+
; CHECK-NEXT: .Ltmp5:
101+
; CHECK-NEXT: lsr x16, x1, #56
102+
; CHECK-NEXT: cmp x16, #255
103+
; CHECK-NEXT: b.eq .Ltmp6
104+
; CHECK-NEXT: stp x0, x1, [sp, #-256]!
105+
; CHECK-NEXT: stp x29, x30, [sp, #232]
106+
; CHECK-NEXT: mov x0, x1
107+
; CHECK-NEXT: mov x1, #0
108+
; CHECK-NEXT: b __hwasan_tag_mismatch

‎llvm/test/Instrumentation/HWAddressSanitizer/kernel.ll

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s --check-prefixes=CHECK,NOOFFSET,MATCH-ALL
55
; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s --check-prefixes=CHECK,OFFSET,MATCH-ALL
66
; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-match-all-tag=-1 -S | FileCheck %s --check-prefixes=CHECK,NOOFFSET,NO-MATCH-ALL
7+
; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-inline-all-checks=0 -hwasan-mapping-offset=12345678 -S | FileCheck %s --check-prefixes=OUTLINE
78

89
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
910
target triple = "aarch64--linux-android"
@@ -36,6 +37,8 @@ define i8 @test_load(i8* %a) sanitize_hwaddress {
3637
; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
3738
; CHECK: ret i8 %[[G]]
3839

40+
; OUTLINE: %[[SHADOW:[^ ]*]] = call i8* asm "", "=r,0"(i8* inttoptr (i64 12345678 to i8*))
41+
; OUTLINE: call void @llvm.hwasan.check.memaccess(i8* %[[SHADOW]], i8* %a, i32 67043360)
3942
entry:
4043
%b = load i8, i8* %a, align 4
4144
ret i8 %b

0 commit comments

Comments
 (0)