Skip to content

Commit 947fe7e

Browse files
matthiaskrgrmatthiaskrgr
authored
Rollup merge of #105109 - rcvalle:rust-kcfi, r=bjorn3
Add LLVM KCFI support to the Rust compiler This PR adds LLVM Kernel Control Flow Integrity (KCFI) support to the Rust compiler. It initially provides forward-edge control flow protection for operating systems kernels for Rust-compiled code only by aggregating function pointers in groups identified by their return and parameter types. (See llvm/llvm-project@cff5bef.) Forward-edge control flow protection for C or C++ and Rust -compiled code "mixed binaries" (i.e., for when C or C++ and Rust -compiled code share the same virtual address space) will be provided in later work as part of this project by identifying C char and integer type uses at the time types are encoded (see Type metadata in the design document in the tracking issue #89653). LLVM KCFI can be enabled with -Zsanitizer=kcfi. Thank you again, `@bjorn3,` `@eddyb,` `@nagisa,` and `@ojeda,` for all the help!
2 parents 020d7af + e1741ba commit 947fe7e

File tree

27 files changed

+261
-28
lines changed

27 files changed

+261
-28
lines changed

Diff for: ‎Cargo.lock

+12
Original file line numberDiff line numberDiff line change
@@ -4403,6 +4403,7 @@ dependencies = [
44034403
"rustc_span",
44044404
"rustc_target",
44054405
"tracing",
4406+
"twox-hash",
44064407
]
44074408

44084409
[[package]]
@@ -5393,6 +5394,17 @@ dependencies = [
53935394
"tracing-subscriber",
53945395
]
53955396

5397+
[[package]]
5398+
name = "twox-hash"
5399+
version = "1.6.3"
5400+
source = "registry+https://github.com/rust-lang/crates.io-index"
5401+
checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
5402+
dependencies = [
5403+
"cfg-if 1.0.0",
5404+
"rand 0.8.5",
5405+
"static_assertions",
5406+
]
5407+
53965408
[[package]]
53975409
name = "type-map"
53985410
version = "0.4.0"

Diff for: ‎compiler/rustc_codegen_gcc/src/type_.rs

+4
Original file line numberDiff line numberDiff line change
@@ -300,4 +300,8 @@ impl<'gcc, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
300300
// Unsupported.
301301
self.context.new_rvalue_from_int(self.int_type, 0)
302302
}
303+
304+
fn set_kcfi_type_metadata(&self, _function: RValue<'gcc>, _kcfi_typeid: u32) {
305+
// Unsupported.
306+
}
303307
}

Diff for: ‎compiler/rustc_codegen_llvm/src/allocator.rs

+11-3
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ pub(crate) unsafe fn codegen(
8888
callee,
8989
args.as_ptr(),
9090
args.len() as c_uint,
91-
None,
91+
[].as_ptr(),
92+
0 as c_uint,
9293
);
9394
llvm::LLVMSetTailCall(ret, True);
9495
if output.is_some() {
@@ -132,8 +133,15 @@ pub(crate) unsafe fn codegen(
132133
.enumerate()
133134
.map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint))
134135
.collect::<Vec<_>>();
135-
let ret =
136-
llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None);
136+
let ret = llvm::LLVMRustBuildCall(
137+
llbuilder,
138+
ty,
139+
callee,
140+
args.as_ptr(),
141+
args.len() as c_uint,
142+
[].as_ptr(),
143+
0 as c_uint,
144+
);
137145
llvm::LLVMSetTailCall(ret, True);
138146
llvm::LLVMBuildRetVoid(llbuilder);
139147
llvm::LLVMDisposeBuilder(llbuilder);

Diff for: ‎compiler/rustc_codegen_llvm/src/builder.rs

+43-7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use rustc_middle::ty::layout::{
2020
};
2121
use rustc_middle::ty::{self, Ty, TyCtxt};
2222
use rustc_span::Span;
23+
use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi;
2324
use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
2425
use rustc_target::spec::{HasTargetSpec, Target};
2526
use std::borrow::Cow;
@@ -225,9 +226,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
225226
debug!("invoke {:?} with args ({:?})", llfn, args);
226227

227228
let args = self.check_call("invoke", llty, llfn, args);
228-
let bundle = funclet.map(|funclet| funclet.bundle());
229-
let bundle = bundle.as_ref().map(|b| &*b.raw);
229+
let funclet_bundle = funclet.map(|funclet| funclet.bundle());
230+
let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
231+
let mut bundles = vec![funclet_bundle];
232+
233+
// Set KCFI operand bundle
234+
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
235+
let kcfi_bundle =
236+
if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
237+
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
238+
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
239+
} else {
240+
None
241+
};
242+
if kcfi_bundle.is_some() {
243+
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
244+
bundles.push(kcfi_bundle);
245+
}
230246

247+
bundles.retain(|bundle| bundle.is_some());
231248
let invoke = unsafe {
232249
llvm::LLVMRustBuildInvoke(
233250
self.llbuilder,
@@ -237,7 +254,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
237254
args.len() as c_uint,
238255
then,
239256
catch,
240-
bundle,
257+
bundles.as_ptr(),
258+
bundles.len() as c_uint,
241259
UNNAMED,
242260
)
243261
};
@@ -1143,7 +1161,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
11431161
llfn,
11441162
args.as_ptr() as *const &llvm::Value,
11451163
args.len() as c_uint,
1146-
None,
1164+
[].as_ptr(),
1165+
0 as c_uint,
11471166
);
11481167
}
11491168
}
@@ -1159,17 +1178,34 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
11591178
debug!("call {:?} with args ({:?})", llfn, args);
11601179

11611180
let args = self.check_call("call", llty, llfn, args);
1162-
let bundle = funclet.map(|funclet| funclet.bundle());
1163-
let bundle = bundle.as_ref().map(|b| &*b.raw);
1181+
let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1182+
let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
1183+
let mut bundles = vec![funclet_bundle];
1184+
1185+
// Set KCFI operand bundle
1186+
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
1187+
let kcfi_bundle =
1188+
if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call {
1189+
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap());
1190+
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1191+
} else {
1192+
None
1193+
};
1194+
if kcfi_bundle.is_some() {
1195+
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
1196+
bundles.push(kcfi_bundle);
1197+
}
11641198

1199+
bundles.retain(|bundle| bundle.is_some());
11651200
let call = unsafe {
11661201
llvm::LLVMRustBuildCall(
11671202
self.llbuilder,
11681203
llty,
11691204
llfn,
11701205
args.as_ptr() as *const &llvm::Value,
11711206
args.len() as c_uint,
1172-
bundle,
1207+
bundles.as_ptr(),
1208+
bundles.len() as c_uint,
11731209
)
11741210
};
11751211
if let Some(fn_abi) = fn_abi {

Diff for: ‎compiler/rustc_codegen_llvm/src/context.rs

+5
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,11 @@ pub unsafe fn create_module<'ll>(
250250
);
251251
}
252252

253+
if sess.is_sanitizer_kcfi_enabled() {
254+
let kcfi = "kcfi\0".as_ptr().cast();
255+
llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1);
256+
}
257+
253258
// Control Flow Guard is currently only supported by the MSVC linker on Windows.
254259
if sess.target.is_like_msvc {
255260
match sess.opts.cg.control_flow_guard {

Diff for: ‎compiler/rustc_codegen_llvm/src/declare.rs

+6-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ use crate::type_::Type;
2020
use crate::value::Value;
2121
use rustc_codegen_ssa::traits::TypeMembershipMethods;
2222
use rustc_middle::ty::Ty;
23-
use rustc_symbol_mangling::typeid::typeid_for_fnabi;
23+
use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi};
2424
use smallvec::SmallVec;
2525

2626
/// Declare a function.
@@ -136,6 +136,11 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
136136
self.set_type_metadata(llfn, typeid);
137137
}
138138

139+
if self.tcx.sess.is_sanitizer_kcfi_enabled() {
140+
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
141+
self.set_kcfi_type_metadata(llfn, kcfi_typeid);
142+
}
143+
139144
llfn
140145
}
141146

Diff for: ‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs

+6-2
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ pub enum MetadataType {
427427
MD_type = 19,
428428
MD_vcall_visibility = 28,
429429
MD_noundef = 29,
430+
MD_kcfi_type = 36,
430431
}
431432

432433
/// LLVMRustAsmDialect
@@ -1063,6 +1064,7 @@ extern "C" {
10631064
pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
10641065
pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
10651066
pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;
1067+
pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>;
10661068

10671069
// Operations on constants of any type
10681070
pub fn LLVMConstNull(Ty: &Type) -> &Value;
@@ -1273,7 +1275,8 @@ extern "C" {
12731275
NumArgs: c_uint,
12741276
Then: &'a BasicBlock,
12751277
Catch: &'a BasicBlock,
1276-
Bundle: Option<&OperandBundleDef<'a>>,
1278+
OpBundles: *const Option<&OperandBundleDef<'a>>,
1279+
NumOpBundles: c_uint,
12771280
Name: *const c_char,
12781281
) -> &'a Value;
12791282
pub fn LLVMBuildLandingPad<'a>(
@@ -1643,7 +1646,8 @@ extern "C" {
16431646
Fn: &'a Value,
16441647
Args: *const &'a Value,
16451648
NumArgs: c_uint,
1646-
Bundle: Option<&OperandBundleDef<'a>>,
1649+
OpBundles: *const Option<&OperandBundleDef<'a>>,
1650+
NumOpBundles: c_uint,
16471651
) -> &'a Value;
16481652
pub fn LLVMRustBuildMemCpy<'a>(
16491653
B: &Builder<'a>,

Diff for: ‎compiler/rustc_codegen_llvm/src/type_.rs

+15
Original file line numberDiff line numberDiff line change
@@ -316,4 +316,19 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> {
316316
)
317317
}
318318
}
319+
320+
fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) {
321+
let kcfi_type_metadata = self.const_u32(kcfi_typeid);
322+
unsafe {
323+
llvm::LLVMGlobalSetMetadata(
324+
function,
325+
llvm::MD_kcfi_type as c_uint,
326+
llvm::LLVMMDNodeInContext2(
327+
self.llcx,
328+
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
329+
1,
330+
),
331+
)
332+
}
333+
}
319334
}

Diff for: ‎compiler/rustc_codegen_ssa/src/traits/type_.rs

+1
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> {
122122
pub trait TypeMembershipMethods<'tcx>: Backend<'tcx> {
123123
fn set_type_metadata(&self, function: Self::Function, typeid: String);
124124
fn typeid_metadata(&self, typeid: String) -> Self::Value;
125+
fn set_kcfi_type_metadata(&self, function: Self::Function, typeid: u32);
125126
}
126127

127128
pub trait ArgAbiMethods<'tcx>: HasCodegen<'tcx> {

Diff for: ‎compiler/rustc_feature/src/builtin_attrs.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
394394
ungated!(instruction_set, Normal, template!(List: "set"), ErrorPreceding),
395395
gated!(
396396
no_sanitize, Normal,
397-
template!(List: "address, memory, thread"), DuplicatesOk,
397+
template!(List: "address, kcfi, memory, thread"), DuplicatesOk,
398398
experimental!(no_sanitize)
399399
),
400400
gated!(no_coverage, Normal, template!(Word), WarnFollowing, experimental!(no_coverage)),

Diff for: ‎compiler/rustc_hir_analysis/src/collect.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -1846,6 +1846,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
18461846
codegen_fn_attrs.no_sanitize |= SanitizerSet::ADDRESS;
18471847
} else if item.has_name(sym::cfi) {
18481848
codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI;
1849+
} else if item.has_name(sym::kcfi) {
1850+
codegen_fn_attrs.no_sanitize |= SanitizerSet::KCFI;
18491851
} else if item.has_name(sym::memory) {
18501852
codegen_fn_attrs.no_sanitize |= SanitizerSet::MEMORY;
18511853
} else if item.has_name(sym::memtag) {
@@ -1859,7 +1861,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
18591861
} else {
18601862
tcx.sess
18611863
.struct_span_err(item.span(), "invalid argument for `no_sanitize`")
1862-
.note("expected one of: `address`, `cfi`, `hwaddress`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
1864+
.note("expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread`")
18631865
.emit();
18641866
}
18651867
}

Diff for: ‎compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

+8-8
Original file line numberDiff line numberDiff line change
@@ -1476,13 +1476,13 @@ extern "C" void LLVMRustFreeOperandBundleDef(OperandBundleDef *Bundle) {
14761476

14771477
extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
14781478
LLVMValueRef *Args, unsigned NumArgs,
1479-
OperandBundleDef *Bundle) {
1479+
OperandBundleDef **OpBundles,
1480+
unsigned NumOpBundles) {
14801481
Value *Callee = unwrap(Fn);
14811482
FunctionType *FTy = unwrap<FunctionType>(Ty);
1482-
unsigned Len = Bundle ? 1 : 0;
1483-
ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len);
14841483
return wrap(unwrap(B)->CreateCall(
1485-
FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles));
1484+
FTy, Callee, makeArrayRef(unwrap(Args), NumArgs),
1485+
makeArrayRef(*OpBundles, NumOpBundles)));
14861486
}
14871487

14881488
extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
@@ -1522,14 +1522,14 @@ extern "C" LLVMValueRef
15221522
LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
15231523
LLVMValueRef *Args, unsigned NumArgs,
15241524
LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
1525-
OperandBundleDef *Bundle, const char *Name) {
1525+
OperandBundleDef **OpBundles, unsigned NumOpBundles,
1526+
const char *Name) {
15261527
Value *Callee = unwrap(Fn);
15271528
FunctionType *FTy = unwrap<FunctionType>(Ty);
1528-
unsigned Len = Bundle ? 1 : 0;
1529-
ArrayRef<OperandBundleDef> Bundles = makeArrayRef(Bundle, Len);
15301529
return wrap(unwrap(B)->CreateInvoke(FTy, Callee, unwrap(Then), unwrap(Catch),
15311530
makeArrayRef(unwrap(Args), NumArgs),
1532-
Bundles, Name));
1531+
makeArrayRef(*OpBundles, NumOpBundles),
1532+
Name));
15331533
}
15341534

15351535
extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B,

Diff for: ‎compiler/rustc_session/src/options.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ mod desc {
368368
pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
369369
pub const parse_oom_strategy: &str = "either `panic` or `abort`";
370370
pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
371-
pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
371+
pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `kcfi`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
372372
pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2";
373373
pub const parse_cfguard: &str =
374374
"either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
@@ -675,6 +675,7 @@ mod parse {
675675
*slot |= match s {
676676
"address" => SanitizerSet::ADDRESS,
677677
"cfi" => SanitizerSet::CFI,
678+
"kcfi" => SanitizerSet::KCFI,
678679
"leak" => SanitizerSet::LEAK,
679680
"memory" => SanitizerSet::MEMORY,
680681
"memtag" => SanitizerSet::MEMTAG,

Diff for: ‎compiler/rustc_session/src/session.rs

+12
Original file line numberDiff line numberDiff line change
@@ -686,6 +686,10 @@ impl Session {
686686
self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI)
687687
}
688688

689+
pub fn is_sanitizer_kcfi_enabled(&self) -> bool {
690+
self.opts.unstable_opts.sanitizer.contains(SanitizerSet::KCFI)
691+
}
692+
689693
/// Check whether this compile session and crate type use static crt.
690694
pub fn crt_static(&self, crate_type: Option<CrateType>) -> bool {
691695
if !self.target.crt_static_respected {
@@ -1544,6 +1548,14 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
15441548
}
15451549
}
15461550

1551+
// LLVM CFI and KCFI are mutually exclusive
1552+
if sess.is_sanitizer_cfi_enabled() && sess.is_sanitizer_kcfi_enabled() {
1553+
sess.emit_err(CannotMixAndMatchSanitizers {
1554+
first: "cfi".to_string(),
1555+
second: "kcfi".to_string(),
1556+
});
1557+
}
1558+
15471559
if sess.opts.unstable_opts.stack_protector != StackProtector::None {
15481560
if !sess.target.options.supports_stack_protector {
15491561
sess.emit_warning(StackProtectorNotSupportedForTarget {

Diff for: ‎compiler/rustc_span/src/symbol.rs

+1
Original file line numberDiff line numberDiff line change
@@ -828,6 +828,7 @@ symbols! {
828828
item_like_imports,
829829
iter,
830830
iter_repeat,
831+
kcfi,
831832
keyword,
832833
kind,
833834
kreg,

Diff for: ‎compiler/rustc_symbol_mangling/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ bitflags = "1.2.1"
1010
tracing = "0.1"
1111
punycode = "0.4.0"
1212
rustc-demangle = "0.1.21"
13+
twox-hash = "1.6.3"
1314

1415
rustc_span = { path = "../rustc_span" }
1516
rustc_middle = { path = "../rustc_middle" }

0 commit comments

Comments
 (0)