Skip to content

Commit 30d3ce0

Browse files
erikdesjardinserikdesjardins
committed
Add LLVM attributes in batches instead of individually
This should improve performance.
1 parent 6f681a8 commit 30d3ce0

File tree

11 files changed

+417
-439
lines changed

11 files changed

+417
-439
lines changed

‎compiler/rustc_codegen_llvm/src/abi.rs

+85-129
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
use crate::attributes;
12
use crate::builder::Builder;
23
use crate::context::CodegenCx;
3-
use crate::llvm::{self, AttributePlace};
4+
use crate::llvm::{self, Attribute, AttributePlace};
45
use crate::type_::Type;
56
use crate::type_of::LayoutLlvmExt;
67
use crate::value::Value;
@@ -20,6 +21,7 @@ use rustc_target::abi::{self, HasDataLayout, Int};
2021
pub use rustc_target::spec::abi::Abi;
2122

2223
use libc::c_uint;
24+
use smallvec::SmallVec;
2325

2426
pub trait ArgAttributesExt {
2527
fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value);
@@ -38,57 +40,68 @@ fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool {
3840
cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true)
3941
}
4042

41-
const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::Attribute); 1] =
42-
[(ArgAttribute::InReg, llvm::Attribute::InReg)];
43+
const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] =
44+
[(ArgAttribute::InReg, llvm::AttributeKind::InReg)];
4345

44-
const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::Attribute); 5] = [
45-
(ArgAttribute::NoAlias, llvm::Attribute::NoAlias),
46-
(ArgAttribute::NoCapture, llvm::Attribute::NoCapture),
47-
(ArgAttribute::NonNull, llvm::Attribute::NonNull),
48-
(ArgAttribute::ReadOnly, llvm::Attribute::ReadOnly),
49-
(ArgAttribute::NoUndef, llvm::Attribute::NoUndef),
46+
const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [
47+
(ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias),
48+
(ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture),
49+
(ArgAttribute::NonNull, llvm::AttributeKind::NonNull),
50+
(ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly),
51+
(ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef),
5052
];
5153

52-
impl ArgAttributesExt for ArgAttributes {
53-
fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
54-
let mut regular = self.regular;
55-
unsafe {
56-
// ABI-affecting attributes must always be applied
57-
for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
58-
if regular.contains(attr) {
59-
llattr.apply_llfn(idx, llfn);
60-
}
61-
}
62-
if let Some(align) = self.pointee_align {
63-
llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32);
64-
}
65-
match self.arg_ext {
66-
ArgExtension::None => {}
67-
ArgExtension::Zext => llvm::Attribute::ZExt.apply_llfn(idx, llfn),
68-
ArgExtension::Sext => llvm::Attribute::SExt.apply_llfn(idx, llfn),
69-
}
70-
// Only apply remaining attributes when optimizing
71-
if cx.sess().opts.optimize == config::OptLevel::No {
72-
return;
73-
}
74-
let deref = self.pointee_size.bytes();
75-
if deref != 0 {
76-
if regular.contains(ArgAttribute::NonNull) {
77-
llvm::LLVMRustAddDereferenceableAttr(llfn, idx.as_uint(), deref);
78-
} else {
79-
llvm::LLVMRustAddDereferenceableOrNullAttr(llfn, idx.as_uint(), deref);
80-
}
81-
regular -= ArgAttribute::NonNull;
82-
}
83-
for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
84-
if regular.contains(attr) {
85-
llattr.apply_llfn(idx, llfn);
86-
}
54+
fn get_attrs<'ll>(
55+
this: &ArgAttributes,
56+
cx: &CodegenCx<'ll, '_>,
57+
) -> SmallVec<impl smallvec::Array<Item = &'ll Attribute>> {
58+
let mut regular = this.regular;
59+
60+
let mut attrs = SmallVec::<[_; 8]>::new();
61+
62+
// ABI-affecting attributes must always be applied
63+
for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
64+
if regular.contains(attr) {
65+
attrs.push(llattr.create_attr(cx.llcx));
66+
}
67+
}
68+
if let Some(align) = this.pointee_align {
69+
attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes()));
70+
}
71+
match this.arg_ext {
72+
ArgExtension::None => {}
73+
ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)),
74+
ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)),
75+
}
76+
77+
// Only apply remaining attributes when optimizing
78+
if cx.sess().opts.optimize != config::OptLevel::No {
79+
let deref = this.pointee_size.bytes();
80+
if deref != 0 {
81+
if regular.contains(ArgAttribute::NonNull) {
82+
attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref));
83+
} else {
84+
attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref));
8785
}
88-
if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
89-
llvm::Attribute::NoAlias.apply_llfn(idx, llfn);
86+
regular -= ArgAttribute::NonNull;
87+
}
88+
for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
89+
if regular.contains(attr) {
90+
attrs.push(llattr.create_attr(cx.llcx));
9091
}
9192
}
93+
if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
94+
attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx));
95+
}
96+
}
97+
98+
attrs
99+
}
100+
101+
impl ArgAttributesExt for ArgAttributes {
102+
fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
103+
let attrs = get_attrs(self, cx);
104+
attributes::apply_to_llfn(llfn, idx, &attrs);
92105
}
93106

94107
fn apply_attrs_to_callsite(
@@ -97,52 +110,8 @@ impl ArgAttributesExt for ArgAttributes {
97110
cx: &CodegenCx<'_, '_>,
98111
callsite: &Value,
99112
) {
100-
let mut regular = self.regular;
101-
unsafe {
102-
// ABI-affecting attributes must always be applied
103-
for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
104-
if regular.contains(attr) {
105-
llattr.apply_callsite(idx, callsite);
106-
}
107-
}
108-
if let Some(align) = self.pointee_align {
109-
llvm::LLVMRustAddAlignmentCallSiteAttr(
110-
callsite,
111-
idx.as_uint(),
112-
align.bytes() as u32,
113-
);
114-
}
115-
match self.arg_ext {
116-
ArgExtension::None => {}
117-
ArgExtension::Zext => llvm::Attribute::ZExt.apply_callsite(idx, callsite),
118-
ArgExtension::Sext => llvm::Attribute::SExt.apply_callsite(idx, callsite),
119-
}
120-
// Only apply remaining attributes when optimizing
121-
if cx.sess().opts.optimize == config::OptLevel::No {
122-
return;
123-
}
124-
let deref = self.pointee_size.bytes();
125-
if deref != 0 {
126-
if regular.contains(ArgAttribute::NonNull) {
127-
llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite, idx.as_uint(), deref);
128-
} else {
129-
llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr(
130-
callsite,
131-
idx.as_uint(),
132-
deref,
133-
);
134-
}
135-
regular -= ArgAttribute::NonNull;
136-
}
137-
for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
138-
if regular.contains(attr) {
139-
llattr.apply_callsite(idx, callsite);
140-
}
141-
}
142-
if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
143-
llvm::Attribute::NoAlias.apply_callsite(idx, callsite);
144-
}
145-
}
113+
let attrs = get_attrs(self, cx);
114+
attributes::apply_to_callsite(callsite, idx, &attrs);
146115
}
147116
}
148117

@@ -444,15 +413,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
444413
}
445414

446415
fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) {
447-
// FIXME(eddyb) can this also be applied to callsites?
416+
let mut func_attrs = SmallVec::<[_; 2]>::new();
448417
if self.ret.layout.abi.is_uninhabited() {
449-
llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
418+
func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx));
450419
}
451-
452-
// FIXME(eddyb, wesleywiser): apply this to callsites as well?
453420
if !self.can_unwind {
454-
llvm::Attribute::NoUnwind.apply_llfn(llvm::AttributePlace::Function, llfn);
421+
func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx));
455422
}
423+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs });
456424

457425
let mut i = 0;
458426
let mut apply = |attrs: &ArgAttributes| {
@@ -467,13 +435,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
467435
PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
468436
assert!(!on_stack);
469437
let i = apply(attrs);
470-
unsafe {
471-
llvm::LLVMRustAddStructRetAttr(
472-
llfn,
473-
llvm::AttributePlace::Argument(i).as_uint(),
474-
self.ret.layout.llvm_type(cx),
475-
);
476-
}
438+
let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx));
439+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]);
477440
}
478441
PassMode::Cast(cast) => {
479442
cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
@@ -488,13 +451,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
488451
PassMode::Ignore => {}
489452
PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
490453
let i = apply(attrs);
491-
unsafe {
492-
llvm::LLVMRustAddByValAttr(
493-
llfn,
494-
llvm::AttributePlace::Argument(i).as_uint(),
495-
arg.layout.llvm_type(cx),
496-
);
497-
}
454+
let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx));
455+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
498456
}
499457
PassMode::Direct(ref attrs)
500458
| PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
@@ -517,12 +475,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
517475
}
518476

519477
fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
478+
let mut func_attrs = SmallVec::<[_; 2]>::new();
520479
if self.ret.layout.abi.is_uninhabited() {
521-
llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite);
480+
func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));
522481
}
523482
if !self.can_unwind {
524-
llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite);
483+
func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx));
525484
}
485+
attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs });
526486

527487
let mut i = 0;
528488
let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| {
@@ -537,13 +497,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
537497
PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
538498
assert!(!on_stack);
539499
let i = apply(bx.cx, attrs);
540-
unsafe {
541-
llvm::LLVMRustAddStructRetCallSiteAttr(
542-
callsite,
543-
llvm::AttributePlace::Argument(i).as_uint(),
544-
self.ret.layout.llvm_type(bx),
545-
);
546-
}
500+
let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx));
501+
attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]);
547502
}
548503
PassMode::Cast(cast) => {
549504
cast.attrs.apply_attrs_to_callsite(
@@ -572,13 +527,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
572527
PassMode::Ignore => {}
573528
PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
574529
let i = apply(bx.cx, attrs);
575-
unsafe {
576-
llvm::LLVMRustAddByValCallSiteAttr(
577-
callsite,
578-
llvm::AttributePlace::Argument(i).as_uint(),
579-
arg.layout.llvm_type(bx),
580-
);
581-
}
530+
let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx));
531+
attributes::apply_to_callsite(
532+
callsite,
533+
llvm::AttributePlace::Argument(i),
534+
&[byval],
535+
);
582536
}
583537
PassMode::Direct(ref attrs)
584538
| PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
@@ -610,10 +564,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
610564
if self.conv == Conv::CCmseNonSecureCall {
611565
// This will probably get ignored on all targets but those supporting the TrustZone-M
612566
// extension (thumbv8m targets).
613-
llvm::AddCallSiteAttrString(
567+
let cmse_nonsecure_call =
568+
llvm::CreateAttrString(bx.cx.llcx, cstr::cstr!("cmse_nonsecure_call"));
569+
attributes::apply_to_callsite(
614570
callsite,
615571
llvm::AttributePlace::Function,
616-
cstr::cstr!("cmse_nonsecure_call"),
572+
&[cmse_nonsecure_call],
617573
);
618574
}
619575
}

‎compiler/rustc_codegen_llvm/src/allocator.rs

+7-4
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ pub(crate) unsafe fn codegen(
6464
llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
6565
}
6666
if tcx.sess.must_emit_unwind_tables() {
67-
attributes::emit_uwtable(llfn);
67+
let uwtable = attributes::uwtable_attr(llcx);
68+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]);
6869
}
6970

7071
let callee = kind.fn_name(method.name);
@@ -105,20 +106,22 @@ pub(crate) unsafe fn codegen(
105106
let name = "__rust_alloc_error_handler";
106107
let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty);
107108
// -> ! DIFlagNoReturn
108-
llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
109+
let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx);
110+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]);
109111

110112
if tcx.sess.target.default_hidden_visibility {
111113
llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
112114
}
113115
if tcx.sess.must_emit_unwind_tables() {
114-
attributes::emit_uwtable(llfn);
116+
let uwtable = attributes::uwtable_attr(llcx);
117+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]);
115118
}
116119

117120
let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default };
118121
let callee = kind.fn_name(sym::oom);
119122
let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty);
120123
// -> ! DIFlagNoReturn
121-
llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee);
124+
attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]);
122125
llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden);
123126

124127
let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast());

‎compiler/rustc_codegen_llvm/src/asm.rs

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::attributes;
12
use crate::builder::Builder;
23
use crate::common::Funclet;
34
use crate::context::CodegenCx;
@@ -18,6 +19,7 @@ use rustc_target::abi::*;
1819
use rustc_target::asm::*;
1920

2021
use libc::{c_char, c_uint};
22+
use smallvec::SmallVec;
2123
use tracing::debug;
2224

2325
impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
@@ -273,19 +275,20 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
273275
)
274276
.unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
275277

278+
let mut attrs = SmallVec::<[_; 2]>::new();
276279
if options.contains(InlineAsmOptions::PURE) {
277280
if options.contains(InlineAsmOptions::NOMEM) {
278-
llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result);
281+
attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx));
279282
} else if options.contains(InlineAsmOptions::READONLY) {
280-
llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result);
283+
attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx));
281284
}
282-
llvm::Attribute::WillReturn.apply_callsite(llvm::AttributePlace::Function, result);
285+
attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
283286
} else if options.contains(InlineAsmOptions::NOMEM) {
284-
llvm::Attribute::InaccessibleMemOnly
285-
.apply_callsite(llvm::AttributePlace::Function, result);
287+
attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx));
286288
} else {
287289
// LLVM doesn't have an attribute to represent ReadOnly + SideEffect
288290
}
291+
attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
289292

290293
// Write results to outputs
291294
for (idx, op) in operands.iter().enumerate() {

0 commit comments

Comments
 (0)