Skip to content

Commit ffba8e8

Browse files
nbdd0121nbdd0121
committed
Implement asm_const_ptr for global_asm and naked_asm
1 parent a393bbf commit ffba8e8

File tree

12 files changed

+242
-66
lines changed

12 files changed

+242
-66
lines changed

Diff for: ‎compiler/rustc_codegen_llvm/src/asm.rs

+107-32
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use rustc_codegen_ssa::traits::*;
77
use rustc_data_structures::fx::FxHashMap;
88
use rustc_middle::ty::Instance;
99
use rustc_middle::ty::layout::TyAndLayout;
10-
use rustc_middle::{bug, span_bug};
10+
use rustc_middle::{bug, mir, span_bug};
1111
use rustc_span::{Pos, Span, Symbol, sym};
1212
use rustc_target::asm::*;
1313
use smallvec::SmallVec;
@@ -396,6 +396,111 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
396396
let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
397397
&& !options.contains(InlineAsmOptions::ATT_SYNTAX);
398398

399+
// Convert all operands to string interpolations
400+
let converted_operands = operands
401+
.iter()
402+
.enumerate()
403+
.map(|(operand_idx, operand)| {
404+
match *operand {
405+
GlobalAsmOperandRef::Interpolate { ref string } => {
406+
// Const operands get injected directly into the
407+
// template. Note that we don't need to escape $
408+
// here unlike normal inline assembly.
409+
string.to_owned()
410+
}
411+
GlobalAsmOperandRef::ConstPointer { value, instance } => {
412+
let (prov, offset) = value.into_parts();
413+
let global_alloc = self.tcx.global_alloc(prov.alloc_id());
414+
let llval = 'llval: {
415+
let alloc = match global_alloc {
416+
mir::interpret::GlobalAlloc::Function { instance } => {
417+
break 'llval self.get_fn(instance);
418+
}
419+
mir::interpret::GlobalAlloc::VTable(ty, dyn_ty) => self
420+
.tcx
421+
.global_alloc(self.tcx.vtable_allocation((
422+
ty,
423+
dyn_ty.principal().map(|principal| {
424+
self.tcx
425+
.instantiate_bound_regions_with_erased(principal)
426+
}),
427+
)))
428+
.unwrap_memory(),
429+
mir::interpret::GlobalAlloc::Static(def_id) => {
430+
break 'llval self
431+
.renamed_statics
432+
.borrow()
433+
.get(&def_id)
434+
.copied()
435+
.unwrap_or_else(|| self.get_static(def_id));
436+
}
437+
mir::interpret::GlobalAlloc::Memory(alloc) => alloc,
438+
};
439+
440+
// For ZSTs directly codegen an aligned pointer.
441+
if alloc.inner().len() == 0 {
442+
assert_eq!(offset.bytes(), 0);
443+
return format!("{}", alloc.inner().align.bytes());
444+
}
445+
446+
let asm_name = self.tcx.symbol_name(instance);
447+
let sym_name = format!("{asm_name}.{operand_idx}");
448+
449+
let init = crate::consts::const_alloc_to_llvm(
450+
self, alloc, /*static*/ false,
451+
);
452+
let alloc = alloc.inner();
453+
let g = self.static_addr_of_mut(init, alloc.align, None);
454+
if alloc.mutability.is_not() {
455+
// NB: we can't use `static_addr_of_impl` here to avoid sharing
456+
// the global, as we need to set name and linkage.
457+
unsafe { llvm::LLVMSetGlobalConstant(g, llvm::True) };
458+
}
459+
460+
llvm::set_value_name(g, sym_name.as_bytes());
461+
462+
// `static_addr_of_mut` gives us a private global which can't be
463+
// used by global asm. Update it to a hidden internal global instead.
464+
llvm::set_linkage(g, llvm::Linkage::InternalLinkage);
465+
llvm::set_visibility(g, llvm::Visibility::Hidden);
466+
g
467+
};
468+
self.add_compiler_used_global(llval);
469+
let symbol = llvm::build_string(|s| unsafe {
470+
llvm::LLVMRustGetMangledName(llval, s);
471+
})
472+
.expect("symbol is not valid UTF-8");
473+
474+
let offset = offset.bytes();
475+
if offset != 0 { format!("{symbol}+{offset}") } else { symbol }
476+
}
477+
GlobalAsmOperandRef::SymFn { instance } => {
478+
let llval = self.get_fn(instance);
479+
self.add_compiler_used_global(llval);
480+
let symbol = llvm::build_string(|s| unsafe {
481+
llvm::LLVMRustGetMangledName(llval, s);
482+
})
483+
.expect("symbol is not valid UTF-8");
484+
symbol
485+
}
486+
GlobalAsmOperandRef::SymStatic { def_id } => {
487+
let llval = self
488+
.renamed_statics
489+
.borrow()
490+
.get(&def_id)
491+
.copied()
492+
.unwrap_or_else(|| self.get_static(def_id));
493+
self.add_compiler_used_global(llval);
494+
let symbol = llvm::build_string(|s| unsafe {
495+
llvm::LLVMRustGetMangledName(llval, s);
496+
})
497+
.expect("symbol is not valid UTF-8");
498+
symbol
499+
}
500+
}
501+
})
502+
.collect::<Vec<_>>();
503+
399504
// Build the template string
400505
let mut template_str = String::new();
401506
if intel_syntax {
@@ -405,37 +510,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
405510
match *piece {
406511
InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
407512
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
408-
match operands[operand_idx] {
409-
GlobalAsmOperandRef::Interpolate { ref string } => {
410-
// Const operands get injected directly into the
411-
// template. Note that we don't need to escape $
412-
// here unlike normal inline assembly.
413-
template_str.push_str(string);
414-
}
415-
GlobalAsmOperandRef::SymFn { instance } => {
416-
let llval = self.get_fn(instance);
417-
self.add_compiler_used_global(llval);
418-
let symbol = llvm::build_string(|s| unsafe {
419-
llvm::LLVMRustGetMangledName(llval, s);
420-
})
421-
.expect("symbol is not valid UTF-8");
422-
template_str.push_str(&symbol);
423-
}
424-
GlobalAsmOperandRef::SymStatic { def_id } => {
425-
let llval = self
426-
.renamed_statics
427-
.borrow()
428-
.get(&def_id)
429-
.copied()
430-
.unwrap_or_else(|| self.get_static(def_id));
431-
self.add_compiler_used_global(llval);
432-
let symbol = llvm::build_string(|s| unsafe {
433-
llvm::LLVMRustGetMangledName(llval, s);
434-
})
435-
.expect("symbol is not valid UTF-8");
436-
template_str.push_str(&symbol);
437-
}
438-
}
513+
template_str.push_str(&converted_operands[operand_idx])
439514
}
440515
}
441516
}

Diff for: ‎compiler/rustc_codegen_ssa/src/common.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ pub fn asm_const_to_str<'tcx>(
166166
};
167167
let value = scalar.assert_scalar_int().to_bits(ty_and_layout.size);
168168
match ty_and_layout.ty.kind() {
169-
ty::Uint(_) => value.to_string(),
169+
ty::Uint(_) | ty::RawPtr(..) | ty::Ref(..) => value.to_string(),
170170
ty::Int(int_ty) => match int_ty.normalize(tcx.sess.target.pointer_width) {
171171
ty::IntTy::I8 => (value as i8).to_string(),
172172
ty::IntTy::I16 => (value as i16).to_string(),

Diff for: ‎compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

+23-9
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind};
22
use rustc_attr_parsing::InstructionSetAttr;
33
use rustc_hir::def_id::DefId;
44
use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility};
5-
use rustc_middle::mir::{Body, InlineAsmOperand};
5+
use rustc_middle::mir::{self, Body, InlineAsmOperand};
66
use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
77
use rustc_middle::ty::{Instance, Ty, TyCtxt};
88
use rustc_middle::{bug, span_bug, ty};
@@ -69,14 +69,28 @@ fn inline_to_global_operand<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
6969
ty::EarlyBinder::bind(value.ty()),
7070
);
7171

72-
let string = common::asm_const_to_str(
73-
cx.tcx(),
74-
value.span,
75-
const_value,
76-
cx.layout_of(mono_type),
77-
);
78-
79-
GlobalAsmOperandRef::Interpolate { string }
72+
let mir::ConstValue::Scalar(scalar) = const_value else {
73+
span_bug!(
74+
value.span,
75+
"expected Scalar for promoted asm const, but got {:#?}",
76+
const_value
77+
)
78+
};
79+
match scalar {
80+
mir::interpret::Scalar::Int(_) => {
81+
let string = common::asm_const_to_str(
82+
cx.tcx(),
83+
value.span,
84+
const_value,
85+
cx.layout_of(mono_type),
86+
);
87+
GlobalAsmOperandRef::Interpolate { string }
88+
}
89+
mir::interpret::Scalar::Ptr(value, _) => GlobalAsmOperandRef::ConstPointer {
90+
value,
91+
instance: Instance::mono(cx.tcx(), instance.def_id()),
92+
},
93+
}
8094
}
8195
InlineAsmOperand::SymFn { value } => {
8296
let mono_type = instance.instantiate_mir_and_normalize_erasing_regions(

Diff for: ‎compiler/rustc_codegen_ssa/src/mono_item.rs

+19-9
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
use rustc_hir as hir;
22
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
3-
use rustc_middle::mir::interpret::ErrorHandled;
3+
use rustc_middle::mir::interpret::{ErrorHandled, Scalar};
44
use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility};
55
use rustc_middle::ty::Instance;
66
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
7-
use rustc_middle::{span_bug, ty};
7+
use rustc_middle::{mir, span_bug, ty};
88
use tracing::debug;
99

1010
use crate::traits::*;
@@ -48,13 +48,23 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
4848
.tcx()
4949
.typeck_body(anon_const.body)
5050
.node_type(anon_const.hir_id);
51-
let string = common::asm_const_to_str(
52-
cx.tcx(),
53-
*op_sp,
54-
const_value,
55-
cx.layout_of(ty),
56-
);
57-
GlobalAsmOperandRef::Interpolate { string }
51+
let mir::ConstValue::Scalar(scalar) = const_value else {
52+
span_bug!(*op_sp, "expected Scalar for promoted asm const, but got {:#?}", const_value)
53+
};
54+
match scalar {
55+
Scalar::Int(_) => {
56+
let string = common::asm_const_to_str(
57+
cx.tcx(),
58+
*op_sp,
59+
const_value,
60+
cx.layout_of(ty),
61+
);
62+
GlobalAsmOperandRef::Interpolate { string }
63+
}
64+
Scalar::Ptr(value, _) => {
65+
GlobalAsmOperandRef::ConstPointer { value, instance: Instance::mono(cx.tcx(), item_id.owner_id.to_def_id()) }
66+
}
67+
}
5868
}
5969
Err(ErrorHandled::Reported { .. }) => {
6070
// An error has already been reported and

Diff for: ‎compiler/rustc_codegen_ssa/src/traits/asm.rs

+15-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
22
use rustc_hir::def_id::DefId;
3+
use rustc_middle::mir;
34
use rustc_middle::ty::Instance;
45
use rustc_span::Span;
56
use rustc_target::asm::InlineAsmRegOrRegClass;
@@ -44,9 +45,20 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
4445

4546
#[derive(Debug)]
4647
pub enum GlobalAsmOperandRef<'tcx> {
47-
Interpolate { string: String },
48-
SymFn { instance: Instance<'tcx> },
49-
SymStatic { def_id: DefId },
48+
Interpolate {
49+
string: String,
50+
},
51+
ConstPointer {
52+
value: mir::interpret::Pointer,
53+
/// Instance that instantiates this const operand.
54+
instance: Instance<'tcx>,
55+
},
56+
SymFn {
57+
instance: Instance<'tcx>,
58+
},
59+
SymStatic {
60+
def_id: DefId,
61+
},
5062
}
5163

5264
pub trait AsmBuilderMethods<'tcx>: BackendTypes {

Diff for: ‎compiler/rustc_monomorphize/src/collector.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -490,8 +490,7 @@ fn collect_items_rec<'tcx>(
490490
Err(ErrorHandled::TooGeneric(..)) => {
491491
span_bug!(*op_sp, "asm const cannot be resolved; too generic")
492492
}
493-
Err(err @ ErrorHandled::Reported(..)) => {
494-
err.emit_note(tcx);
493+
Err(ErrorHandled::Reported(..)) => {
495494
continue;
496495
}
497496
}

Diff for: ‎tests/assembly/asm/global_asm.rs

+5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
//@ compile-flags: -C symbol-mangling-version=v0
66

77
#![crate_type = "rlib"]
8+
#![feature(asm_const_ptr)]
89

910
use std::arch::global_asm;
1011

@@ -26,6 +27,10 @@ global_asm!("call {}", sym my_func);
2627
global_asm!("lea rax, [rip + {}]", sym MY_STATIC);
2728
// CHECK: call _RNvC[[CRATE_IDENT:[a-zA-Z0-9]{12}]]_10global_asm6foobar
2829
global_asm!("call {}", sym foobar);
30+
// CHECK: lea rax, [rip + _RNSC[[CRATE_IDENT]]_10global_asms4_10global_asm.0]
31+
global_asm!("lea rax, [rip + {}]", const &1);
32+
// CHECK: lea rax, [rip + _RNSC[[CRATE_IDENT]]_10global_asms5_10global_asm.0+4]
33+
global_asm!("lea rax, [rip + {}]", const &[1; 2][1]);
2934
// CHECK: _RNvC[[CRATE_IDENT]]_10global_asm6foobar:
3035
fn foobar() {
3136
loop {}

Diff for: ‎tests/assembly/asm/x86-types.rs

+13-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
//@ compile-flags: -C target-feature=+avx512bw
1010
//@ compile-flags: -Zmerge-functions=disabled
1111

12-
#![feature(no_core, repr_simd, f16, f128)]
12+
#![feature(no_core, repr_simd, f16, f128, asm_const_ptr)]
1313
#![crate_type = "rlib"]
1414
#![no_core]
1515
#![allow(asm_sub_register, non_camel_case_types)]
@@ -92,6 +92,18 @@ pub unsafe fn sym_fn() {
9292
asm!("call {}", sym extern_func);
9393
}
9494

95+
// NOTE: this only works for x64, as this test is compiled with PIC,
96+
// and on x86 PIC symbol can't be constant.
97+
// x86_64-LABEL: const_ptr:
98+
// x86_64: #APP
99+
// x86_64: mov al, byte ptr [{{.*}}anon{{.*}}]
100+
// x86_64: #NO_APP
101+
#[cfg(x86_64)]
102+
#[no_mangle]
103+
pub unsafe fn const_ptr() {
104+
asm!("mov al, byte ptr [{}]", const &1);
105+
}
106+
95107
// CHECK-LABEL: sym_static:
96108
// CHECK: #APP
97109
// CHECK: mov al, byte ptr [extern_static]

Diff for: ‎tests/ui/asm/invalid-const-operand.rs

+6-1
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,21 @@ global_asm!("{}", const 0 as *mut u8);
1717

1818
fn test1() {
1919
unsafe {
20-
// Const operands must be integers and must be constants.
20+
// Const operands must be integers or thin pointers
2121

2222
asm!("{}", const 0);
2323
asm!("{}", const 0i32);
2424
asm!("{}", const 0i128);
2525
asm!("{}", const 0f32);
2626
//~^ ERROR invalid type for `const` operand
2727
asm!("{}", const 0 as *mut u8);
28+
asm!("{}", const &0);
2829
asm!("{}", const b"Foo".as_slice());
2930
//~^ ERROR invalid type for `const` operand
31+
32+
asm!("{}", const test1 as fn());
33+
asm!("{}", const test1);
34+
//~^ ERROR invalid type for `const` operand
3035
}
3136
}
3237

0 commit comments

Comments
 (0)