rustc_codegen_ssa: Use llvm.invariant intrinsics on arguments that are immutable references; off by default.

compiler/rustc_codegen_gcc/src/builder.rs

@@ -1235,6 +1235,16 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // TODO(antoyo)
     }
 
+    fn invariant_start(&mut self, _value_ptr: RValue<'gcc>, _size: Size) -> RValue<'gcc> {
+        // TODO
+        unimplemented!()
+    }
+
+    fn invariant_end(&mut self, _marker_ptr: RValue<'gcc>, _value_ptr: RValue<'gcc>, _size: Size) {
+        // TODO
+        unimplemented!()
+    }
+
     fn call(
         &mut self,
         _typ: Type<'gcc>,

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1113,6 +1113,21 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
     }
 
+    fn invariant_start(&mut self, value_ptr: &'ll Value, size: Size) -> &'ll Value {
+        let size = size.bytes();
+        let value_ptr = self.pointercast(value_ptr, self.cx.type_i8p());
+        self.call_intrinsic("llvm.invariant.start.p0i8", &[self.cx.const_u64(size), value_ptr])
+    }
+
+    fn invariant_end(&mut self, marker_ptr: &'ll Value, value_ptr: &'ll Value, size: Size) {
+        let size = size.bytes();
+        let value_ptr = self.pointercast(value_ptr, self.cx.type_i8p());
+        self.call_intrinsic(
+            "llvm.invariant.end.p0i8",
+            &[marker_ptr, self.cx.const_u64(size), value_ptr],
+        );
+    }
+
     fn instrprof_increment(
         &mut self,
         fn_name: &'ll Value,

compiler/rustc_codegen_llvm/src/context.rs

@@ -629,6 +629,7 @@ impl<'ll> CodegenCx<'ll, '_> {
         }
 
         let i8p = self.type_i8p();
+        let unitp = self.type_ptr_to(self.type_struct(&[], false));
         let void = self.type_void();
         let i1 = self.type_i1();
         let t_i8 = self.type_i8();
@@ -840,6 +841,9 @@ impl<'ll> CodegenCx<'ll, '_> {
         ifn!("llvm.lifetime.start.p0i8", fn(t_i64, i8p) -> void);
         ifn!("llvm.lifetime.end.p0i8", fn(t_i64, i8p) -> void);
 
+        ifn!("llvm.invariant.start.p0i8", fn(t_i64, i8p) -> unitp);
+        ifn!("llvm.invariant.end.p0i8", fn(unitp, t_i64, i8p) -> void);
+
         ifn!("llvm.expect.i1", fn(i1, i1) -> i1);
         ifn!("llvm.eh.typeid.for", fn(i8p) -> t_i32);
         ifn!("llvm.localescape", fn(...) -> void);

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -249,6 +249,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     /// Generates code for a `Resume` terminator.
     fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, mut bx: Bx) {
+        self.emit_invariant_end_markers(&mut bx);
+
         if let Some(funclet) = helper.funclet(self) {
             bx.cleanup_ret(funclet, None);
         } else {
@@ -306,6 +308,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 
     fn codegen_return_terminator(&mut self, mut bx: Bx) {
+        self.emit_invariant_end_markers(&mut bx);
+
         // Call `va_end` if this is the definition of a C-variadic function.
         if self.fn_abi.c_variadic {
             // The `VaList` "spoofed" argument is just after all the real arguments.
@@ -1723,6 +1727,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 }
 
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+    fn emit_invariant_end_markers(&mut self, bx: &mut Bx) {
+        for invariant_value in self.invariant_values.iter().rev() {
+            bx.invariant_end(
+                invariant_value.marker_ptr,
+                invariant_value.value_ptr,
+                invariant_value.size,
+            )
+        }
+    }
+}
+
 enum ReturnDest<'tcx, V> {
     // Do nothing; the return value is indirect or ignored.
     Nothing,

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -1,9 +1,11 @@
 use crate::traits::*;
-use rustc_middle::mir;
 use rustc_middle::mir::interpret::ErrorHandled;
-use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout};
+use rustc_middle::mir::{self, Mutability};
+use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TypeFoldable, TypeVisitable};
-use rustc_target::abi::call::{FnAbi, PassMode};
+use rustc_span::DUMMY_SP;
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
+use rustc_target::abi::Size;
 
 use std::iter;
 
@@ -87,6 +89,11 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
 
     /// Caller location propagated if this function has `#[track_caller]`.
     caller_location: Option<OperandRef<'tcx, Bx::Value>>,
+
+    /// Information about pointers that are known immutable for the entire execution of the
+    /// function. We need to keep this information around so that we can call `invariant_end()`
+    /// before exiting this function.
+    invariant_values: Vec<InvariantValue<Bx::Value>>,
 }
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
@@ -103,6 +110,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 }
 
+/// Information about pointers that are known immutable for the entire execution of the
+/// function. We need to keep this information around so that we can call `invariant_end()`
+/// before exiting this function.
+struct InvariantValue<V> {
+    marker_ptr: V,
+    value_ptr: V,
+    size: Size,
+}
+
 enum LocalRef<'tcx, V> {
     Place(PlaceRef<'tcx, V>),
     /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place).
@@ -178,6 +194,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         debug_context,
         per_local_var_debug_info: None,
         caller_location: None,
+        invariant_values: vec![],
     };
 
     fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut bx);
@@ -325,6 +342,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     PassMode::Direct(_) => {
                         let llarg = bx.get_param(llarg_idx);
                         llarg_idx += 1;
+                        emit_invariant_markers_for(bx, fx, arg, llarg);
                         return local(OperandRef::from_immediate_or_packed_pair(
                             bx, llarg, arg.layout,
                         ));
@@ -398,6 +416,23 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     args
 }
 
+fn emit_invariant_markers_for<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+    bx: &mut Bx,
+    fx: &mut FunctionCx<'a, 'tcx, Bx>,
+    arg: &ArgAbi<'tcx, Ty<'tcx>>,
+    value_ptr: Bx::Value,
+) {
+    if bx.sess().opts.unstable_opts.emit_invariant_markers {
+        if let ty::Ref(_, subty, Mutability::Not) = arg.layout.ty.kind() {
+            if subty.is_freeze(bx.tcx().at(DUMMY_SP), ty::ParamEnv::reveal_all()) {
+                let size = bx.cx().layout_of(*subty).size;
+                let marker_ptr = bx.invariant_start(value_ptr, size);
+                fx.invariant_values.push(InvariantValue { marker_ptr, value_ptr, size });
+            }
+        }
+    }
+}
+
 mod analyze;
 mod block;
 pub mod constant;

compiler/rustc_codegen_ssa/src/traits/builder.rs

@@ -310,6 +310,9 @@ pub trait BuilderMethods<'a, 'tcx>:
     /// Called for `StorageDead`
     fn lifetime_end(&mut self, ptr: Self::Value, size: Size);
 
+    fn invariant_start(&mut self, ptr: Self::Value, size: Size) -> Self::Value;
+    fn invariant_end(&mut self, marker_ptr: Self::Value, value_ptr: Self::Value, size: Size);
+
     fn instrprof_increment(
         &mut self,
         fn_name: Self::Value,

compiler/rustc_session/src/options.rs

@@ -1295,6 +1295,8 @@ options! {
         an additional `.html` file showing the computed coverage spans."),
     dwarf_version: Option<u32> = (None, parse_opt_number, [TRACKED],
         "version of DWARF debug information to emit (default: 2 or 4, depending on platform)"),
+    emit_invariant_markers: bool = (false, parse_bool, [UNTRACKED],
+        "emit `llvm.invariant` markers, which may enable better optimization (default: no)"),
     emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED],
         "emit a section containing stack size metadata (default: no)"),
     emit_thin_lto: bool = (true, parse_bool, [TRACKED],

src/test/codegen/invariant-basic.rs

@@ -0,0 +1,12 @@
+// compile-flags: -C opt-level=0 -Z emit-invariant-markers=yes
+#![crate_type="lib"]
+
+#[no_mangle]
+pub fn foo(x: &i32) {
+    // CHECK-LABEL: @foo
+    // CHECK: call {{.*}} @llvm.invariant.start{{[^(]*}}(i64 4
+    // CHECK: call void @{{.*}}drop{{.*}}
+    // CHECK: call void @llvm.invariant.end
+    // CHECK: ret void
+    drop(x);
+}

src/test/rustdoc-ui/z-help.stdout

@@ -36,6 +36,7 @@
     -Z                       dump-mir-graphviz=val -- in addition to `.mir` files, create graphviz `.dot` files (and with `-Z instrument-coverage`, also create a `.dot` file for the MIR-derived coverage graph) (default: no)
     -Z                       dump-mir-spanview=val -- in addition to `.mir` files, create `.html` files to view spans for all `statement`s (including terminators), only `terminator` spans, or computed `block` spans (one span encompassing a block's terminator and all statements). If `-Z instrument-coverage` is also enabled, create an additional `.html` file showing the computed coverage spans.
     -Z                           dwarf-version=val -- version of DWARF debug information to emit (default: 2 or 4, depending on platform)
+    -Z                  emit-invariant-markers=val -- emit `llvm.invariant` markers, which may enable better optimization (default: no)
     -Z                        emit-stack-sizes=val -- emit a section containing stack size metadata (default: no)
     -Z                           emit-thin-lto=val -- emit the bc module with thin LTO info (default: yes)
     -Z               export-executable-symbols=val -- export symbols from executables, as if they were dynamic libraries