[CIR] Make use of !invariant.group metadata for const allocas (#1159)

This PR updates the LLVM lowering part of load and stores to const
allocas and makes use of the !invariant.group metadata in the result
LLVM IR.

The HoistAlloca pass is also updated. The const flag on a hoisted alloca
is removed for now since their uses are not always invariants. Will
update in later PRs to teach their invariants.

Author: Lancern

clang/lib/CIR/Dialect/Transforms/HoistAllocas.cpp

@@ -48,8 +48,18 @@ static void process(cir::FuncOp func) {
 
   mlir::Operation *insertPoint = &*entryBlock.begin();
 
-  for (auto alloca : allocas)
+  for (auto alloca : allocas) {
     alloca->moveBefore(insertPoint);
+    if (alloca.getConstant()) {
+      // Hoisted alloca may come from the body of a loop, in which case the
+      // stack slot is re-used by multiple objects alive in different iterations
+      // of the loop. In theory, each of these objects are still constant within
+      // their lifetimes, but currently we're not emitting metadata to further
+      // describe this. So for now let's behave conservatively and remove the
+      // const flag on nested allocas when hoisting them.
+      alloca.setConstant(false);
+    }
+  }
 }
 
 void HoistAllocasPass::runOnOperation() {

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp

@@ -959,8 +959,7 @@ mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite(
 
   if (op.getSymAddr()) {
     if (op.getOffset() == 0) {
-      rewriter.replaceAllUsesWith(op, llvmAddr);
-      rewriter.eraseOp(op);
+      rewriter.replaceOp(op, {llvmAddr});
       return mlir::success();
     }
 
@@ -1490,11 +1489,21 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
     alignment = *alignOpt;
   }
 
-  // TODO: nontemporal, invariant, syncscope.
+  auto invariant = false;
+  // Under -O1 or higher optimization levels, add the invariant metadata if the
+  // load operation loads from a constant object.
+  if (lowerMod &&
+      lowerMod->getContext().getCodeGenOpts().OptimizationLevel > 0) {
+    auto addrAllocaOp =
+        mlir::dyn_cast_if_present<cir::AllocaOp>(op.getAddr().getDefiningOp());
+    invariant = addrAllocaOp && addrAllocaOp.getConstant();
+  }
+
+  // TODO: nontemporal, syncscope.
   rewriter.replaceOpWithNewOp<mlir::LLVM::LoadOp>(
       op, llvmTy, adaptor.getAddr(), /* alignment */ alignment,
       op.getIsVolatile(), /* nontemporal */ false,
-      /* invariant */ false, /* invariantGroup */ false, ordering);
+      /* invariant */ false, /* invariantGroup */ invariant, ordering);
   return mlir::LogicalResult::success();
 }
 
@@ -1515,10 +1524,20 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
     alignment = *alignOpt;
   }
 
+  auto invariant = false;
+  // Under -O1 or higher optimization levels, add the invariant metadata if the
+  // store operation stores to a constant object.
+  if (lowerMod &&
+      lowerMod->getContext().getCodeGenOpts().OptimizationLevel > 0) {
+    auto addrAllocaOp =
+        mlir::dyn_cast_if_present<cir::AllocaOp>(op.getAddr().getDefiningOp());
+    invariant = addrAllocaOp && addrAllocaOp.getConstant();
+  }
+
   // TODO: nontemporal, syncscope.
   rewriter.replaceOpWithNewOp<mlir::LLVM::StoreOp>(
       op, adaptor.getValue(), adaptor.getAddr(), alignment, op.getIsVolatile(),
-      /* nontemporal */ false, /* invariantGroup */ false, ordering);
+      /* nontemporal */ false, /* invariantGroup */ invariant, ordering);
   return mlir::LogicalResult::success();
 }
 
@@ -3887,7 +3906,9 @@ void populateCIRToLLVMConversionPatterns(
       CIRToLLVMConstantOpLowering,
       CIRToLLVMDerivedDataMemberOpLowering,
       CIRToLLVMGetRuntimeMemberOpLowering,
-      CIRToLLVMGlobalOpLowering
+      CIRToLLVMGlobalOpLowering,
+      CIRToLLVMLoadOpLowering,
+      CIRToLLVMStoreOpLowering
       // clang-format on
       >(converter, patterns.getContext(), lowerModule);
   patterns.add<
@@ -3938,7 +3959,6 @@ void populateCIRToLLVMConversionPatterns(
       CIRToLLVMIsConstantOpLowering,
       CIRToLLVMIsFPClassOpLowering,
       CIRToLLVMLLVMIntrinsicCallOpLowering,
-      CIRToLLVMLoadOpLowering,
       CIRToLLVMMemChrOpLowering,
       CIRToLLVMMemCpyInlineOpLowering,
       CIRToLLVMMemCpyOpLowering,
@@ -3958,7 +3978,6 @@ void populateCIRToLLVMConversionPatterns(
       CIRToLLVMShiftOpLowering,
       CIRToLLVMSignBitOpLowering,
       CIRToLLVMStackSaveOpLowering,
-      CIRToLLVMStoreOpLowering,
       CIRToLLVMSwitchFlatOpLowering,
       CIRToLLVMThrowOpLowering,
       CIRToLLVMTrapOpLowering,

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h

@@ -301,8 +301,13 @@ class CIRToLLVMAllocaOpLowering
 };
 
 class CIRToLLVMLoadOpLowering : public mlir::OpConversionPattern<cir::LoadOp> {
+  cir::LowerModule *lowerMod;
+
 public:
-  using mlir::OpConversionPattern<cir::LoadOp>::OpConversionPattern;
+  CIRToLLVMLoadOpLowering(const mlir::TypeConverter &typeConverter,
+                          mlir::MLIRContext *context,
+                          cir::LowerModule *lowerModule)
+      : OpConversionPattern(typeConverter, context), lowerMod(lowerModule) {}
 
   mlir::LogicalResult
   matchAndRewrite(cir::LoadOp op, OpAdaptor,
@@ -311,8 +316,13 @@ class CIRToLLVMLoadOpLowering : public mlir::OpConversionPattern<cir::LoadOp> {
 
 class CIRToLLVMStoreOpLowering
     : public mlir::OpConversionPattern<cir::StoreOp> {
+  cir::LowerModule *lowerMod;
+
 public:
-  using mlir::OpConversionPattern<cir::StoreOp>::OpConversionPattern;
+  CIRToLLVMStoreOpLowering(const mlir::TypeConverter &typeConverter,
+                           mlir::MLIRContext *context,
+                           cir::LowerModule *lowerModule)
+      : OpConversionPattern(typeConverter, context), lowerMod(lowerModule) {}
 
   mlir::LogicalResult
   matchAndRewrite(cir::StoreOp op, OpAdaptor,

clang/test/CIR/CodeGen/const-alloca.cpp

@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir --check-prefix=CIR %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -fclangir -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll --check-prefix=LLVM %s
 
 int produce_int();
 void blackbox(const int &);
@@ -8,33 +10,33 @@ void local_const_int() {
   const int x = produce_int();
 }
 
-// CHECK-LABEL: @_Z15local_const_intv
-// CHECK:   %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const]
-// CHECK: }
+// CIR-LABEL: @_Z15local_const_intv
+// CIR:   %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const]
+// CIR: }
 
 void param_const_int(const int x) {}
 
-// CHECK-LABEL: @_Z15param_const_inti
-// CHECK:  %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const]
-// CHECK: }
+// CIR-LABEL: @_Z15param_const_inti
+// CIR:  %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const]
+// CIR: }
 
 void local_constexpr_int() {
   constexpr int x = 42;
   blackbox(x);
 }
 
-// CHECK-LABEL: @_Z19local_constexpr_intv
-// CHECK:   %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const]
-// CHECK: }
+// CIR-LABEL: @_Z19local_constexpr_intv
+// CIR:   %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const]
+// CIR: }
 
 void local_reference() {
   int x = 0;
   int &r = x;
 }
 
-// CHECK-LABEL: @_Z15local_referencev
-// CHECK:   %{{.+}} = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["r", init, const]
-// CHECK: }
+// CIR-LABEL: @_Z15local_referencev
+// CIR:   %{{.+}} = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["r", init, const]
+// CIR: }
 
 struct Foo {
   int a;
@@ -47,6 +49,39 @@ void local_const_struct() {
   const Foo x = produce_foo();
 }
 
-// CHECK-LABEL: @_Z18local_const_structv
-// CHECK:   %{{.+}} = cir.alloca !ty_Foo, !cir.ptr<!ty_Foo>, ["x", init, const]
-// CHECK: }
+// CIR-LABEL: @_Z18local_const_structv
+// CIR:   %{{.+}} = cir.alloca !ty_Foo, !cir.ptr<!ty_Foo>, ["x", init, const]
+// CIR: }
+
+[[clang::optnone]]
+int local_const_load_store() {
+  const int x = produce_int();
+  int y = x;
+  return y;
+}
+
+// CIR-LABEL: @_Z22local_const_load_storev
+// CIR: %{{.+}} = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init, const] {alignment = 4 : i64}
+// CIR: }
+
+// LLVM-LABEL: @_Z22local_const_load_storev
+//      LLVM: %[[#INIT:]] = call i32 @_Z11produce_intv()
+// LLVM-NEXT: store i32 %[[#INIT]], ptr %[[#SLOT:]], align 4, !invariant.group !{{.+}}
+// LLVM-NEXT: %{{.+}} = load i32, ptr %[[#SLOT]], align 4, !invariant.group !{{.+}}
+// LLVM: }
+
+int local_const_optimize() {
+  const int x = produce_int();
+  blackbox(x);
+  blackbox(x);
+  return x;
+}
+
+// LLVM-LABEL: @_Z20local_const_optimizev()
+// LLVM-NEXT:    %[[#slot:]] = alloca i32, align 4
+// LLVM-NEXT:    %[[#init:]] = tail call i32 @_Z11produce_intv()
+// LLVM-NEXT:    store i32 %[[#init]], ptr %[[#slot]], align 4, !invariant.group !{{.+}}
+// LLVM-NEXT:    call void @_Z8blackboxRKi(ptr nonnull %[[#slot]])
+// LLVM-NEXT:    call void @_Z8blackboxRKi(ptr nonnull %[[#slot]])
+// LLVM-NEXT:    ret i32 %[[#init]]
+// LLVM-NEXT:  }