Translate the llvm.fshl intrinsic function

"Funnel shift left" doesn't have an analogue in the OpenCL
ExtInst set. We unroll `llvm.fshl.i<n>(i<n>, i<n>, i<n>)`
into a small algorithm that performs the actual funnel shift.

A detailed description of FSHL can be found at
https://llvm.org/docs/LangRef.html#llvm-fshl-intrinsic

Signed-off-by: Artem Gindinson <[email protected]>

Author: Artem Gindinson

llvm-spirv/lib/SPIRV/SPIRVRegularizeLLVM.cpp

@@ -85,6 +85,20 @@ class SPIRVRegularizeLLVM : public ModulePass {
   /// @spirv.llvm_memset_* and replace it with @llvm.memset.
   void lowerMemset(MemSetInst *MSI);
 
+  /// No SPIR-V counterpart for @llvm.fshl.i* intrinsic. It will be lowered
+  /// to a newly generated @spirv.llvm_fshl_i* function.
+  /// Conceptually, FSHL:
+  /// 1. concatenates the ints, the first one being the more significant;
+  /// 2. performs a left shift-rotate on the resulting doubled-sized int;
+  /// 3. returns the most significant bits of the shift-rotate result,
+  ///    the number of bits being equal to the size of the original integers.
+  /// The actual implementation algorithm will be slightly different to speed
+  /// things up.
+  void lowerFunnelShiftLeft(IntrinsicInst *FSHLIntrinsic);
+  void buildFunnelShiftLeftFunc(Function *FSHLFunc);
+
+  static std::string lowerLLVMIntrinsicName(IntrinsicInst *II);
+
   static char ID;
 
 private:
@@ -94,17 +108,22 @@ class SPIRVRegularizeLLVM : public ModulePass {
 
 char SPIRVRegularizeLLVM::ID = 0;
 
-void SPIRVRegularizeLLVM::lowerMemset(MemSetInst *MSI) {
-  if (isa<Constant>(MSI->getValue()) && isa<ConstantInt>(MSI->getLength()))
-    return; // To be handled in LLVMToSPIRV::transIntrinsicInst
-  Function *IntrinsicFunc = MSI->getCalledFunction();
+std::string SPIRVRegularizeLLVM::lowerLLVMIntrinsicName(IntrinsicInst *II) {
+  Function *IntrinsicFunc = II->getCalledFunction();
   assert(IntrinsicFunc && "Missing function");
   std::string FuncName = IntrinsicFunc->getName().str();
   std::replace(FuncName.begin(), FuncName.end(), '.', '_');
   FuncName = "spirv." + FuncName;
+  return FuncName;
+}
+
+void SPIRVRegularizeLLVM::lowerMemset(MemSetInst *MSI) {
+  if (isa<Constant>(MSI->getValue()) && isa<ConstantInt>(MSI->getLength()))
+    return; // To be handled in LLVMToSPIRV::transIntrinsicInst
+
+  std::string FuncName = lowerLLVMIntrinsicName(MSI);
   if (MSI->isVolatile())
     FuncName += ".volatile";
-
   // Redirect @llvm.memset.* call to @spirv.llvm_memset_*
   Function *F = M->getFunction(FuncName);
   if (F) {
@@ -137,6 +156,75 @@ void SPIRVRegularizeLLVM::lowerMemset(MemSetInst *MSI) {
   return;
 }
 
+void SPIRVRegularizeLLVM::buildFunnelShiftLeftFunc(Function *FSHLFunc) {
+  if (!FSHLFunc->empty())
+    return;
+
+  auto *IntTy = dyn_cast<IntegerType>(FSHLFunc->getReturnType());
+  assert(IntTy && "llvm.fshl: expected an integer return type");
+  assert(FSHLFunc->arg_size() == 3 && "llvm.fshl: expected 3 arguments");
+  for (Argument &Arg : FSHLFunc->args())
+    assert(Arg.getType()->getTypeID() == IntTy->getTypeID() &&
+           "llvm.fshl: mismatched return type and argument types");
+
+  // Our function will require 3 basic blocks; the purpose of each will be
+  // clarified below.
+  auto *CondBB = BasicBlock::Create(M->getContext(), "cond", FSHLFunc);
+  auto *RotateBB =
+      BasicBlock::Create(M->getContext(), "rotate", FSHLFunc); // Main logic
+  auto *PhiBB = BasicBlock::Create(M->getContext(), "phi", FSHLFunc);
+
+  IRBuilder<> Builder(CondBB);
+  // If the number of bits to rotate for is divisible by the bitsize,
+  // the shift becomes useless, and we should bypass the main logic in that
+  // case.
+  unsigned BitWidth = IntTy->getIntegerBitWidth();
+  ConstantInt *BitWidthConstant = Builder.getInt({BitWidth, BitWidth});
+  auto *RotateModVal =
+      Builder.CreateURem(/*Rotate*/ FSHLFunc->getArg(2), BitWidthConstant);
+  ConstantInt *ZeroConstant = Builder.getInt({BitWidth, 0});
+  auto *CheckRotateModIfZero = Builder.CreateICmpEQ(RotateModVal, ZeroConstant);
+  Builder.CreateCondBr(CheckRotateModIfZero, /*True*/ PhiBB,
+                       /*False*/ RotateBB);
+
+  // Build the actual funnel shift rotate logic.
+  Builder.SetInsertPoint(RotateBB);
+  // Shift the more significant number left, the "rotate" number of bits
+  // will be 0-filled on the right as a result of this regular shift.
+  auto *ShiftLeft = Builder.CreateShl(FSHLFunc->getArg(0), RotateModVal);
+  // We want the "rotate" number of the second int's MSBs to occupy the
+  // rightmost "0 space" left by the previous operation. Therefore,
+  // subtract the "rotate" number from the integer bitsize...
+  auto *SubRotateVal = Builder.CreateSub(BitWidthConstant, RotateModVal);
+  // ...and right-shift the second int by this number, zero-filling the MSBs.
+  auto *ShiftRight = Builder.CreateLShr(FSHLFunc->getArg(1), SubRotateVal);
+  // A simple binary addition of the shifted ints yields the final result.
+  auto *FunnelShiftRes = Builder.CreateOr(ShiftLeft, ShiftRight);
+  Builder.CreateBr(PhiBB);
+
+  // PHI basic block. If no actual rotate was required, return the first, more
+  // significant int. E.g. for 32-bit integers, it's equivalent to concatenating
+  // the 2 ints and taking 32 MSBs.
+  Builder.SetInsertPoint(PhiBB);
+  PHINode *Phi = Builder.CreatePHI(IntTy, 0);
+  Phi->addIncoming(FunnelShiftRes, RotateBB);
+  Phi->addIncoming(FSHLFunc->getArg(0), CondBB);
+  Builder.CreateRet(Phi);
+}
+
+void SPIRVRegularizeLLVM::lowerFunnelShiftLeft(IntrinsicInst *FSHLIntrinsic) {
+  // Get a separate function - otherwise, we'd have to rework the CFG of the
+  // current one. Then simply replace the intrinsic uses with a call to the new
+  // function.
+  FunctionType *FSHLFuncTy = FSHLIntrinsic->getFunctionType();
+  Type *FSHLRetTy = FSHLFuncTy->getReturnType();
+  const std::string FuncName = lowerLLVMIntrinsicName(FSHLIntrinsic);
+  Function *FSHLFunc =
+      getOrCreateFunction(M, FSHLRetTy, FSHLFuncTy->params(), FuncName);
+  buildFunnelShiftLeftFunc(FSHLFunc);
+  FSHLIntrinsic->setCalledFunction(FSHLFunc);
+}
+
 bool SPIRVRegularizeLLVM::runOnModule(Module &Module) {
   M = &Module;
   Ctx = &M->getContext();
@@ -170,8 +258,11 @@ bool SPIRVRegularizeLLVM::regularize() {
           Function *CF = Call->getCalledFunction();
           if (CF && CF->isIntrinsic()) {
             removeFnAttr(Call, Attribute::NoUnwind);
-            if (auto *MSI = dyn_cast<MemSetInst>(Call))
+            auto *II = cast<IntrinsicInst>(Call);
+            if (auto *MSI = dyn_cast<MemSetInst>(II))
               lowerMemset(MSI);
+            else if (II->getIntrinsicID() == Intrinsic::fshl)
+              lowerFunnelShiftLeft(II);
           }
         }
 
@@ -254,7 +345,7 @@ bool SPIRVRegularizeLLVM::regularize() {
       }
     }
     for (Instruction *V : ToErase) {
-      assert(V->user_empty());
+      assert(V->user_empty() && "User non-empty\n");
       V->eraseFromParent();
     }
   }

llvm-spirv/test/llvm.fshl.ll

@@ -0,0 +1,63 @@
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc -spirv-text -o - | FileCheck %s
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: spirv-val %t.spv
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64-unknown-unknown"
+
+; Function Attrs: nounwind readnone
+define spir_func i32 @Test(i32 %x, i32 %y) local_unnamed_addr #0 {
+entry:
+  %0 = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 8)
+  ret i32 %0
+}
+
+; CHECK: TypeInt [[TYPE_INT:[0-9]+]] 32 0
+; CHECK-DAG: Constant [[TYPE_INT]] [[CONST_ROTATE:[0-9]+]] 8
+; CHECK-DAG: Constant [[TYPE_INT]] [[CONST_TYPE_SIZE:[0-9]+]] 32
+; CHECK-DAG: Constant [[TYPE_INT]] [[CONST_0:[0-9]+]] 0
+; CHECK: TypeFunction [[TYPE_ORIG_FUNC:[0-9]+]] [[TYPE_INT]] [[TYPE_INT]] [[TYPE_INT]]
+; CHECK: TypeFunction [[TYPE_FSHL_FUNC:[0-9]+]] [[TYPE_INT]] [[TYPE_INT]] [[TYPE_INT]] [[TYPE_INT]]
+; CHECK: TypeBool [[TYPE_BOOL:[0-9]+]]
+
+; CHECK: Function [[TYPE_INT]] {{[0-9]+}} {{[0-9]+}} [[TYPE_ORIG_FUNC]]
+; CHECK: FunctionParameter [[TYPE_INT]] [[X:[0-9]+]]
+; CHECK: FunctionParameter [[TYPE_INT]] [[Y:[0-9]+]]
+; CHECK: FunctionCall [[TYPE_INT]] [[CALL:[0-9]+]] [[FSHL_FUNC:[0-9]+]] [[X]] [[Y]] [[CONST_ROTATE]]
+; CHECK: ReturnValue [[CALL]]
+
+; CHECK: Function [[TYPE_INT]] [[FSHL_FUNC]] {{[0-9]+}} [[TYPE_FSHL_FUNC]]
+; CHECK: FunctionParameter [[TYPE_INT]] [[X_FSHL:[0-9]+]]
+; CHECK: FunctionParameter [[TYPE_INT]] [[Y_FSHL:[0-9]+]]
+; CHECK: FunctionParameter [[TYPE_INT]] [[ROT:[0-9]+]]
+
+; CHECK: Label [[MAIN_BB:[0-9]+]]
+; CHECK: UMod [[TYPE_INT]] [[ROTATE_MOD_SIZE:[0-9]+]] [[ROT]] [[CONST_TYPE_SIZE]]
+; CHECK: IEqual [[TYPE_BOOL]] [[ZERO_COND:[0-9]+]] [[ROTATE_MOD_SIZE]] [[CONST_0]]
+; CHECK: BranchConditional [[ZERO_COND]] [[PHI_BB:[0-9]+]] [[ROTATE_BB:[0-9]+]]
+
+; CHECK: Label [[ROTATE_BB]]
+; CHECK: ShiftLeftLogical [[TYPE_INT]] [[X_SHIFT_LEFT:[0-9]+]] [[X_FSHL]] [[ROTATE_MOD_SIZE]]
+; CHECK: ISub [[TYPE_INT]] [[NEG_ROTATE:[0-9]+]] [[CONST_TYPE_SIZE]] [[ROTATE_MOD_SIZE]]
+; CHECK: ShiftRightLogical [[TYPE_INT]] [[Y_SHIFT_RIGHT:[0-9]+]] [[Y_FSHL]] [[NEG_ROTATE]]
+; CHECK: BitwiseOr [[TYPE_INT]] [[FSHL_RESULT:[0-9]+]] [[X_SHIFT_LEFT]] [[Y_SHIFT_RIGHT]]
+; CHECK: Branch [[PHI_BB]]
+
+; CHECK: Label [[PHI_BB]]
+; CHECK: Phi [[TYPE_INT]] [[PHI_INST:[0-9]+]] [[FSHL_RESULT]] [[ROTATE_BB]] [[X_FSHL]] [[MAIN_BB]]
+; CHECK: ReturnValue [[PHI_INST]]
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i32 @llvm.fshl.i32(i32, i32, i32) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "denorms-are-zero"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable willreturn }
+
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, i32 0}
+!2 = !{i32 1, i32 2}