diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index a8e4ea9429f50..278f10a670070 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -399,6 +399,13 @@ class SILateBranchLoweringPass static bool isRequired() { return true; } }; +class AMDGPUSetWavePriorityPass + : public PassInfoMixin { +public: + PreservedAnalyses run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM); +}; + FunctionPass *createAMDGPUAnnotateUniformValuesLegacy(); ModulePass *createAMDGPUPrintfRuntimeBinding(); @@ -511,7 +518,7 @@ void initializeGCNPreRAOptimizationsLegacyPass(PassRegistry &); extern char &GCNPreRAOptimizationsID; FunctionPass *createAMDGPUSetWavePriorityPass(); -void initializeAMDGPUSetWavePriorityPass(PassRegistry &); +void initializeAMDGPUSetWavePriorityLegacyPass(PassRegistry &); void initializeGCNRewritePartialRegUsesLegacyPass(llvm::PassRegistry &); extern char &GCNRewritePartialRegUsesID; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def index 6a45392b5f099..bebb69d765654 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def +++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def @@ -102,6 +102,7 @@ MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoad MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass()) MACHINE_FUNCTION_PASS("amdgpu-reserve-wwm-regs", AMDGPUReserveWWMRegsPass()) MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass()) +MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass()) MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass()) MACHINE_FUNCTION_PASS("amdgpu-nsa-reassign", GCNNSAReassignPass()) MACHINE_FUNCTION_PASS("gcn-create-vopd", GCNCreateVOPDPass()) @@ -133,7 +134,6 @@ MACHINE_FUNCTION_PASS("si-wqm", SIWholeQuadModePass()) #define DUMMY_MACHINE_FUNCTION_PASS(NAME, CREATE_PASS) DUMMY_MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass()) DUMMY_MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass()) -DUMMY_MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass()) DUMMY_MACHINE_FUNCTION_PASS("si-pre-emit-peephole", SIPreEmitPeepholePass()) // TODO: Move amdgpu-preload-kern-arg-prolog to MACHINE_FUNCTION_PASS since it diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp index ec9922db9af12..ecce50f476777 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp @@ -19,6 +19,7 @@ #include "SIInstrInfo.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachinePassManager.h" using namespace llvm; @@ -40,15 +41,11 @@ struct MBBInfo { using MBBInfoSet = DenseMap; -class AMDGPUSetWavePriority : public MachineFunctionPass { +class AMDGPUSetWavePriority { public: static char ID; - AMDGPUSetWavePriority() : MachineFunctionPass(ID) {} - - StringRef getPassName() const override { return "Set wave priority"; } - - bool runOnMachineFunction(MachineFunction &MF) override; + bool run(MachineFunction &MF); private: MachineInstr *BuildSetprioMI(MachineBasicBlock &MBB, @@ -58,15 +55,31 @@ class AMDGPUSetWavePriority : public MachineFunctionPass { const SIInstrInfo *TII; }; +class AMDGPUSetWavePriorityLegacy : public MachineFunctionPass { +public: + static char ID; + + AMDGPUSetWavePriorityLegacy() : MachineFunctionPass(ID) {} + + StringRef getPassName() const override { return "Set wave priority"; } + + bool runOnMachineFunction(MachineFunction &MF) override { + if (skipFunction(MF.getFunction())) + return false; + + return AMDGPUSetWavePriority().run(MF); + } +}; + } // End anonymous namespace. -INITIALIZE_PASS(AMDGPUSetWavePriority, DEBUG_TYPE, "Set wave priority", false, - false) +INITIALIZE_PASS(AMDGPUSetWavePriorityLegacy, DEBUG_TYPE, "Set wave priority", + false, false) -char AMDGPUSetWavePriority::ID = 0; +char AMDGPUSetWavePriorityLegacy::ID = 0; FunctionPass *llvm::createAMDGPUSetWavePriorityPass() { - return new AMDGPUSetWavePriority(); + return new AMDGPUSetWavePriorityLegacy(); } MachineInstr * @@ -96,12 +109,21 @@ static bool isVMEMLoad(const MachineInstr &MI) { return SIInstrInfo::isVMEM(MI) && MI.mayLoad(); } -bool AMDGPUSetWavePriority::runOnMachineFunction(MachineFunction &MF) { +PreservedAnalyses +llvm::AMDGPUSetWavePriorityPass::run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + if (!AMDGPUSetWavePriority().run(MF)) + return PreservedAnalyses::all(); + + return getMachineFunctionPassPreservedAnalyses(); +} + +bool AMDGPUSetWavePriority::run(MachineFunction &MF) { const unsigned HighPriority = 3; const unsigned LowPriority = 0; Function &F = MF.getFunction(); - if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv())) + if (!AMDGPU::isEntryFunctionCC(F.getCallingConv())) return false; const GCNSubtarget &ST = MF.getSubtarget(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index 4937b434bc955..4b5c70f09155f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -2168,9 +2168,8 @@ void AMDGPUCodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const { addPass(SILateBranchLoweringPass()); - if (isPassEnabled(EnableSetWavePriority, CodeGenOptLevel::Less)) { - // TODO: addPass(AMDGPUSetWavePriorityPass()); - } + if (isPassEnabled(EnableSetWavePriority, CodeGenOptLevel::Less)) + addPass(AMDGPUSetWavePriorityPass()); if (TM.getOptLevel() > CodeGenOptLevel::None) { // TODO: addPass(SIPreEmitPeepholePass()); diff --git a/llvm/test/CodeGen/AMDGPU/set-wave-priority.ll b/llvm/test/CodeGen/AMDGPU/set-wave-priority.ll index ab6877ac4e6d1..a27d1217031ca 100644 --- a/llvm/test/CodeGen/AMDGPU/set-wave-priority.ll +++ b/llvm/test/CodeGen/AMDGPU/set-wave-priority.ll @@ -1,6 +1,11 @@ ; RUN: llc -mtriple=amdgcn -amdgpu-set-wave-priority=true -o - %s | \ ; RUN: FileCheck %s +; RUN: llc -mtriple=amdgcn -stop-after=si-late-branch-lowering -o - %s | \ +; RUN: llc -x mir -mtriple=amdgcn -passes=amdgpu-set-wave-priority -o - | \ +; RUN: llc -x mir -mtriple=amdgcn -start-after=si-late-branch-lowering -o - | \ +; RUN: FileCheck %s + ; CHECK-LABEL: no_setprio: ; CHECK-NOT: s_setprio ; CHECK: ; return to shader part epilog