1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief Insert wait instructions for memory reads and writes.
13 /// Memory reads and writes are issued asynchronously, so we need to insert
14 /// S_WAITCNT instructions when we want to access any of their results or
15 /// overwrite any register that's used asynchronously.
17 //===----------------------------------------------------------------------===//
20 #include "SIInstrInfo.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineFunctionPass.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 /// \brief One variable for each of the hardware counters
42 typedef Counters RegCounters[512];
43 typedef std::pair<unsigned, unsigned> RegInterval;
45 class SIInsertWaits : public MachineFunctionPass {
49 const SIInstrInfo *TII;
50 const SIRegisterInfo &TRI;
51 const MachineRegisterInfo *MRI;
53 /// \brief Constant hardware limits
54 static const Counters WaitCounts;
56 /// \brief Constant zero value
57 static const Counters ZeroCounts;
59 /// \brief Counter values we have already waited on.
62 /// \brief Counter values for last instruction issued.
65 /// \brief Registers used by async instructions.
68 /// \brief Registers defined by async instructions.
69 RegCounters DefinedRegs;
71 /// \brief Different export instruction types seen since last wait.
72 unsigned ExpInstrTypesSeen;
74 /// \brief Get increment/decrement amount for this instruction.
75 Counters getHwCounts(MachineInstr &MI);
77 /// \brief Is operand relevant for async execution?
78 bool isOpRelevant(MachineOperand &Op);
80 /// \brief Get register interval an operand affects.
81 RegInterval getRegInterval(MachineOperand &Op);
83 /// \brief Handle instructions async components
84 void pushInstruction(MachineInstr &MI);
86 /// \brief Insert the actual wait instruction
87 bool insertWait(MachineBasicBlock &MBB,
88 MachineBasicBlock::iterator I,
89 const Counters &Counts);
91 /// \brief Do we need def2def checks?
92 bool unorderedDefines(MachineInstr &MI);
94 /// \brief Resolve all operand dependencies to counter requirements
95 Counters handleOperands(MachineInstr &MI);
98 SIInsertWaits(TargetMachine &tm) :
99 MachineFunctionPass(ID),
100 TII(static_cast<const SIInstrInfo*>(tm.getInstrInfo())),
101 TRI(TII->getRegisterInfo()) { }
103 virtual bool runOnMachineFunction(MachineFunction &MF);
105 const char *getPassName() const {
106 return "SI insert wait instructions";
111 } // End anonymous namespace
113 char SIInsertWaits::ID = 0;
115 const Counters SIInsertWaits::WaitCounts = { { 15, 7, 7 } };
116 const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };
118 FunctionPass *llvm::createSIInsertWaits(TargetMachine &tm) {
119 return new SIInsertWaits(tm);
122 Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
124 uint64_t TSFlags = TII->get(MI.getOpcode()).TSFlags;
127 Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT);
129 // Only consider stores or EXP for EXP_CNT
130 Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT &&
131 (MI.getOpcode() == AMDGPU::EXP || MI.getDesc().mayStore()));
133 // LGKM may uses larger values
134 if (TSFlags & SIInstrFlags::LGKM_CNT) {
136 MachineOperand &Op = MI.getOperand(0);
137 assert(Op.isReg() && "First LGKM operand must be a register!");
139 unsigned Reg = Op.getReg();
140 unsigned Size = TRI.getMinimalPhysRegClass(Reg)->getSize();
141 Result.Named.LGKM = Size > 4 ? 2 : 1;
144 Result.Named.LGKM = 0;
150 bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
152 // Constants are always irrelevant
156 // Defines are always relevant
160 // For exports all registers are relevant
161 MachineInstr &MI = *Op.getParent();
162 if (MI.getOpcode() == AMDGPU::EXP)
165 // For stores the stored value is also relevant
166 if (!MI.getDesc().mayStore())
169 for (MachineInstr::mop_iterator I = MI.operands_begin(),
170 E = MI.operands_end(); I != E; ++I) {
172 if (I->isReg() && I->isUse())
173 return Op.isIdenticalTo(*I);
179 RegInterval SIInsertWaits::getRegInterval(MachineOperand &Op) {
182 return std::make_pair(0, 0);
184 unsigned Reg = Op.getReg();
185 unsigned Size = TRI.getMinimalPhysRegClass(Reg)->getSize();
190 Result.first = TRI.getEncodingValue(Reg);
191 Result.second = Result.first + Size / 4;
196 void SIInsertWaits::pushInstruction(MachineInstr &MI) {
198 // Get the hardware counter increments and sum them up
199 Counters Increment = getHwCounts(MI);
202 for (unsigned i = 0; i < 3; ++i) {
203 LastIssued.Array[i] += Increment.Array[i];
204 Sum += Increment.Array[i];
207 // If we don't increase anything then that's it
211 // Remember which export instructions we have seen
212 if (Increment.Named.EXP) {
213 ExpInstrTypesSeen |= MI.getOpcode() == AMDGPU::EXP ? 1 : 2;
216 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
218 MachineOperand &Op = MI.getOperand(i);
219 if (!isOpRelevant(Op))
222 RegInterval Interval = getRegInterval(Op);
223 for (unsigned j = Interval.first; j < Interval.second; ++j) {
225 // Remember which registers we define
227 DefinedRegs[j] = LastIssued;
229 // and which one we are using
231 UsedRegs[j] = LastIssued;
236 bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
237 MachineBasicBlock::iterator I,
238 const Counters &Required) {
240 // End of program? No need to wait on anything
241 if (I != MBB.end() && I->getOpcode() == AMDGPU::S_ENDPGM)
244 // Figure out if the async instructions execute in order
247 // VM_CNT is always ordered
250 // EXP_CNT is unordered if we have both EXP & VM-writes
251 Ordered[1] = ExpInstrTypesSeen == 3;
253 // LGKM_CNT is handled as always unordered. TODO: Handle LDS and GDS
256 // The values we are going to put into the S_WAITCNT instruction
257 Counters Counts = WaitCounts;
259 // Do we really need to wait?
260 bool NeedWait = false;
262 for (unsigned i = 0; i < 3; ++i) {
264 if (Required.Array[i] <= WaitedOn.Array[i])
270 unsigned Value = LastIssued.Array[i] - Required.Array[i];
272 // adjust the value to the real hardware posibilities
273 Counts.Array[i] = std::min(Value, WaitCounts.Array[i]);
278 // Remember on what we have waited on
279 WaitedOn.Array[i] = LastIssued.Array[i] - Counts.Array[i];
285 // Reset EXP_CNT instruction types
286 if (Counts.Named.EXP == 0)
287 ExpInstrTypesSeen = 0;
289 // Build the wait instruction
290 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
291 .addImm((Counts.Named.VM & 0xF) |
292 ((Counts.Named.EXP & 0x7) << 4) |
293 ((Counts.Named.LGKM & 0x7) << 8));
298 /// \brief helper function for handleOperands
299 static void increaseCounters(Counters &Dst, const Counters &Src) {
301 for (unsigned i = 0; i < 3; ++i)
302 Dst.Array[i] = std::max(Dst.Array[i], Src.Array[i]);
305 Counters SIInsertWaits::handleOperands(MachineInstr &MI) {
307 Counters Result = ZeroCounts;
309 // For each register affected by this
310 // instruction increase the result sequence
311 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
313 MachineOperand &Op = MI.getOperand(i);
314 RegInterval Interval = getRegInterval(Op);
315 for (unsigned j = Interval.first; j < Interval.second; ++j) {
318 increaseCounters(Result, UsedRegs[j]);
319 increaseCounters(Result, DefinedRegs[j]);
323 increaseCounters(Result, DefinedRegs[j]);
330 bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
332 bool Changes = false;
334 MRI = &MF.getRegInfo();
336 WaitedOn = ZeroCounts;
337 LastIssued = ZeroCounts;
339 memset(&UsedRegs, 0, sizeof(UsedRegs));
340 memset(&DefinedRegs, 0, sizeof(DefinedRegs));
342 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
345 MachineBasicBlock &MBB = *BI;
346 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
349 Changes |= insertWait(MBB, I, handleOperands(*I));
353 // Wait for everything at the end of the MBB
354 Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);