1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief Insert wait instructions for memory reads and writes.
13 /// Memory reads and writes are issued asynchronously, so we need to insert
14 /// S_WAITCNT instructions when we want to access any of their results or
15 /// overwrite any register that's used asynchronously.
17 //===----------------------------------------------------------------------===//
20 #include "AMDGPUSubtarget.h"
21 #include "SIDefines.h"
22 #include "SIInstrInfo.h"
23 #include "SIMachineFunctionInfo.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineFunctionPass.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
33 /// \brief One variable for each of the hardware counters
44 typedef Counters RegCounters[512];
45 typedef std::pair<unsigned, unsigned> RegInterval;
47 class SIInsertWaits : public MachineFunctionPass {
51 const SIInstrInfo *TII;
52 const SIRegisterInfo *TRI;
53 const MachineRegisterInfo *MRI;
55 /// \brief Constant hardware limits
56 static const Counters WaitCounts;
58 /// \brief Constant zero value
59 static const Counters ZeroCounts;
61 /// \brief Counter values we have already waited on.
64 /// \brief Counter values for last instruction issued.
67 /// \brief Registers used by async instructions.
70 /// \brief Registers defined by async instructions.
71 RegCounters DefinedRegs;
73 /// \brief Different export instruction types seen since last wait.
74 unsigned ExpInstrTypesSeen;
76 /// \brief Get increment/decrement amount for this instruction.
77 Counters getHwCounts(MachineInstr &MI);
79 /// \brief Is operand relevant for async execution?
80 bool isOpRelevant(MachineOperand &Op);
82 /// \brief Get register interval an operand affects.
83 RegInterval getRegInterval(MachineOperand &Op);
85 /// \brief Handle instructions async components
86 void pushInstruction(MachineInstr &MI);
88 /// \brief Insert the actual wait instruction
89 bool insertWait(MachineBasicBlock &MBB,
90 MachineBasicBlock::iterator I,
91 const Counters &Counts);
93 /// \brief Do we need def2def checks?
94 bool unorderedDefines(MachineInstr &MI);
96 /// \brief Resolve all operand dependencies to counter requirements
97 Counters handleOperands(MachineInstr &MI);
100 SIInsertWaits(TargetMachine &tm) :
101 MachineFunctionPass(ID),
104 ExpInstrTypesSeen(0) { }
106 bool runOnMachineFunction(MachineFunction &MF) override;
108 const char *getPassName() const override {
109 return "SI insert wait instructions";
114 } // End anonymous namespace
116 char SIInsertWaits::ID = 0;
118 const Counters SIInsertWaits::WaitCounts = { { 15, 7, 7 } };
119 const Counters SIInsertWaits::ZeroCounts = { { 0, 0, 0 } };
121 FunctionPass *llvm::createSIInsertWaits(TargetMachine &tm) {
122 return new SIInsertWaits(tm);
125 Counters SIInsertWaits::getHwCounts(MachineInstr &MI) {
127 uint64_t TSFlags = TII->get(MI.getOpcode()).TSFlags;
130 Result.Named.VM = !!(TSFlags & SIInstrFlags::VM_CNT);
132 // Only consider stores or EXP for EXP_CNT
133 Result.Named.EXP = !!(TSFlags & SIInstrFlags::EXP_CNT &&
134 (MI.getOpcode() == AMDGPU::EXP || MI.getDesc().mayStore()));
136 // LGKM may uses larger values
137 if (TSFlags & SIInstrFlags::LGKM_CNT) {
139 if (TII->isSMRD(MI.getOpcode())) {
141 MachineOperand &Op = MI.getOperand(0);
142 assert(Op.isReg() && "First LGKM operand must be a register!");
144 unsigned Reg = Op.getReg();
145 unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
146 Result.Named.LGKM = Size > 4 ? 2 : 1;
150 Result.Named.LGKM = 1;
154 Result.Named.LGKM = 0;
160 bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
162 // Constants are always irrelevant
166 // Defines are always relevant
170 // For exports all registers are relevant
171 MachineInstr &MI = *Op.getParent();
172 if (MI.getOpcode() == AMDGPU::EXP)
175 // For stores the stored value is also relevant
176 if (!MI.getDesc().mayStore())
179 for (MachineInstr::mop_iterator I = MI.operands_begin(),
180 E = MI.operands_end(); I != E; ++I) {
182 if (I->isReg() && I->isUse())
183 return Op.isIdenticalTo(*I);
189 RegInterval SIInsertWaits::getRegInterval(MachineOperand &Op) {
191 if (!Op.isReg() || !TRI->isInAllocatableClass(Op.getReg()))
192 return std::make_pair(0, 0);
194 unsigned Reg = Op.getReg();
195 unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize();
200 Result.first = TRI->getEncodingValue(Reg);
201 Result.second = Result.first + Size / 4;
206 void SIInsertWaits::pushInstruction(MachineInstr &MI) {
208 // Get the hardware counter increments and sum them up
209 Counters Increment = getHwCounts(MI);
212 for (unsigned i = 0; i < 3; ++i) {
213 LastIssued.Array[i] += Increment.Array[i];
214 Sum += Increment.Array[i];
217 // If we don't increase anything then that's it
221 // Remember which export instructions we have seen
222 if (Increment.Named.EXP) {
223 ExpInstrTypesSeen |= MI.getOpcode() == AMDGPU::EXP ? 1 : 2;
226 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
228 MachineOperand &Op = MI.getOperand(i);
229 if (!isOpRelevant(Op))
232 RegInterval Interval = getRegInterval(Op);
233 for (unsigned j = Interval.first; j < Interval.second; ++j) {
235 // Remember which registers we define
237 DefinedRegs[j] = LastIssued;
239 // and which one we are using
241 UsedRegs[j] = LastIssued;
246 bool SIInsertWaits::insertWait(MachineBasicBlock &MBB,
247 MachineBasicBlock::iterator I,
248 const Counters &Required) {
250 // End of program? No need to wait on anything
251 if (I != MBB.end() && I->getOpcode() == AMDGPU::S_ENDPGM)
254 // Figure out if the async instructions execute in order
257 // VM_CNT is always ordered
260 // EXP_CNT is unordered if we have both EXP & VM-writes
261 Ordered[1] = ExpInstrTypesSeen == 3;
263 // LGKM_CNT is handled as always unordered. TODO: Handle LDS and GDS
266 // The values we are going to put into the S_WAITCNT instruction
267 Counters Counts = WaitCounts;
269 // Do we really need to wait?
270 bool NeedWait = false;
272 for (unsigned i = 0; i < 3; ++i) {
274 if (Required.Array[i] <= WaitedOn.Array[i])
280 unsigned Value = LastIssued.Array[i] - Required.Array[i];
282 // Adjust the value to the real hardware possibilities.
283 Counts.Array[i] = std::min(Value, WaitCounts.Array[i]);
288 // Remember on what we have waited on.
289 WaitedOn.Array[i] = LastIssued.Array[i] - Counts.Array[i];
295 // Reset EXP_CNT instruction types
296 if (Counts.Named.EXP == 0)
297 ExpInstrTypesSeen = 0;
299 // Build the wait instruction
300 BuildMI(MBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT))
301 .addImm((Counts.Named.VM & 0xF) |
302 ((Counts.Named.EXP & 0x7) << 4) |
303 ((Counts.Named.LGKM & 0x7) << 8));
308 /// \brief helper function for handleOperands
309 static void increaseCounters(Counters &Dst, const Counters &Src) {
311 for (unsigned i = 0; i < 3; ++i)
312 Dst.Array[i] = std::max(Dst.Array[i], Src.Array[i]);
315 Counters SIInsertWaits::handleOperands(MachineInstr &MI) {
317 Counters Result = ZeroCounts;
319 // S_SENDMSG implicitly waits for all outstanding LGKM transfers to finish,
320 // but we also want to wait for any other outstanding transfers before
321 // signalling other hardware blocks
322 if (MI.getOpcode() == AMDGPU::S_SENDMSG)
325 // For each register affected by this
326 // instruction increase the result sequence
327 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
329 MachineOperand &Op = MI.getOperand(i);
330 RegInterval Interval = getRegInterval(Op);
331 for (unsigned j = Interval.first; j < Interval.second; ++j) {
334 increaseCounters(Result, UsedRegs[j]);
335 increaseCounters(Result, DefinedRegs[j]);
339 increaseCounters(Result, DefinedRegs[j]);
346 // FIXME: Insert waits listed in Table 4.2 "Required User-Inserted Wait States"
347 // around other non-memory instructions.
348 bool SIInsertWaits::runOnMachineFunction(MachineFunction &MF) {
349 bool Changes = false;
351 TII = static_cast<const SIInstrInfo *>(MF.getSubtarget().getInstrInfo());
353 static_cast<const SIRegisterInfo *>(MF.getSubtarget().getRegisterInfo());
355 MRI = &MF.getRegInfo();
357 WaitedOn = ZeroCounts;
358 LastIssued = ZeroCounts;
360 memset(&UsedRegs, 0, sizeof(UsedRegs));
361 memset(&DefinedRegs, 0, sizeof(DefinedRegs));
363 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
366 MachineBasicBlock &MBB = *BI;
367 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
370 Changes |= insertWait(MBB, I, handleOperands(*I));
374 // Wait for everything at the end of the MBB
375 Changes |= insertWait(MBB, MBB.getFirstTerminator(), LastIssued);