1 //===-- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter -----------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the pass which inserts x86 AVX vzeroupper instructions
11 // before calls to SSE encoded functions. This avoids transition latency
12 // penalty when tranfering control between AVX encoded instructions and old
15 //===----------------------------------------------------------------------===//
18 #include "X86InstrInfo.h"
19 #include "X86Subtarget.h"
20 #include "llvm/ADT/Statistic.h"
21 #include "llvm/CodeGen/MachineFunctionPass.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/CodeGen/Passes.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/raw_ostream.h"
27 #include "llvm/Target/TargetInstrInfo.h"
30 #define DEBUG_TYPE "x86-vzeroupper"
32 STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
36 class VZeroUpperInserter : public MachineFunctionPass {
39 VZeroUpperInserter() : MachineFunctionPass(ID) {}
40 bool runOnMachineFunction(MachineFunction &MF) override;
41 const char *getPassName() const override {return "X86 vzeroupper inserter";}
45 void processBasicBlock(MachineBasicBlock &MBB);
46 void insertVZeroUpper(MachineBasicBlock::iterator I,
47 MachineBasicBlock &MBB);
48 void addDirtySuccessor(MachineBasicBlock &MBB);
50 typedef enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY } BlockExitState;
51 static const char* getBlockExitStateName(BlockExitState ST);
53 // Core algorithm state:
54 // BlockState - Each block is either:
55 // - PASS_THROUGH: There are neither YMM dirtying instructions nor
56 // vzeroupper instructions in this block.
57 // - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
58 // block that will ensure that YMM is clean on exit.
59 // - EXITS_DIRTY: An instruction in the block dirties YMM and no
60 // subsequent vzeroupper in the block clears it.
62 // AddedToDirtySuccessors - This flag is raised when a block is added to the
63 // DirtySuccessors list to ensure that it's not
64 // added multiple times.
66 // FirstUnguardedCall - Records the location of the first unguarded call in
67 // each basic block that may need to be guarded by a
68 // vzeroupper. We won't know whether it actually needs
69 // to be guarded until we discover a predecessor that
72 BlockState() : ExitState(PASS_THROUGH), AddedToDirtySuccessors(false) {}
73 BlockExitState ExitState;
74 bool AddedToDirtySuccessors;
75 MachineBasicBlock::iterator FirstUnguardedCall;
77 typedef SmallVector<BlockState, 8> BlockStateMap;
78 typedef SmallVector<MachineBasicBlock*, 8> DirtySuccessorsWorkList;
80 BlockStateMap BlockStates;
81 DirtySuccessorsWorkList DirtySuccessors;
83 const TargetInstrInfo *TII;
88 char VZeroUpperInserter::ID = 0;
91 FunctionPass *llvm::createX86IssueVZeroUpperPass() {
92 return new VZeroUpperInserter();
95 const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
97 case PASS_THROUGH: return "Pass-through";
98 case EXITS_DIRTY: return "Exits-dirty";
99 case EXITS_CLEAN: return "Exits-clean";
101 llvm_unreachable("Invalid block exit state.");
104 static bool isYmmReg(unsigned Reg) {
105 return (Reg >= X86::YMM0 && Reg <= X86::YMM15);
108 static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) {
109 for (MachineRegisterInfo::livein_iterator I = MRI.livein_begin(),
110 E = MRI.livein_end(); I != E; ++I)
111 if (isYmmReg(I->first))
117 static bool clobbersAllYmmRegs(const MachineOperand &MO) {
118 for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
119 if (!MO.clobbersPhysReg(reg))
125 static bool hasYmmReg(MachineInstr *MI) {
126 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
127 const MachineOperand &MO = MI->getOperand(i);
128 if (MI->isCall() && MO.isRegMask() && !clobbersAllYmmRegs(MO))
134 if (isYmmReg(MO.getReg()))
140 /// clobbersAnyYmmReg() - Check if any YMM register will be clobbered by this
142 static bool callClobbersAnyYmmReg(MachineInstr *MI) {
143 assert(MI->isCall() && "Can only be called on call instructions.");
144 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
145 const MachineOperand &MO = MI->getOperand(i);
148 for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
149 if (MO.clobbersPhysReg(reg))
156 // Insert a vzeroupper instruction before I.
157 void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
158 MachineBasicBlock &MBB) {
159 DebugLoc dl = I->getDebugLoc();
160 BuildMI(MBB, I, dl, TII->get(X86::VZEROUPPER));
162 EverMadeChange = true;
165 // Add MBB to the DirtySuccessors list if it hasn't already been added.
166 void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
167 if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
168 DirtySuccessors.push_back(&MBB);
169 BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
173 /// processBasicBlock - Loop over all of the instructions in the basic block,
174 /// inserting vzero upper instructions before function calls.
175 void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
177 // Start by assuming that the block PASS_THROUGH, which implies no unguarded
179 BlockExitState CurState = PASS_THROUGH;
180 BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
182 for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
183 MachineInstr *MI = I;
184 bool isControlFlow = MI->isCall() || MI->isReturn();
186 // Shortcut: don't need to check regular instructions in dirty state.
187 if (!isControlFlow && CurState == EXITS_DIRTY)
191 // We found a ymm-using instruction; this could be an AVX instruction,
192 // or it could be control flow.
193 CurState = EXITS_DIRTY;
197 // Check for control-flow out of the current function (which might
198 // indirectly execute SSE instructions).
202 // If the call won't clobber any YMM register, skip it as well. It usually
203 // happens on helper function calls (such as '_chkstk', '_ftol2') where
204 // standard calling convention is not used (RegMask is not used to mark
205 // register clobbered and register usage (def/imp-def/use) is well-dfined
206 // and explicitly specified.
207 if (MI->isCall() && !callClobbersAnyYmmReg(MI))
210 // The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX
211 // registers. This instruction has zero latency. In addition, the processor
212 // changes back to Clean state, after which execution of Intel SSE
213 // instructions or Intel AVX instructions has no transition penalty. Add
214 // the VZEROUPPER instruction before any function call/return that might
216 // FIXME: In some cases, we may want to move the VZEROUPPER into a
217 // predecessor block.
218 if (CurState == EXITS_DIRTY) {
219 // After the inserted VZEROUPPER the state becomes clean again, but
220 // other YMM may appear before other subsequent calls or even before
221 // the end of the BB.
222 insertVZeroUpper(I, MBB);
223 CurState = EXITS_CLEAN;
224 } else if (CurState == PASS_THROUGH) {
225 // If this block is currently in pass-through state and we encounter a
226 // call then whether we need a vzeroupper or not depends on whether this
227 // block has successors that exit dirty. Record the location of the call,
228 // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
229 // It will be inserted later if necessary.
230 BlockStates[MBB.getNumber()].FirstUnguardedCall = I;
231 CurState = EXITS_CLEAN;
235 DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
236 << getBlockExitStateName(CurState) << '\n');
238 if (CurState == EXITS_DIRTY)
239 for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
242 addDirtySuccessor(**SI);
244 BlockStates[MBB.getNumber()].ExitState = CurState;
247 /// runOnMachineFunction - Loop over all of the basic blocks, inserting
248 /// vzero upper instructions before function calls.
249 bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
250 const X86Subtarget &ST = MF.getTarget().getSubtarget<X86Subtarget>();
251 if (!ST.hasAVX() || ST.hasAVX512())
253 TII = MF.getTarget().getInstrInfo();
254 MachineRegisterInfo &MRI = MF.getRegInfo();
255 EverMadeChange = false;
257 // Fast check: if the function doesn't use any ymm registers, we don't need
258 // to insert any VZEROUPPER instructions. This is constant-time, so it is
259 // cheap in the common case of no ymm use.
260 bool YMMUsed = false;
261 const TargetRegisterClass *RC = &X86::VR256RegClass;
262 for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end();
264 if (!MRI.reg_nodbg_empty(*i)) {
273 assert(BlockStates.empty() && DirtySuccessors.empty() &&
274 "X86VZeroUpper state should be clear");
275 BlockStates.resize(MF.getNumBlockIDs());
277 // Process all blocks. This will compute block exit states, record the first
278 // unguarded call in each block, and add successors of dirty blocks to the
279 // DirtySuccessors list.
280 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
281 processBasicBlock(*I);
283 // If any YMM regs are live in to this function, add the entry block to the
284 // DirtySuccessors list
285 if (checkFnHasLiveInYmm(MRI))
286 addDirtySuccessor(MF.front());
288 // Re-visit all blocks that are successors of EXITS_DIRTY bsocks. Add
289 // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
290 // through PASS_THROUGH blocks.
291 while (!DirtySuccessors.empty()) {
292 MachineBasicBlock &MBB = *DirtySuccessors.back();
293 DirtySuccessors.pop_back();
294 BlockState &BBState = BlockStates[MBB.getNumber()];
296 // MBB is a successor of a dirty block, so its first call needs to be
298 if (BBState.FirstUnguardedCall != MBB.end())
299 insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
301 // If this successor was a pass-through block then it is now dirty, and its
302 // successors need to be added to the worklist (if they haven't been
304 if (BBState.ExitState == PASS_THROUGH) {
305 DEBUG(dbgs() << "MBB #" << MBB.getNumber()
306 << " was Pass-through, is now Dirty-out.\n");
307 for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
310 addDirtySuccessor(**SI);
315 return EverMadeChange;