1 //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
9 // For best-case performance on Cortex-A57, we should try to use a balanced
10 // mix of odd and even D-registers when performing a critical sequence of
11 // independent, non-quadword FP/ASIMD floating-point multiply or
12 // multiply-accumulate operations.
14 // This pass attempts to detect situations where the register allocation may
15 // adversely affect this load balancing and to change the registers used so as
16 // to better utilize the CPU.
18 // Ideally we'd just take each multiply or multiply-accumulate in turn and
19 // allocate it alternating even or odd registers. However, multiply-accumulates
20 // are most efficiently performed in the same functional unit as their
21 // accumulation operand. Therefore this pass tries to find maximal sequences
22 // ("Chains") of multiply-accumulates linked via their accumulation operand,
23 // and assign them all the same "color" (oddness/evenness).
25 // This optimization affects S-register and D-register floating point
26 // multiplies and FMADD/FMAs, as well as vector (floating point only) muls and
27 // FMADD/FMA. Q register instructions (and 128-bit vector instructions) are
29 //===----------------------------------------------------------------------===//
32 #include "AArch64InstrInfo.h"
33 #include "AArch64Subtarget.h"
34 #include "llvm/ADT/BitVector.h"
35 #include "llvm/ADT/EquivalenceClasses.h"
36 #include "llvm/CodeGen/MachineFunction.h"
37 #include "llvm/CodeGen/MachineFunctionPass.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineRegisterInfo.h"
41 #include "llvm/CodeGen/RegisterClassInfo.h"
42 #include "llvm/CodeGen/RegisterScavenging.h"
43 #include "llvm/Support/CommandLine.h"
44 #include "llvm/Support/Debug.h"
45 #include "llvm/Support/raw_ostream.h"
49 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing"
51 // Enforce the algorithm to use the scavenged register even when the original
52 // destination register is the correct color. Used for testing.
54 TransformAll("aarch64-a57-fp-load-balancing-force-all",
55 cl::desc("Always modify dest registers regardless of color"),
56 cl::init(false), cl::Hidden);
58 // Never use the balance information obtained from chains - return a specific
59 // color always. Used for testing.
60 static cl::opt<unsigned>
61 OverrideBalance("aarch64-a57-fp-load-balancing-override",
62 cl::desc("Ignore balance information, always return "
63 "(1: Even, 2: Odd)."),
64 cl::init(0), cl::Hidden);
66 //===----------------------------------------------------------------------===//
69 // Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs?
70 static bool isMul(MachineInstr *MI) {
71 switch (MI->getOpcode()) {
72 case AArch64::FMULSrr:
73 case AArch64::FNMULSrr:
74 case AArch64::FMULDrr:
75 case AArch64::FNMULDrr:
82 // Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs?
83 static bool isMla(MachineInstr *MI) {
84 switch (MI->getOpcode()) {
85 case AArch64::FMSUBSrrr:
86 case AArch64::FMADDSrrr:
87 case AArch64::FNMSUBSrrr:
88 case AArch64::FNMADDSrrr:
89 case AArch64::FMSUBDrrr:
90 case AArch64::FMADDDrrr:
91 case AArch64::FNMSUBDrrr:
92 case AArch64::FNMADDDrrr:
100 static void initializeAArch64A57FPLoadBalancingPass(PassRegistry &);
103 //===----------------------------------------------------------------------===//
106 /// A "color", which is either even or odd. Yes, these aren't really colors
107 /// but the algorithm is conceptually doing two-color graph coloring.
108 enum class Color { Even, Odd };
110 static const char *ColorNames[2] = { "Even", "Odd" };
115 class AArch64A57FPLoadBalancing : public MachineFunctionPass {
116 MachineRegisterInfo *MRI;
117 const TargetRegisterInfo *TRI;
118 RegisterClassInfo RCI;
122 explicit AArch64A57FPLoadBalancing() : MachineFunctionPass(ID) {
123 initializeAArch64A57FPLoadBalancingPass(*PassRegistry::getPassRegistry());
126 bool runOnMachineFunction(MachineFunction &F) override;
128 const char *getPassName() const override {
129 return "A57 FP Anti-dependency breaker";
132 void getAnalysisUsage(AnalysisUsage &AU) const override {
133 AU.setPreservesCFG();
134 MachineFunctionPass::getAnalysisUsage(AU);
138 bool runOnBasicBlock(MachineBasicBlock &MBB);
139 bool colorChainSet(std::vector<Chain*> GV, MachineBasicBlock &MBB,
141 bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB);
142 int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB);
143 void scanInstruction(MachineInstr *MI, unsigned Idx,
144 std::map<unsigned, Chain*> &Active,
145 std::set<std::unique_ptr<Chain>> &AllChains);
146 void maybeKillChain(MachineOperand &MO, unsigned Idx,
147 std::map<unsigned, Chain*> &RegChains);
148 Color getColor(unsigned Register);
149 Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L);
153 char AArch64A57FPLoadBalancing::ID = 0;
155 INITIALIZE_PASS_BEGIN(AArch64A57FPLoadBalancing, DEBUG_TYPE,
156 "AArch64 A57 FP Load-Balancing", false, false)
157 INITIALIZE_PASS_END(AArch64A57FPLoadBalancing, DEBUG_TYPE,
158 "AArch64 A57 FP Load-Balancing", false, false)
161 /// A Chain is a sequence of instructions that are linked together by
162 /// an accumulation operand. For example:
165 /// fmla d1<def>, ?, ?, d0<kill>
166 /// fmla d2<def>, ?, ?, d1<kill>
168 /// There may be other instructions interleaved in the sequence that
169 /// do not belong to the chain. These other instructions must not use
170 /// the "chain" register at any point.
172 /// We currently only support chains where the "chain" operand is killed
173 /// at each link in the chain for simplicity.
174 /// A chain has three important instructions - Start, Last and Kill.
175 /// * The start instruction is the first instruction in the chain.
176 /// * Last is the final instruction in the chain.
177 /// * Kill may or may not be defined. If defined, Kill is the instruction
178 /// where the outgoing value of the Last instruction is killed.
179 /// This information is important as if we know the outgoing value is
180 /// killed with no intervening uses, we can safely change its register.
182 /// Without a kill instruction, we must assume the outgoing value escapes
183 /// beyond our model and either must not change its register or must
184 /// create a fixup FMOV to keep the old register value consistent.
188 /// The important (marker) instructions.
189 MachineInstr *StartInst, *LastInst, *KillInst;
190 /// The index, from the start of the basic block, that each marker
191 /// appears. These are stored so we can do quick interval tests.
192 unsigned StartInstIdx, LastInstIdx, KillInstIdx;
193 /// All instructions in the chain.
194 std::set<MachineInstr*> Insts;
195 /// True if KillInst cannot be modified. If this is true,
196 /// we cannot change LastInst's outgoing register.
197 /// This will be true for tied values and regmasks.
198 bool KillIsImmutable;
199 /// The "color" of LastInst. This will be the preferred chain color,
200 /// as changing intermediate nodes is easy but changing the last
201 /// instruction can be more tricky.
204 Chain(MachineInstr *MI, unsigned Idx, Color C)
205 : StartInst(MI), LastInst(MI), KillInst(nullptr),
206 StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0),
211 /// Add a new instruction into the chain. The instruction's dest operand
212 /// has the given color.
213 void add(MachineInstr *MI, unsigned Idx, Color C) {
217 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
218 "Chain: broken invariant. A Chain can only be killed after its last "
224 /// Return true if MI is a member of the chain.
225 bool contains(MachineInstr *MI) { return Insts.count(MI) > 0; }
227 /// Return the number of instructions in the chain.
228 unsigned size() const {
232 /// Inform the chain that its last active register (the dest register of
233 /// LastInst) is killed by MI with no intervening uses or defs.
234 void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) {
237 KillIsImmutable = Immutable;
238 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
239 "Chain: broken invariant. A Chain can only be killed after its last "
243 /// Return the first instruction in the chain.
244 MachineInstr *getStart() const { return StartInst; }
245 /// Return the last instruction in the chain.
246 MachineInstr *getLast() const { return LastInst; }
247 /// Return the "kill" instruction (as set with setKill()) or NULL.
248 MachineInstr *getKill() const { return KillInst; }
249 /// Return an instruction that can be used as an iterator for the end
250 /// of the chain. This is the maximum of KillInst (if set) and LastInst.
251 MachineBasicBlock::iterator getEnd() const {
252 return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst);
255 /// Can the Kill instruction (assuming one exists) be modified?
256 bool isKillImmutable() const { return KillIsImmutable; }
258 /// Return the preferred color of this chain.
259 Color getPreferredColor() {
260 if (OverrideBalance != 0)
261 return OverrideBalance == 1 ? Color::Even : Color::Odd;
265 /// Return true if this chain (StartInst..KillInst) overlaps with Other.
266 bool rangeOverlapsWith(const Chain &Other) const {
267 unsigned End = KillInst ? KillInstIdx : LastInstIdx;
268 unsigned OtherEnd = Other.KillInst ?
269 Other.KillInstIdx : Other.LastInstIdx;
271 return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End;
274 /// Return true if this chain starts before Other.
275 bool startsBefore(const Chain *Other) const {
276 return StartInstIdx < Other->StartInstIdx;
279 /// Return true if the group will require a fixup MOV at the end.
280 bool requiresFixup() const {
281 return (getKill() && isKillImmutable()) || !getKill();
284 /// Return a simple string representation of the chain.
285 std::string str() const {
287 raw_string_ostream OS(S);
290 StartInst->print(OS, NULL, true);
292 LastInst->print(OS, NULL, true);
295 KillInst->print(OS, NULL, true);
305 } // end anonymous namespace
307 //===----------------------------------------------------------------------===//
309 bool AArch64A57FPLoadBalancing::runOnMachineFunction(MachineFunction &F) {
310 bool Changed = false;
311 DEBUG(dbgs() << "***** AArch64A57FPLoadBalancing *****\n");
313 MRI = &F.getRegInfo();
314 TRI = F.getRegInfo().getTargetRegisterInfo();
315 RCI.runOnMachineFunction(F);
317 for (auto &MBB : F) {
318 Changed |= runOnBasicBlock(MBB);
324 bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) {
325 bool Changed = false;
326 DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
328 // First, scan the basic block producing a set of chains.
330 // The currently "active" chains - chains that can be added to and haven't
331 // been killed yet. This is keyed by register - all chains can only have one
332 // "link" register between each inst in the chain.
333 std::map<unsigned, Chain*> ActiveChains;
334 std::set<std::unique_ptr<Chain>> AllChains;
337 scanInstruction(&MI, Idx++, ActiveChains, AllChains);
339 DEBUG(dbgs() << "Scan complete, "<< AllChains.size() << " chains created.\n");
341 // Group the chains into disjoint sets based on their liveness range. This is
342 // a poor-man's version of graph coloring. Ideally we'd create an interference
343 // graph and perform full-on graph coloring on that, but;
344 // (a) That's rather heavyweight for only two colors.
345 // (b) We expect multiple disjoint interference regions - in practice the live
346 // range of chains is quite small and they are clustered between loads
348 EquivalenceClasses<Chain*> EC;
349 for (auto &I : AllChains)
352 for (auto &I : AllChains)
353 for (auto &J : AllChains)
354 if (I != J && I->rangeOverlapsWith(*J))
355 EC.unionSets(I.get(), J.get());
356 DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n");
358 // Now we assume that every member of an equivalence class interferes
359 // with every other member of that class, and with no members of other classes.
361 // Convert the EquivalenceClasses to a simpler set of sets.
362 std::vector<std::vector<Chain*> > V;
363 for (auto I = EC.begin(), E = EC.end(); I != E; ++I) {
364 std::vector<Chain*> Cs(EC.member_begin(I), EC.member_end());
365 if (Cs.empty()) continue;
366 V.push_back(std::move(Cs));
369 // Now we have a set of sets, order them by start address so
370 // we can iterate over them sequentially.
371 std::sort(V.begin(), V.end(),
372 [](const std::vector<Chain*> &A,
373 const std::vector<Chain*> &B) {
374 return A.front()->startsBefore(B.front());
377 // As we only have two colors, we can track the global (BB-level) balance of
378 // odds versus evens. We aim to keep this near zero to keep both execution
380 // Positive means we're even-heavy, negative we're odd-heavy.
382 // FIXME: If chains have interdependencies, for example:
385 // We do not model this and may color each one differently, assuming we'll
386 // get ILP when we obviously can't. This hasn't been seen to be a problem
387 // in practice so far, so we simplify the algorithm by ignoring it.
391 Changed |= colorChainSet(std::move(I), MBB, Parity);
396 Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor,
397 std::vector<Chain*> &L) {
401 // We try and get the best candidate from L to color next, given that our
402 // preferred color is "PreferredColor". L is ordered from larger to smaller
403 // chains. It is beneficial to color the large chains before the small chains,
404 // but if we can't find a chain of the maximum length with the preferred color,
405 // we fuzz the size and look for slightly smaller chains before giving up and
406 // returning a chain that must be recolored.
408 // FIXME: Does this need to be configurable?
409 const unsigned SizeFuzz = 1;
410 unsigned MinSize = L.front()->size() - SizeFuzz;
411 for (auto I = L.begin(), E = L.end(); I != E; ++I) {
412 if ((*I)->size() <= MinSize) {
413 // We've gone past the size limit. Return the previous item.
419 if ((*I)->getPreferredColor() == PreferredColor) {
426 // Bailout case - just return the first item.
427 Chain *Ch = L.front();
432 bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV,
433 MachineBasicBlock &MBB,
435 bool Changed = false;
436 DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n");
438 // Sort by descending size order so that we allocate the most important
440 // Tie-break equivalent sizes by sorting chains requiring fixups before
441 // those without fixups. The logic here is that we should look at the
442 // chains that we cannot change before we look at those we can,
443 // so the parity counter is updated and we know what color we should
445 // Final tie-break with instruction order so pass output is stable (i.e. not
446 // dependent on malloc'd pointer values).
447 std::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) {
448 if (G1->size() != G2->size())
449 return G1->size() > G2->size();
450 if (G1->requiresFixup() != G2->requiresFixup())
451 return G1->requiresFixup() > G2->requiresFixup();
452 // Make sure startsBefore() produces a stable final order.
453 assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) &&
454 "Starts before not total order!");
455 return G1->startsBefore(G2);
458 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
459 while (Chain *G = getAndEraseNext(PreferredColor, GV)) {
460 // Start off by assuming we'll color to our own preferred color.
461 Color C = PreferredColor;
463 // But if we really don't care, use the chain's preferred color.
464 C = G->getPreferredColor();
466 DEBUG(dbgs() << " - Parity=" << Parity << ", Color="
467 << ColorNames[(int)C] << "\n");
469 // If we'll need a fixup FMOV, don't bother. Testing has shown that this
470 // happens infrequently and when it does it has at least a 50% chance of
471 // slowing code down instead of speeding it up.
472 if (G->requiresFixup() && C != G->getPreferredColor()) {
473 C = G->getPreferredColor();
474 DEBUG(dbgs() << " - " << G->str() << " - not worthwhile changing; "
475 "color remains " << ColorNames[(int)C] << "\n");
478 Changed |= colorChain(G, C, MBB);
480 Parity += (C == Color::Even) ? G->size() : -G->size();
481 PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
487 int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C,
488 MachineBasicBlock &MBB) {
490 RS.enterBasicBlock(&MBB);
491 RS.forward(MachineBasicBlock::iterator(G->getStart()));
493 // Can we find an appropriate register that is available throughout the life
495 unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass;
496 BitVector AvailableRegs = RS.getRegsAvailable(TRI->getRegClass(RegClassID));
497 for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd();
500 AvailableRegs &= RS.getRegsAvailable(TRI->getRegClass(RegClassID));
502 // Remove any registers clobbered by a regmask or any def register that is
504 for (auto J : I->operands()) {
506 AvailableRegs.clearBitsNotInMask(J.getRegMask());
508 if (J.isReg() && J.isDef() && AvailableRegs[J.getReg()]) {
509 assert(J.isDead() && "Non-dead def should have been removed by now!");
510 AvailableRegs.reset(J.getReg());
515 // Make sure we allocate in-order, to get the cheapest registers first.
516 auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID));
517 for (auto Reg : Ord) {
518 if (!AvailableRegs[Reg])
520 if ((C == Color::Even && (Reg % 2) == 0) ||
521 (C == Color::Odd && (Reg % 2) == 1))
528 bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C,
529 MachineBasicBlock &MBB) {
530 bool Changed = false;
531 DEBUG(dbgs() << " - colorChain(" << G->str() << ", "
532 << ColorNames[(int)C] << ")\n");
534 // Try and obtain a free register of the right class. Without a register
535 // to play with we cannot continue.
536 int Reg = scavengeRegister(G, C, MBB);
538 DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n");
541 DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n");
543 std::map<unsigned, unsigned> Substs;
544 for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd();
546 if (!G->contains(I) &&
547 (&*I != G->getKill() || G->isKillImmutable()))
550 // I is a member of G, or I is a mutable instruction that kills G.
552 std::vector<unsigned> ToErase;
553 for (auto &U : I->operands()) {
554 if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) {
555 unsigned OrigReg = U.getReg();
556 U.setReg(Substs[OrigReg]);
558 // Don't erase straight away, because there may be other operands
559 // that also reference this substitution!
560 ToErase.push_back(OrigReg);
561 } else if (U.isRegMask()) {
562 for (auto J : Substs) {
563 if (U.clobbersPhysReg(J.first))
564 ToErase.push_back(J.first);
568 // Now it's safe to remove the substs identified earlier.
569 for (auto J : ToErase)
572 // Only change the def if this isn't the last instruction.
573 if (&*I != G->getKill()) {
574 MachineOperand &MO = I->getOperand(0);
576 bool Change = TransformAll || getColor(MO.getReg()) != C;
577 if (G->requiresFixup() && &*I == G->getLast())
581 Substs[MO.getReg()] = Reg;
583 MRI->setPhysRegUsed(Reg);
589 assert(Substs.size() == 0 && "No substitutions should be left active!");
592 DEBUG(dbgs() << " - Kill instruction seen.\n");
594 // We didn't have a kill instruction, but we didn't seem to need to change
595 // the destination register anyway.
596 DEBUG(dbgs() << " - Destination register not changed.\n");
601 void AArch64A57FPLoadBalancing::
602 scanInstruction(MachineInstr *MI, unsigned Idx,
603 std::map<unsigned, Chain*> &ActiveChains,
604 std::set<std::unique_ptr<Chain>> &AllChains) {
605 // Inspect "MI", updating ActiveChains and AllChains.
609 for (auto &I : MI->uses())
610 maybeKillChain(I, Idx, ActiveChains);
611 for (auto &I : MI->defs())
612 maybeKillChain(I, Idx, ActiveChains);
614 // Create a new chain. Multiplies don't require forwarding so can go on any
616 unsigned DestReg = MI->getOperand(0).getReg();
618 DEBUG(dbgs() << "New chain started for register "
619 << TRI->getName(DestReg) << " at " << *MI);
621 auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg));
622 ActiveChains[DestReg] = G.get();
623 AllChains.insert(std::move(G));
625 } else if (isMla(MI)) {
627 // It is beneficial to keep MLAs on the same functional unit as their
628 // accumulator operand.
629 unsigned DestReg = MI->getOperand(0).getReg();
630 unsigned AccumReg = MI->getOperand(3).getReg();
632 maybeKillChain(MI->getOperand(1), Idx, ActiveChains);
633 maybeKillChain(MI->getOperand(2), Idx, ActiveChains);
634 if (DestReg != AccumReg)
635 maybeKillChain(MI->getOperand(0), Idx, ActiveChains);
637 if (ActiveChains.find(AccumReg) != ActiveChains.end()) {
638 DEBUG(dbgs() << "Chain found for accumulator register "
639 << TRI->getName(AccumReg) << " in MI " << *MI);
641 // For simplicity we only chain together sequences of MULs/MLAs where the
642 // accumulator register is killed on each instruction. This means we don't
643 // need to track other uses of the registers we want to rewrite.
645 // FIXME: We could extend to handle the non-kill cases for more coverage.
646 if (MI->getOperand(3).isKill()) {
648 DEBUG(dbgs() << "Instruction was successfully added to chain.\n");
649 ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg));
650 // Handle cases where the destination is not the same as the accumulator.
651 if (DestReg != AccumReg) {
652 ActiveChains[DestReg] = ActiveChains[AccumReg];
653 ActiveChains.erase(AccumReg);
658 DEBUG(dbgs() << "Cannot add to chain because accumulator operand wasn't "
659 << "marked <kill>!\n");
660 maybeKillChain(MI->getOperand(3), Idx, ActiveChains);
663 DEBUG(dbgs() << "Creating new chain for dest register "
664 << TRI->getName(DestReg) << "\n");
665 auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg));
666 ActiveChains[DestReg] = G.get();
667 AllChains.insert(std::move(G));
671 // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs
673 for (auto &I : MI->uses())
674 maybeKillChain(I, Idx, ActiveChains);
675 for (auto &I : MI->defs())
676 maybeKillChain(I, Idx, ActiveChains);
681 void AArch64A57FPLoadBalancing::
682 maybeKillChain(MachineOperand &MO, unsigned Idx,
683 std::map<unsigned, Chain*> &ActiveChains) {
684 // Given an operand and the set of active chains (keyed by register),
685 // determine if a chain should be ended and remove from ActiveChains.
686 MachineInstr *MI = MO.getParent();
690 // If this is a KILL of a current chain, record it.
691 if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) {
692 DEBUG(dbgs() << "Kill seen for chain " << TRI->getName(MO.getReg())
694 ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied());
696 ActiveChains.erase(MO.getReg());
698 } else if (MO.isRegMask()) {
700 for (auto I = ActiveChains.begin(), E = ActiveChains.end();
702 if (MO.clobbersPhysReg(I->first)) {
703 DEBUG(dbgs() << "Kill (regmask) seen for chain "
704 << TRI->getName(I->first) << "\n");
705 I->second->setKill(MI, Idx, /*Immutable=*/true);
706 ActiveChains.erase(I++);
714 Color AArch64A57FPLoadBalancing::getColor(unsigned Reg) {
715 if ((TRI->getEncodingValue(Reg) % 2) == 0)
721 // Factory function used by AArch64TargetMachine to add the pass to the passmanager.
722 FunctionPass *llvm::createAArch64A57FPLoadBalancing() {
723 return new AArch64A57FPLoadBalancing();