X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FTarget%2FR600%2FR600EmitClauseMarkers.cpp;h=fdc20302f4a327a2634b865f8673d4ea42a42a81;hb=12af22e8cc217827cf4f118b0f5e4ebbda9925ae;hp=7c7469a04b2386f9f9bead41dc65f86242a09afe;hpb=8e59191eb8033133f5b2923d2056d4362af913ce;p=oota-llvm.git diff --git a/lib/Target/R600/R600EmitClauseMarkers.cpp b/lib/Target/R600/R600EmitClauseMarkers.cpp index 7c7469a04b2..fdc20302f4a 100644 --- a/lib/Target/R600/R600EmitClauseMarkers.cpp +++ b/lib/Target/R600/R600EmitClauseMarkers.cpp @@ -19,25 +19,31 @@ #include "R600InstrInfo.h" #include "R600MachineFunctionInfo.h" #include "R600RegisterInfo.h" +#include "AMDGPUSubtarget.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +using namespace llvm; + namespace llvm { + void initializeR600EmitClauseMarkersPass(PassRegistry&); +} + +namespace { -class R600EmitClauseMarkersPass : public MachineFunctionPass { +class R600EmitClauseMarkers : public MachineFunctionPass { private: - static char ID; const R600InstrInfo *TII; + int Address; unsigned OccupiedDwords(MachineInstr *MI) const { switch (MI->getOpcode()) { case AMDGPU::INTERP_PAIR_XY: case AMDGPU::INTERP_PAIR_ZW: case AMDGPU::INTERP_VEC_LOAD: - case AMDGPU::DOT4_eg_pseudo: - case AMDGPU::DOT4_r600_pseudo: + case AMDGPU::DOT_4: return 4; case AMDGPU::KILL: return 0; @@ -45,6 +51,11 @@ private: break; } + // These will be expanded to two ALU instructions in the + // ExpandSpecialInstructions pass. + if (TII->isLDSRetInstr(MI->getOpcode())) + return 2; + if(TII->isVector(*MI) || TII->isCubeOp(MI->getOpcode()) || TII->isReductionOp(MI->getOpcode())) @@ -61,8 +72,6 @@ private: } bool isALU(const MachineInstr *MI) const { - if (MI->getOpcode() == AMDGPU::KILLGT) - return false; if (TII->isALUInstr(MI->getOpcode())) return true; if (TII->isVector(*MI) || TII->isCubeOp(MI->getOpcode())) @@ -73,8 +82,7 @@ private: case AMDGPU::INTERP_PAIR_ZW: case AMDGPU::INTERP_VEC_LOAD: case AMDGPU::COPY: - case AMDGPU::DOT4_eg_pseudo: - case AMDGPU::DOT4_r600_pseudo: + case AMDGPU::DOT_4: return true; default: return false; @@ -85,37 +93,13 @@ private: switch (MI->getOpcode()) { case AMDGPU::KILL: case AMDGPU::RETURN: + case AMDGPU::IMPLICIT_DEF: return true; default: return false; } } - // Register Idx, then Const value - std::vector > ExtractConstRead(MachineInstr *MI) - const { - const R600Operands::Ops OpTable[3][2] = { - {R600Operands::SRC0, R600Operands::SRC0_SEL}, - {R600Operands::SRC1, R600Operands::SRC1_SEL}, - {R600Operands::SRC2, R600Operands::SRC2_SEL}, - }; - std::vector > Result; - - if (!TII->isALUInstr(MI->getOpcode())) - return Result; - for (unsigned j = 0; j < 3; j++) { - int SrcIdx = TII->getOperandIdx(MI->getOpcode(), OpTable[j][0]); - if (SrcIdx < 0) - break; - if (MI->getOperand(SrcIdx).getReg() == AMDGPU::ALU_CONST) { - unsigned Const = MI->getOperand( - TII->getOperandIdx(MI->getOpcode(), OpTable[j][1])).getImm(); - Result.push_back(std::pair(SrcIdx, Const)); - } - } - return Result; - } - std::pair getAccessedBankLine(unsigned Sel) const { // Sel is (512 + (kc_bank << 12) + ConstIndex) << 2 // (See also R600ISelLowering.cpp) @@ -131,11 +115,20 @@ private: } bool SubstituteKCacheBank(MachineInstr *MI, - std::vector > &CachedConsts) const { + std::vector > &CachedConsts, + bool UpdateInstr = true) const { std::vector > UsedKCache; - std::vector > Consts = ExtractConstRead(MI); - assert(TII->isALUInstr(MI->getOpcode()) && "Can't assign Const"); + + if (!TII->isALUInstr(MI->getOpcode()) && MI->getOpcode() != AMDGPU::DOT_4) + return true; + + const SmallVectorImpl > &Consts = + TII->getSrcs(MI); + assert((TII->isALUInstr(MI->getOpcode()) || + MI->getOpcode() == AMDGPU::DOT_4) && "Can't assign Const"); for (unsigned i = 0, n = Consts.size(); i < n; ++i) { + if (Consts[i].first->getReg() != AMDGPU::ALU_CONST) + continue; unsigned Sel = Consts[i].second; unsigned Chan = Sel & 3, Index = ((Sel >> 2) - 512) & 31; unsigned KCacheIndex = Index * 4 + Chan; @@ -161,25 +154,77 @@ private: return false; } - for (unsigned i = 0, n = Consts.size(); i < n; ++i) { - switch(UsedKCache[i].first) { + if (!UpdateInstr) + return true; + + for (unsigned i = 0, j = 0, n = Consts.size(); i < n; ++i) { + if (Consts[i].first->getReg() != AMDGPU::ALU_CONST) + continue; + switch(UsedKCache[j].first) { case 0: - MI->getOperand(Consts[i].first).setReg( - AMDGPU::R600_KC0RegClass.getRegister(UsedKCache[i].second)); + Consts[i].first->setReg( + AMDGPU::R600_KC0RegClass.getRegister(UsedKCache[j].second)); break; case 1: - MI->getOperand(Consts[i].first).setReg( - AMDGPU::R600_KC1RegClass.getRegister(UsedKCache[i].second)); + Consts[i].first->setReg( + AMDGPU::R600_KC1RegClass.getRegister(UsedKCache[j].second)); break; default: llvm_unreachable("Wrong Cache Line"); } + j++; + } + return true; + } + + bool canClauseLocalKillFitInClause( + unsigned AluInstCount, + std::vector > KCacheBanks, + MachineBasicBlock::iterator Def, + MachineBasicBlock::iterator BBEnd) { + const R600RegisterInfo &TRI = TII->getRegisterInfo(); + for (MachineInstr::const_mop_iterator + MOI = Def->operands_begin(), + MOE = Def->operands_end(); MOI != MOE; ++MOI) { + if (!MOI->isReg() || !MOI->isDef() || + TRI.isPhysRegLiveAcrossClauses(MOI->getReg())) + continue; + + // Def defines a clause local register, so check that its use will fit + // in the clause. + unsigned LastUseCount = 0; + for (MachineBasicBlock::iterator UseI = Def; UseI != BBEnd; ++UseI) { + AluInstCount += OccupiedDwords(UseI); + // Make sure we won't need to end the clause due to KCache limitations. + if (!SubstituteKCacheBank(UseI, KCacheBanks, false)) + return false; + + // We have reached the maximum instruction limit before finding the + // use that kills this register, so we cannot use this def in the + // current clause. + if (AluInstCount >= TII->getMaxAlusPerClause()) + return false; + + // Register kill flags have been cleared by the time we get to this + // pass, but it is safe to assume that all uses of this register + // occur in the same basic block as its definition, because + // it is illegal for the scheduler to schedule them in + // different blocks. + if (UseI->findRegisterUseOperandIdx(MOI->getReg())) + LastUseCount = AluInstCount; + + if (UseI != Def && UseI->findRegisterDefOperandIdx(MOI->getReg()) != -1) + break; + } + if (LastUseCount) + return LastUseCount <= TII->getMaxAlusPerClause(); + llvm_unreachable("Clause local register live at end of clause."); } return true; } MachineBasicBlock::iterator - MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const { + MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator I) { MachineBasicBlock::iterator ClauseHead = I; std::vector > KCacheBanks; bool PushBeforeModifier = false; @@ -189,38 +234,72 @@ private: continue; if (!isALU(I)) break; + if (AluInstCount > TII->getMaxAlusPerClause()) + break; if (I->getOpcode() == AMDGPU::PRED_X) { + // We put PRED_X in its own clause to ensure that ifcvt won't create + // clauses with more than 128 insts. + // IfCvt is indeed checking that "then" and "else" branches of an if + // statement have less than ~60 insts thus converted clauses can't be + // bigger than ~121 insts (predicate setter needs to be in the same + // clause as predicated alus). + if (AluInstCount > 0) + break; if (TII->getFlagOp(I).getImm() & MO_FLAG_PUSH) PushBeforeModifier = true; AluInstCount ++; continue; } - if (TII->isALUInstr(I->getOpcode()) && - !SubstituteKCacheBank(I, KCacheBanks)) + // XXX: GROUP_BARRIER instructions cannot be in the same ALU clause as: + // + // * KILL or INTERP instructions + // * Any instruction that sets UPDATE_EXEC_MASK or UPDATE_PRED bits + // * Uses waterfalling (i.e. INDEX_MODE = AR.X) + // + // XXX: These checks have not been implemented yet. + if (TII->mustBeLastInClause(I->getOpcode())) { + I++; break; - AluInstCount += OccupiedDwords(I); - if (AluInstCount > 124) + } + + // If this instruction defines a clause local register, make sure + // its use can fit in this clause. + if (!canClauseLocalKillFitInClause(AluInstCount, KCacheBanks, I, E)) break; + + if (!SubstituteKCacheBank(I, KCacheBanks)) + break; + AluInstCount += OccupiedDwords(I); } unsigned Opcode = PushBeforeModifier ? AMDGPU::CF_ALU_PUSH_BEFORE : AMDGPU::CF_ALU; BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead), TII->get(Opcode)) - .addImm(0) // ADDR + // We don't use the ADDR field until R600ControlFlowFinalizer pass, where + // it is safe to assume it is 0. However if we always put 0 here, the ifcvt + // pass may assume that identical ALU clause starter at the beginning of a + // true and false branch can be factorized which is not the case. + .addImm(Address++) // ADDR .addImm(KCacheBanks.empty()?0:KCacheBanks[0].first) // KB0 .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].first) // KB1 .addImm(KCacheBanks.empty()?0:2) // KM0 .addImm((KCacheBanks.size() < 2)?0:2) // KM1 .addImm(KCacheBanks.empty()?0:KCacheBanks[0].second) // KLINE0 .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].second) // KLINE1 - .addImm(AluInstCount); // COUNT + .addImm(AluInstCount) // COUNT + .addImm(1); // Enabled return I; } public: - R600EmitClauseMarkersPass(TargetMachine &tm) : MachineFunctionPass(ID), - TII (static_cast(tm.getInstrInfo())) { } + static char ID; + R600EmitClauseMarkers() : MachineFunctionPass(ID), TII(nullptr), Address(0) { + + initializeR600EmitClauseMarkersPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override { + TII = static_cast(MF.getSubtarget().getInstrInfo()); - virtual bool runOnMachineFunction(MachineFunction &MF) { for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end(); BB != BB_E; ++BB) { MachineBasicBlock &MBB = *BB; @@ -237,17 +316,21 @@ public: return false; } - const char *getPassName() const { + const char *getPassName() const override { return "R600 Emit Clause Markers Pass"; } }; -char R600EmitClauseMarkersPass::ID = 0; +char R600EmitClauseMarkers::ID = 0; -} +} // end anonymous namespace +INITIALIZE_PASS_BEGIN(R600EmitClauseMarkers, "emitclausemarkers", + "R600 Emit Clause Markters", false, false) +INITIALIZE_PASS_END(R600EmitClauseMarkers, "emitclausemarkers", + "R600 Emit Clause Markters", false, false) -llvm::FunctionPass *llvm::createR600EmitClauseMarkers(TargetMachine &TM) { - return new R600EmitClauseMarkersPass(TM); +llvm::FunctionPass *llvm::createR600EmitClauseMarkers() { + return new R600EmitClauseMarkers(); }