X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FScheduleDAG.cpp;h=d33f3d2cb62920b6795ff0ef94f4d58231ee18e9;hb=edfba7e707a4f2f2e800843a7ef980c27d7f4eff;hp=2bf8d5dc3c826f9f61a1aa8073396ea7b3713365;hpb=9e23336d0c99fc5cae04037ead6d8f2b677e8764;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp index 2bf8d5dc3c8..d33f3d2cb62 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp @@ -14,11 +14,11 @@ //===----------------------------------------------------------------------===// #define DEBUG_TYPE "pre-RA-sched" -#include "llvm/Constants.h" #include "llvm/Type.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetMachine.h" @@ -42,10 +42,11 @@ namespace { ScheduleDAG::ScheduleDAG(SelectionDAG &dag, MachineBasicBlock *bb, const TargetMachine &tm) : DAG(dag), BB(bb), TM(tm), MRI(BB->getParent()->getRegInfo()) { - TII = TM.getInstrInfo(); - MF = &DAG.getMachineFunction(); - TRI = TM.getRegisterInfo(); - ConstPool = BB->getParent()->getConstantPool(); + TII = TM.getInstrInfo(); + MF = &DAG.getMachineFunction(); + TRI = TM.getRegisterInfo(); + TLI = &DAG.getTargetLoweringInfo(); + ConstPool = BB->getParent()->getConstantPool(); } /// CheckForPhysRegDependency - Check if the dependency between def and use of @@ -77,13 +78,12 @@ static void CheckForPhysRegDependency(SDNode *Def, SDNode *Use, unsigned Op, SUnit *ScheduleDAG::Clone(SUnit *Old) { SUnit *SU = NewSUnit(Old->Node); + SU->OrigNode = Old->OrigNode; SU->FlaggedNodes = Old->FlaggedNodes; - SU->InstanceNo = SUnitMap[Old->Node].size(); SU->Latency = Old->Latency; SU->isTwoAddress = Old->isTwoAddress; SU->isCommutable = Old->isCommutable; SU->hasPhysRegDefs = Old->hasPhysRegDefs; - SUnitMap[Old->Node].push_back(SU); return SU; } @@ -95,15 +95,22 @@ void ScheduleDAG::BuildSchedUnits() { // Reserve entries in the vector for each of the SUnits we are creating. This // ensure that reallocation of the vector won't happen, so SUnit*'s won't get // invalidated. - SUnits.reserve(std::distance(DAG.allnodes_begin(), DAG.allnodes_end())); + SUnits.reserve(DAG.allnodes_size()); + // During scheduling, the NodeId field of SDNode is used to map SDNodes + // to their associated SUnits by holding SUnits table indices. A value + // of -1 means the SDNode does not yet have an associated SUnit. + for (SelectionDAG::allnodes_iterator NI = DAG.allnodes_begin(), + E = DAG.allnodes_end(); NI != E; ++NI) + NI->setNodeId(-1); + for (SelectionDAG::allnodes_iterator NI = DAG.allnodes_begin(), E = DAG.allnodes_end(); NI != E; ++NI) { if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate. continue; // If this node has already been processed, stop now. - if (SUnitMap[NI].size()) continue; + if (NI->getNodeId() != -1) continue; SUnit *NodeSUnit = NewSUnit(NI); @@ -118,7 +125,8 @@ void ScheduleDAG::BuildSchedUnits() { do { N = N->getOperand(N->getNumOperands()-1).Val; NodeSUnit->FlaggedNodes.push_back(N); - SUnitMap[N].push_back(NodeSUnit); + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); } while (N->getNumOperands() && N->getOperand(N->getNumOperands()-1).getValueType()== MVT::Flag); std::reverse(NodeSUnit->FlaggedNodes.begin(), @@ -135,11 +143,12 @@ void ScheduleDAG::BuildSchedUnits() { bool HasFlagUse = false; for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) - if (FlagVal.isOperandOf(*UI)) { + if (FlagVal.isOperandOf(UI->getUser())) { HasFlagUse = true; NodeSUnit->FlaggedNodes.push_back(N); - SUnitMap[N].push_back(NodeSUnit); - N = *UI; + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); + N = UI->getUser(); break; } if (!HasFlagUse) break; @@ -148,7 +157,8 @@ void ScheduleDAG::BuildSchedUnits() { // Now all flagged nodes are in FlaggedNodes and N is the bottom-most node. // Update the SUnit NodeSUnit->Node = N; - SUnitMap[N].push_back(NodeSUnit); + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); ComputeLatency(NodeSUnit); } @@ -185,11 +195,11 @@ void ScheduleDAG::BuildSchedUnits() { for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { SDNode *OpN = N->getOperand(i).Val; if (isPassiveNode(OpN)) continue; // Not scheduled. - SUnit *OpSU = SUnitMap[OpN].front(); + SUnit *OpSU = &SUnits[OpN->getNodeId()]; assert(OpSU && "Node has no SUnit!"); if (OpSU == SU) continue; // In the same group. - MVT::ValueType OpVT = N->getOperand(i).getValueType(); + MVT OpVT = N->getOperand(i).getValueType(); assert(OpVT != MVT::Flag && "Flagged nodes should be in same sunit!"); bool isChain = OpVT == MVT::Other; @@ -204,8 +214,6 @@ void ScheduleDAG::BuildSchedUnits() { // Remove MainNode from FlaggedNodes again. SU->FlaggedNodes.pop_back(); } - - return; } void ScheduleDAG::ComputeLatency(SUnit *SU) { @@ -216,26 +224,26 @@ void ScheduleDAG::ComputeLatency(SUnit *SU) { if (InstrItins.isEmpty()) { // No latency information. SU->Latency = 1; - } else { - SU->Latency = 0; - if (SU->Node->isTargetOpcode()) { - unsigned SchedClass = - TII->get(SU->Node->getTargetOpcode()).getSchedClass(); - InstrStage *S = InstrItins.begin(SchedClass); - InstrStage *E = InstrItins.end(SchedClass); + return; + } + + SU->Latency = 0; + if (SU->Node->isTargetOpcode()) { + unsigned SchedClass = TII->get(SU->Node->getTargetOpcode()).getSchedClass(); + const InstrStage *S = InstrItins.begin(SchedClass); + const InstrStage *E = InstrItins.end(SchedClass); + for (; S != E; ++S) + SU->Latency += S->Cycles; + } + for (unsigned i = 0, e = SU->FlaggedNodes.size(); i != e; ++i) { + SDNode *FNode = SU->FlaggedNodes[i]; + if (FNode->isTargetOpcode()) { + unsigned SchedClass = TII->get(FNode->getTargetOpcode()).getSchedClass(); + const InstrStage *S = InstrItins.begin(SchedClass); + const InstrStage *E = InstrItins.end(SchedClass); for (; S != E; ++S) SU->Latency += S->Cycles; } - for (unsigned i = 0, e = SU->FlaggedNodes.size(); i != e; ++i) { - SDNode *FNode = SU->FlaggedNodes[i]; - if (FNode->isTargetOpcode()) { - unsigned SchedClass =TII->get(FNode->getTargetOpcode()).getSchedClass(); - InstrStage *S = InstrItins.begin(SchedClass); - InstrStage *E = InstrItins.end(SchedClass); - for (; S != E; ++S) - SU->Latency += S->Cycles; - } - } } } @@ -367,7 +375,7 @@ unsigned ScheduleDAG::CountResults(SDNode *Node) { unsigned ScheduleDAG::CountOperands(SDNode *Node) { unsigned N = ComputeMemOperandsEnd(Node); while (N && isa(Node->getOperand(N - 1).Val)) - --N; // Ignore MemOperand nodes + --N; // Ignore MEMOPERAND nodes return N; } @@ -382,11 +390,12 @@ unsigned ScheduleDAG::ComputeMemOperandsEnd(SDNode *Node) { return N; } -static const TargetRegisterClass *getInstrOperandRegClass( - const TargetRegisterInfo *TRI, - const TargetInstrInfo *TII, - const TargetInstrDesc &II, - unsigned Op) { +/// getInstrOperandRegClass - Return register class of the operand of an +/// instruction of the specified TargetInstrDesc. +static const TargetRegisterClass* +getInstrOperandRegClass(const TargetRegisterInfo *TRI, + const TargetInstrInfo *TII, const TargetInstrDesc &II, + unsigned Op) { if (Op >= II.getNumOperands()) { assert(II.isVariadic() && "Invalid operand # of instruction"); return NULL; @@ -396,15 +405,19 @@ static const TargetRegisterClass *getInstrOperandRegClass( return TRI->getRegClass(II.OpInfo[Op].RegClass); } +/// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an +/// implicit physical register output. void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, - unsigned InstanceNo, unsigned SrcReg, + bool IsClone, unsigned SrcReg, DenseMap &VRBaseMap) { unsigned VRBase = 0; if (TargetRegisterInfo::isVirtualRegister(SrcReg)) { // Just use the input register directly! - if (InstanceNo > 0) - VRBaseMap.erase(SDOperand(Node, ResNo)); - bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,ResNo),SrcReg)); + SDOperand Op(Node, ResNo); + if (IsClone) + VRBaseMap.erase(Op); + bool isNew = VRBaseMap.insert(std::make_pair(Op, SrcReg)).second; + isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); return; } @@ -414,7 +427,7 @@ void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool MatchReg = true; for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); UI != E; ++UI) { - SDNode *Use = *UI; + SDNode *Use = UI->getUser(); bool Match = true; if (Use->getOpcode() == ISD::CopyToReg && Use->getOperand(2).Val == Node && @@ -430,7 +443,7 @@ void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, SDOperand Op = Use->getOperand(i); if (Op.Val != Node || Op.ResNo != ResNo) continue; - MVT::ValueType VT = Node->getValueType(Op.ResNo); + MVT VT = Node->getValueType(Op.ResNo); if (VT != MVT::Other && VT != MVT::Flag) Match = false; } @@ -447,8 +460,7 @@ void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, if (VRBase) { DstRC = MRI.getRegClass(VRBase); } else { - DstRC = DAG.getTargetLoweringInfo() - .getRegClassFor(Node->getValueType(ResNo)); + DstRC = TLI->getRegClassFor(Node->getValueType(ResNo)); } // If all uses are reading from the src physical register and copying the @@ -461,16 +473,38 @@ void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC); } - if (InstanceNo > 0) - VRBaseMap.erase(SDOperand(Node, ResNo)); - bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,ResNo), VRBase)); + SDOperand Op(Node, ResNo); + if (IsClone) + VRBaseMap.erase(Op); + bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; + isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); } -void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, - MachineInstr *MI, - const TargetInstrDesc &II, - DenseMap &VRBaseMap) { +/// getDstOfCopyToRegUse - If the only use of the specified result number of +/// node is a CopyToReg, return its destination register. Return 0 otherwise. +unsigned ScheduleDAG::getDstOfOnlyCopyToRegUse(SDNode *Node, + unsigned ResNo) const { + if (!Node->hasOneUse()) + return 0; + + SDNode *Use = Node->use_begin()->getUser(); + if (Use->getOpcode() == ISD::CopyToReg && + Use->getOperand(2).Val == Node && + Use->getOperand(2).ResNo == ResNo) { + unsigned Reg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return Reg; + } + return 0; +} + +void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, + const TargetInstrDesc &II, + DenseMap &VRBaseMap) { + assert(Node->getTargetOpcode() != TargetInstrInfo::IMPLICIT_DEF && + "IMPLICIT_DEF should have been handled as a special case elsewhere!"); + for (unsigned i = 0; i < II.getNumDefs(); ++i) { // If the specific node value is only used by a CopyToReg and the dest reg // is a vreg, use the CopyToReg'd destination register instead of creating @@ -478,7 +512,7 @@ void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, unsigned VRBase = 0; for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); UI != E; ++UI) { - SDNode *Use = *UI; + SDNode *Use = UI->getUser(); if (Use->getOpcode() == ISD::CopyToReg && Use->getOperand(2).Val == Node && Use->getOperand(2).ResNo == i) { @@ -500,14 +534,31 @@ void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, MI->addOperand(MachineOperand::CreateReg(VRBase, true)); } - bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,i), VRBase)); + SDOperand Op(Node, i); + bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; + isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); } } /// getVR - Return the virtual register corresponding to the specified result /// of the specified node. -static unsigned getVR(SDOperand Op, DenseMap &VRBaseMap) { +unsigned ScheduleDAG::getVR(SDOperand Op, + DenseMap &VRBaseMap) { + if (Op.isTargetOpcode() && + Op.getTargetOpcode() == TargetInstrInfo::IMPLICIT_DEF) { + // Add an IMPLICIT_DEF instruction before every use. + unsigned VReg = getDstOfOnlyCopyToRegUse(Op.Val, Op.ResNo); + // IMPLICIT_DEF can produce any type of result so its TargetInstrDesc + // does not include operand register class info. + if (!VReg) { + const TargetRegisterClass *RC = TLI->getRegClassFor(Op.getValueType()); + VReg = MRI.createVirtualRegister(RC); + } + BuildMI(BB, TII->get(TargetInstrInfo::IMPLICIT_DEF), VReg); + return VReg; + } + DenseMap::iterator I = VRBaseMap.find(Op); assert(I != VRBaseMap.end() && "Node emitted out of order - late"); return I->second; @@ -529,12 +580,11 @@ void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); - // Get/emit the operand. unsigned VReg = getVR(Op, VRBaseMap); const TargetInstrDesc &TID = MI->getDesc(); - bool isOptDef = (IIOpNum < TID.getNumOperands()) - ? (TID.OpInfo[IIOpNum].isOptionalDef()) : false; + bool isOptDef = IIOpNum < TID.getNumOperands() && + TID.OpInfo[IIOpNum].isOptionalDef(); MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef)); // Verify that it is right. @@ -566,8 +616,7 @@ void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, } else if (ConstantSDNode *C = dyn_cast(Op)) { MI->addOperand(MachineOperand::CreateImm(C->getValue())); } else if (ConstantFPSDNode *F = dyn_cast(Op)) { - const Type *FType = MVT::getTypeForValueType(Op.getValueType()); - ConstantFP *CFP = ConstantFP::get(FType, F->getValueAPF()); + ConstantFP *CFP = ConstantFP::get(F->getValueAPF()); MI->addOperand(MachineOperand::CreateFPImm(CFP)); } else if (RegisterSDNode *R = dyn_cast(Op)) { MI->addOperand(MachineOperand::CreateReg(R->getReg(), false)); @@ -617,18 +666,17 @@ void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, assert(getInstrOperandRegClass(TRI, TII, *II, IIOpNum) && "Don't have operand info for this instruction!"); } - } - + } } -void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MemOperand &MO) { - MI->addMemOperand(MO); +void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) { + MI->addMemOperand(*MF, MO); } -// Returns the Register Class of a subregister -static const TargetRegisterClass *getSubRegisterRegClass( - const TargetRegisterClass *TRC, - unsigned SubIdx) { +/// getSubRegisterRegClass - Returns the register class of specified register +/// class' "SubIdx"'th sub-register class. +static const TargetRegisterClass* +getSubRegisterRegClass(const TargetRegisterClass *TRC, unsigned SubIdx) { // Pick the register class of the subregister TargetRegisterInfo::regclass_iterator I = TRC->subregclasses_begin() + SubIdx-1; @@ -637,10 +685,12 @@ static const TargetRegisterClass *getSubRegisterRegClass( return *I; } -static const TargetRegisterClass *getSuperregRegisterClass( - const TargetRegisterClass *TRC, - unsigned SubIdx, - MVT::ValueType VT) { +/// getSuperRegisterRegClass - Returns the register class of a superreg A whose +/// "SubIdx"'th sub-register class is the specified register class and whose +/// type matches the specified type. +static const TargetRegisterClass* +getSuperRegisterRegClass(const TargetRegisterClass *TRC, + unsigned SubIdx, MVT VT) { // Pick the register class of the superegister for this type for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(), E = TRC->superregclasses_end(); I != E; ++I) @@ -656,30 +706,27 @@ void ScheduleDAG::EmitSubregNode(SDNode *Node, DenseMap &VRBaseMap) { unsigned VRBase = 0; unsigned Opc = Node->getTargetOpcode(); - if (Opc == TargetInstrInfo::EXTRACT_SUBREG) { - // If the node is only used by a CopyToReg and the dest reg is a vreg, use - // the CopyToReg'd destination register instead of creating a new vreg. - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *Use = *UI; - if (Use->getOpcode() == ISD::CopyToReg && - Use->getOperand(2).Val == Node) { - unsigned DestReg = cast(Use->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(DestReg)) { - VRBase = DestReg; - break; - } + + // If the node is only used by a CopyToReg and the dest reg is a vreg, use + // the CopyToReg'd destination register instead of creating a new vreg. + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *Use = UI->getUser(); + if (Use->getOpcode() == ISD::CopyToReg && + Use->getOperand(2).Val == Node) { + unsigned DestReg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(DestReg)) { + VRBase = DestReg; + break; } } - + } + + if (Opc == TargetInstrInfo::EXTRACT_SUBREG) { unsigned SubIdx = cast(Node->getOperand(1))->getValue(); - - // TODO: If the node is a use of a CopyFromReg from a physical register - // fold the extract into the copy now // Create the extract_subreg machine instruction. - MachineInstr *MI = - new MachineInstr(BB, TII->get(TargetInstrInfo::EXTRACT_SUBREG)); + MachineInstr *MI = BuildMI(*MF, TII->get(TargetInstrInfo::EXTRACT_SUBREG)); // Figure out the register class to create for the destreg. unsigned VReg = getVR(Node->getOperand(0), VRBaseMap); @@ -688,9 +735,11 @@ void ScheduleDAG::EmitSubregNode(SDNode *Node, if (VRBase) { // Grab the destination register +#ifndef NDEBUG const TargetRegisterClass *DRC = MRI.getRegClass(VRBase); assert(SRC && DRC && SRC == DRC && "Source subregister and destination must have the same class"); +#endif } else { // Create the reg assert(SRC && "Couldn't find source register class"); @@ -701,70 +750,54 @@ void ScheduleDAG::EmitSubregNode(SDNode *Node, MI->addOperand(MachineOperand::CreateReg(VRBase, true)); AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap); MI->addOperand(MachineOperand::CreateImm(SubIdx)); - - } else if (Opc == TargetInstrInfo::INSERT_SUBREG) { + BB->push_back(MI); + } else if (Opc == TargetInstrInfo::INSERT_SUBREG || + Opc == TargetInstrInfo::SUBREG_TO_REG) { SDOperand N0 = Node->getOperand(0); SDOperand N1 = Node->getOperand(1); SDOperand N2 = Node->getOperand(2); unsigned SubReg = getVR(N1, VRBaseMap); unsigned SubIdx = cast(N2)->getValue(); - // TODO: Add tracking info to MachineRegisterInfo of which vregs are subregs - // to allow coalescing in the allocator - - // If the node is only used by a CopyToReg and the dest reg is a vreg, use - // the CopyToReg'd destination register instead of creating a new vreg. - // If the CopyToReg'd destination register is physical, then fold the - // insert into the copy - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *Use = *UI; - if (Use->getOpcode() == ISD::CopyToReg && - Use->getOperand(2).Val == Node) { - unsigned DestReg = cast(Use->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(DestReg)) { - VRBase = DestReg; - break; - } - } - } - - // Create the insert_subreg machine instruction. - MachineInstr *MI = - new MachineInstr(BB, TII->get(TargetInstrInfo::INSERT_SUBREG)); // Figure out the register class to create for the destreg. const TargetRegisterClass *TRC = 0; if (VRBase) { TRC = MRI.getRegClass(VRBase); } else { - TRC = getSuperregRegisterClass(MRI.getRegClass(SubReg), SubIdx, + TRC = getSuperRegisterRegClass(MRI.getRegClass(SubReg), SubIdx, Node->getValueType(0)); assert(TRC && "Couldn't determine register class for insert_subreg"); VRBase = MRI.createVirtualRegister(TRC); // Create the reg } + // Create the insert_subreg or subreg_to_reg machine instruction. + MachineInstr *MI = BuildMI(*MF, TII->get(Opc)); MI->addOperand(MachineOperand::CreateReg(VRBase, true)); - // If N0 is a constant then it indicates the insert is being done - // into a target specific constant value, not a register. - if (const ConstantSDNode *SD = dyn_cast(N0)) + // If creating a subreg_to_reg, then the first input operand + // is an implicit value immediate, otherwise it's a register + if (Opc == TargetInstrInfo::SUBREG_TO_REG) { + const ConstantSDNode *SD = cast(N0); MI->addOperand(MachineOperand::CreateImm(SD->getValue())); - else + } else AddOperand(MI, N0, 0, 0, VRBaseMap); // Add the subregster being inserted AddOperand(MI, N1, 0, 0, VRBaseMap); MI->addOperand(MachineOperand::CreateImm(SubIdx)); + BB->push_back(MI); } else - assert(0 && "Node is not a subreg insert or extract"); + assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg"); - bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,0), VRBase)); + SDOperand Op(Node, 0); + bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second; + isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); } /// EmitNode - Generate machine code for an node and needed dependencies. /// -void ScheduleDAG::EmitNode(SDNode *Node, unsigned InstanceNo, +void ScheduleDAG::EmitNode(SDNode *Node, bool IsClone, DenseMap &VRBaseMap) { // If machine instruction if (Node->isTargetOpcode()) { @@ -772,27 +805,31 @@ void ScheduleDAG::EmitNode(SDNode *Node, unsigned InstanceNo, // Handle subreg insert/extract specially if (Opc == TargetInstrInfo::EXTRACT_SUBREG || - Opc == TargetInstrInfo::INSERT_SUBREG) { + Opc == TargetInstrInfo::INSERT_SUBREG || + Opc == TargetInstrInfo::SUBREG_TO_REG) { EmitSubregNode(Node, VRBaseMap); return; } + + if (Opc == TargetInstrInfo::IMPLICIT_DEF) + // We want a unique VR for each IMPLICIT_DEF use. + return; const TargetInstrDesc &II = TII->get(Opc); - unsigned NumResults = CountResults(Node); unsigned NodeOperands = CountOperands(Node); unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node); - unsigned NumMIOperands = NodeOperands + NumResults; bool HasPhysRegOuts = (NumResults > II.getNumDefs()) && II.getImplicitDefs() != 0; #ifndef NDEBUG + unsigned NumMIOperands = NodeOperands + NumResults; assert((II.getNumOperands() == NumMIOperands || HasPhysRegOuts || II.isVariadic()) && "#operands for dag node doesn't match .td file!"); #endif // Create the new machine instruction. - MachineInstr *MI = new MachineInstr(II); + MachineInstr *MI = BuildMI(*MF, II); // Add result register values for things that are defined by this // instruction. @@ -816,7 +853,7 @@ void ScheduleDAG::EmitNode(SDNode *Node, unsigned InstanceNo, else { DOUT << "Sched: COMMUTED TO: " << *NewMI; if (MI != NewMI) { - delete MI; + MF->DeleteMachineInstr(MI); MI = NewMI; } ++NumCommutes; @@ -826,7 +863,7 @@ void ScheduleDAG::EmitNode(SDNode *Node, unsigned InstanceNo, if (II.usesCustomDAGSchedInsertionHook()) // Insert this instruction into the basic block using a target // specific inserter which may returns a new basic block. - BB = DAG.getTargetLoweringInfo().EmitInstrWithCustomInserter(MI, BB); + BB = TLI->EmitInstrWithCustomInserter(MI, BB); else BB->push_back(MI); @@ -835,117 +872,98 @@ void ScheduleDAG::EmitNode(SDNode *Node, unsigned InstanceNo, for (unsigned i = II.getNumDefs(); i < NumResults; ++i) { unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()]; if (Node->hasAnyUseOfValue(i)) - EmitCopyFromReg(Node, i, InstanceNo, Reg, VRBaseMap); + EmitCopyFromReg(Node, i, IsClone, Reg, VRBaseMap); } } - } else { - switch (Node->getOpcode()) { - default: + return; + } + + switch (Node->getOpcode()) { + default: #ifndef NDEBUG - Node->dump(&DAG); + Node->dump(&DAG); #endif - assert(0 && "This target-independent node should have been selected!"); - case ISD::EntryToken: // fall thru - case ISD::TokenFactor: - case ISD::LABEL: - case ISD::DECLARE: - case ISD::SRCVALUE: - break; - case ISD::CopyToReg: { - unsigned SrcReg; - SDOperand SrcVal = Node->getOperand(2); - if (RegisterSDNode *R = dyn_cast(SrcVal)) - SrcReg = R->getReg(); - else - SrcReg = getVR(SrcVal, VRBaseMap); + assert(0 && "This target-independent node should have been selected!"); + break; + case ISD::EntryToken: + assert(0 && "EntryToken should have been excluded from the schedule!"); + break; + case ISD::TokenFactor: // fall thru + break; + case ISD::CopyToReg: { + unsigned SrcReg; + SDOperand SrcVal = Node->getOperand(2); + if (RegisterSDNode *R = dyn_cast(SrcVal)) + SrcReg = R->getReg(); + else + SrcReg = getVR(SrcVal, VRBaseMap); - unsigned DestReg = cast(Node->getOperand(1))->getReg(); - if (SrcReg == DestReg) // Coalesced away the copy? Ignore. - break; + unsigned DestReg = cast(Node->getOperand(1))->getReg(); + if (SrcReg == DestReg) // Coalesced away the copy? Ignore. + break; - const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0; - // Get the register classes of the src/dst. - if (TargetRegisterInfo::isVirtualRegister(SrcReg)) - SrcTRC = MRI.getRegClass(SrcReg); - else - SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType()); + const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0; + // Get the register classes of the src/dst. + if (TargetRegisterInfo::isVirtualRegister(SrcReg)) + SrcTRC = MRI.getRegClass(SrcReg); + else + SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType()); - if (TargetRegisterInfo::isVirtualRegister(DestReg)) - DstTRC = MRI.getRegClass(DestReg); - else - DstTRC = TRI->getPhysicalRegisterRegClass(DestReg, + if (TargetRegisterInfo::isVirtualRegister(DestReg)) + DstTRC = MRI.getRegClass(DestReg); + else + DstTRC = TRI->getPhysicalRegisterRegClass(DestReg, Node->getOperand(1).getValueType()); - TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC); - break; - } - case ISD::CopyFromReg: { - unsigned SrcReg = cast(Node->getOperand(1))->getReg(); - EmitCopyFromReg(Node, 0, InstanceNo, SrcReg, VRBaseMap); - break; - } - case ISD::INLINEASM: { - unsigned NumOps = Node->getNumOperands(); - if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag) - --NumOps; // Ignore the flag operand. + TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC); + break; + } + case ISD::CopyFromReg: { + unsigned SrcReg = cast(Node->getOperand(1))->getReg(); + EmitCopyFromReg(Node, 0, IsClone, SrcReg, VRBaseMap); + break; + } + case ISD::INLINEASM: { + unsigned NumOps = Node->getNumOperands(); + if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag) + --NumOps; // Ignore the flag operand. - // Create the inline asm machine instruction. - MachineInstr *MI = - new MachineInstr(BB, TII->get(TargetInstrInfo::INLINEASM)); - - // Add the asm string as an external symbol operand. - const char *AsmStr = - cast(Node->getOperand(1))->getSymbol(); - MI->addOperand(MachineOperand::CreateES(AsmStr)); + // Create the inline asm machine instruction. + MachineInstr *MI = BuildMI(*MF, TII->get(TargetInstrInfo::INLINEASM)); + + // Add the asm string as an external symbol operand. + const char *AsmStr = + cast(Node->getOperand(1))->getSymbol(); + MI->addOperand(MachineOperand::CreateES(AsmStr)); - // Add all of the operand registers to the instruction. - for (unsigned i = 2; i != NumOps;) { - unsigned Flags = cast(Node->getOperand(i))->getValue(); - unsigned NumVals = Flags >> 3; + // Add all of the operand registers to the instruction. + for (unsigned i = 2; i != NumOps;) { + unsigned Flags = cast(Node->getOperand(i))->getValue(); + unsigned NumVals = Flags >> 3; - MI->addOperand(MachineOperand::CreateImm(Flags)); - ++i; // Skip the ID value. + MI->addOperand(MachineOperand::CreateImm(Flags)); + ++i; // Skip the ID value. - switch (Flags & 7) { - default: assert(0 && "Bad flags!"); - case 1: // Use of register. - for (; NumVals; --NumVals, ++i) { - unsigned Reg = cast(Node->getOperand(i))->getReg(); - MI->addOperand(MachineOperand::CreateReg(Reg, false)); - } - break; - case 2: // Def of register. - for (; NumVals; --NumVals, ++i) { - unsigned Reg = cast(Node->getOperand(i))->getReg(); - MI->addOperand(MachineOperand::CreateReg(Reg, true)); - } - break; - case 3: { // Immediate. - for (; NumVals; --NumVals, ++i) { - if (ConstantSDNode *CS = - dyn_cast(Node->getOperand(i))) { - MI->addOperand(MachineOperand::CreateImm(CS->getValue())); - } else if (GlobalAddressSDNode *GA = - dyn_cast(Node->getOperand(i))) { - MI->addOperand(MachineOperand::CreateGA(GA->getGlobal(), - GA->getOffset())); - } else { - BasicBlockSDNode *BB =cast(Node->getOperand(i)); - MI->addOperand(MachineOperand::CreateMBB(BB->getBasicBlock())); - } - } - break; - } - case 4: // Addressing mode. - // The addressing mode has been selected, just add all of the - // operands to the machine instruction. - for (; NumVals; --NumVals, ++i) - AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap); - break; + switch (Flags & 7) { + default: assert(0 && "Bad flags!"); + case 2: // Def of register. + for (; NumVals; --NumVals, ++i) { + unsigned Reg = cast(Node->getOperand(i))->getReg(); + MI->addOperand(MachineOperand::CreateReg(Reg, true)); } + break; + case 1: // Use of register. + case 3: // Immediate. + case 4: // Addressing mode. + // The addressing mode has been selected, just add all of the + // operands to the machine instruction. + for (; NumVals; --NumVals, ++i) + AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap); + break; } - break; - } } + BB->push_back(MI); + break; + } } } @@ -978,7 +996,8 @@ void ScheduleDAG::EmitCrossRCCopy(SUnit *SU, // Copy from physical register. assert(I->Reg && "Unknown physical register!"); unsigned VRBase = MRI.createVirtualRegister(SU->CopyDstRC); - bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)); + bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)).second; + isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); TII->copyRegToReg(*BB, BB->end(), VRBase, I->Reg, SU->CopyDstRC, SU->CopySrcRC); @@ -987,50 +1006,14 @@ void ScheduleDAG::EmitCrossRCCopy(SUnit *SU, } } -/// regIsLive - Return true if the specified register is live due to a -/// live in copy. -static bool regIsLive(unsigned Reg, BitVector &LiveRegs, - const TargetRegisterInfo *TRI) { - if (LiveRegs[Reg]) - return true; - for (const unsigned *AS = TRI->getAliasSet(Reg); *AS; ++AS) - if (LiveRegs[*AS]) - return true; - return false; -} - -/// regIsClobbered - Return true if the specified register is defined in -/// between the two specific instructions. -static bool regIsClobbered(unsigned Reg, MachineBasicBlock *MBB, - MachineBasicBlock::iterator InsertPos, - MachineBasicBlock::iterator UsePos, - const TargetRegisterInfo *TRI) { - for (MachineBasicBlock::iterator I = InsertPos; I != UsePos; ++I) { - MachineInstr *MI = I; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isRegister() || !MO.isDef()) - continue; - unsigned DefReg = MO.getReg(); - if (TargetRegisterInfo::isVirtualRegister(DefReg)) - continue; - if (TRI->regsOverlap(DefReg, Reg)) - return true; - } - } - return false; -} - /// EmitLiveInCopy - Emit a copy for a live in physical register. If the /// physical register has only a single copy use, then coalesced the copy -/// if possible. It returns the destination register of the emitted copy -/// if it is a physical register; otherwise it returns zero. -unsigned ScheduleDAG::EmitLiveInCopy(MachineBasicBlock *MBB, - MachineBasicBlock::iterator &InsertPos, - unsigned VirtReg, unsigned PhysReg, - const TargetRegisterClass *RC, - BitVector &LiveRegsBefore, - BitVector &LiveRegsAfter) { +/// if possible. +void ScheduleDAG::EmitLiveInCopy(MachineBasicBlock *MBB, + MachineBasicBlock::iterator &InsertPos, + unsigned VirtReg, unsigned PhysReg, + const TargetRegisterClass *RC, + DenseMap &CopyRegMap){ unsigned NumUses = 0; MachineInstr *UseMI = NULL; for (MachineRegisterInfo::use_iterator UI = MRI.use_begin(VirtReg), @@ -1041,95 +1024,61 @@ unsigned ScheduleDAG::EmitLiveInCopy(MachineBasicBlock *MBB, } // If the number of uses is not one, or the use is not a move instruction, - // don't coalesce. + // don't coalesce. Also, only coalesce away a virtual register to virtual + // register copy. + bool Coalesced = false; unsigned SrcReg, DstReg; - if (NumUses != 1 || - !TII->isMoveInstr(*UseMI, SrcReg, DstReg)) { - TII->copyRegToReg(*MBB, InsertPos, VirtReg, PhysReg, RC, RC); - return 0; + if (NumUses == 1 && + TII->isMoveInstr(*UseMI, SrcReg, DstReg) && + TargetRegisterInfo::isVirtualRegister(DstReg)) { + VirtReg = DstReg; + Coalesced = true; } - // Coalesce away a virtual register to virtual register copy. - if (TargetRegisterInfo::isVirtualRegister(DstReg)) { - TII->copyRegToReg(*MBB, InsertPos, DstReg, PhysReg, RC, RC); - if (&*InsertPos == UseMI) ++InsertPos; - MBB->erase(UseMI); - return 0; - } - - // If the destination is a physical register, check if it's safe to - // coalesce. If there is a def of the register between the insertion point and - // the use, then it's not safe. - if (regIsClobbered(DstReg, MBB, InsertPos, UseMI, TRI)) { - TII->copyRegToReg(*MBB, InsertPos, VirtReg, PhysReg, RC, RC); - return 0; + // Now find an ideal location to insert the copy. + MachineBasicBlock::iterator Pos = InsertPos; + while (Pos != MBB->begin()) { + MachineInstr *PrevMI = prior(Pos); + DenseMap::iterator RI = CopyRegMap.find(PrevMI); + // copyRegToReg might emit multiple instructions to do a copy. + unsigned CopyDstReg = (RI == CopyRegMap.end()) ? 0 : RI->second; + if (CopyDstReg && !TRI->regsOverlap(CopyDstReg, PhysReg)) + // This is what the BB looks like right now: + // r1024 = mov r0 + // ... + // r1 = mov r1024 + // + // We want to insert "r1025 = mov r1". Inserting this copy below the + // move to r1024 makes it impossible for that move to be coalesced. + // + // r1025 = mov r1 + // r1024 = mov r0 + // ... + // r1 = mov 1024 + // r2 = mov 1025 + break; // Woot! Found a good location. + --Pos; } - // Also check already processed livein copies and determine the safe location - // to insert the copy. e.g. Suppose livein r0 is already processed and now - // we are inserting r1 copy to vr1025 which will be coalesced to r0. - // vr1024 = r0 - // - // ... - // It's safe to insert the copy from r1 to r0. - // vr1024 = r0 - // r0 = r1 - // - // However, if livein r0 copy is coalesced to r1: - // r1 = r0 - // - // ... - // Then it's not safe to insert the copy from r1 to r0 at the insertion pt. - // Nor is it safe to insert it at the start of the MBB. - // - // If livein r3 is already processed and it's coalesced to r1. - // -- safe to insert here - // r1 = r3 - // -- not safe - // Then it's safe to insert at the start of the MBB. - if (regIsLive(DstReg, LiveRegsAfter, TRI)) { - if (regIsLive(PhysReg, LiveRegsBefore, TRI)) { - // FIXME: Still possible to find a safe place to insert the copy. - TII->copyRegToReg(*MBB, InsertPos, VirtReg, PhysReg, RC, RC); - return 0; - } - TII->copyRegToReg(*MBB, MBB->begin(), DstReg, PhysReg, RC, RC); + TII->copyRegToReg(*MBB, Pos, VirtReg, PhysReg, RC, RC); + CopyRegMap.insert(std::make_pair(prior(Pos), VirtReg)); + if (Coalesced) { if (&*InsertPos == UseMI) ++InsertPos; MBB->erase(UseMI); - return DstReg; } - TII->copyRegToReg(*MBB, InsertPos, DstReg, PhysReg, RC, RC); - if (&*InsertPos == UseMI) ++InsertPos; - MBB->erase(UseMI); - return DstReg; } /// EmitLiveInCopies - If this is the first basic block in the function, /// and if it has live ins that need to be copied into vregs, emit the /// copies into the top of the block. void ScheduleDAG::EmitLiveInCopies(MachineBasicBlock *MBB) { - BitVector LiveRegsBefore; // Live registers before insertion pt. - BitVector LiveRegsAfter; // Live registers after insertion pt. - LiveRegsBefore.resize(TRI->getNumRegs()); - LiveRegsAfter.resize(TRI->getNumRegs()); - + DenseMap CopyRegMap; MachineBasicBlock::iterator InsertPos = MBB->begin(); for (MachineRegisterInfo::livein_iterator LI = MRI.livein_begin(), E = MRI.livein_end(); LI != E; ++LI) if (LI->second) { const TargetRegisterClass *RC = MRI.getRegClass(LI->second); - unsigned Reg = EmitLiveInCopy(MBB, InsertPos, LI->second, LI->first, RC, - LiveRegsBefore, LiveRegsAfter); - if (Reg) { - LiveRegsAfter.set(Reg); - for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); - unsigned SubReg = *SubRegs; ++SubRegs) - LiveRegsAfter.set(SubReg); - } - LiveRegsBefore.set(LI->first); - for (const unsigned *SubRegs = TRI->getSubRegisters(LI->first); - unsigned SubReg = *SubRegs; ++SubRegs) - LiveRegsBefore.set(SubReg); + EmitLiveInCopy(MBB, InsertPos, LI->second, LI->first, RC, CopyRegMap); } } @@ -1154,17 +1103,18 @@ void ScheduleDAG::EmitSchedule() { DenseMap VRBaseMap; DenseMap CopyVRBaseMap; for (unsigned i = 0, e = Sequence.size(); i != e; i++) { - if (SUnit *SU = Sequence[i]) { - for (unsigned j = 0, ee = SU->FlaggedNodes.size(); j != ee; ++j) - EmitNode(SU->FlaggedNodes[j], SU->InstanceNo, VRBaseMap); - if (SU->Node) - EmitNode(SU->Node, SU->InstanceNo, VRBaseMap); - else - EmitCrossRCCopy(SU, CopyVRBaseMap); - } else { + SUnit *SU = Sequence[i]; + if (!SU) { // Null SUnit* is a noop. EmitNoop(); + continue; } + for (unsigned j = 0, ee = SU->FlaggedNodes.size(); j != ee; ++j) + EmitNode(SU->FlaggedNodes[j], SU->OrigNode != SU, VRBaseMap); + if (!SU->Node) + EmitCrossRCCopy(SU, CopyVRBaseMap); + else + EmitNode(SU->Node, SU->OrigNode != SU, VRBaseMap); } if (isEntryBB && SchedLiveInCopies)