X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FScheduleDAG.cpp;h=7194ed0ab60fd61d9ccf300672dec65b7661a51a;hb=8d96144b0e9688a7d252be877d08c0699f879d70;hp=d290d88615ea424c1b3d0ae9088ed4d84edaafe1;hpb=831e0374a7da654071279f2b1f93f52f7eeb5eae;p=oota-llvm.git diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp index d290d88615e..7194ed0ab60 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp @@ -2,8 +2,8 @@ // // The LLVM Compiler Infrastructure // -// This file was developed by James M. Laskey and is distributed under the -// University of Illinois Open Source License. See LICENSE.TXT for details. +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // @@ -13,20 +13,81 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "sched" +#define DEBUG_TYPE "pre-RA-sched" #include "llvm/Type.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/SSARegMap.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetData.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" using namespace llvm; +STATISTIC(NumCommutes, "Number of instructions commuted"); + +namespace { + static cl::opt + SchedLiveInCopies("schedule-livein-copies", + cl::desc("Schedule copies of livein registers"), + cl::init(false)); +} + +ScheduleDAG::ScheduleDAG(SelectionDAG &dag, MachineBasicBlock *bb, + const TargetMachine &tm) + : DAG(dag), BB(bb), TM(tm), MRI(BB->getParent()->getRegInfo()) { + TII = TM.getInstrInfo(); + MF = &DAG.getMachineFunction(); + TRI = TM.getRegisterInfo(); + TLI = &DAG.getTargetLoweringInfo(); + ConstPool = BB->getParent()->getConstantPool(); +} + +/// CheckForPhysRegDependency - Check if the dependency between def and use of +/// a specified operand is a physical register dependency. If so, returns the +/// register and the cost of copying the register. +static void CheckForPhysRegDependency(SDNode *Def, SDNode *Use, unsigned Op, + const TargetRegisterInfo *TRI, + const TargetInstrInfo *TII, + unsigned &PhysReg, int &Cost) { + if (Op != 2 || Use->getOpcode() != ISD::CopyToReg) + return; + + unsigned Reg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return; + + unsigned ResNo = Use->getOperand(2).ResNo; + if (Def->isTargetOpcode()) { + const TargetInstrDesc &II = TII->get(Def->getTargetOpcode()); + if (ResNo >= II.getNumDefs() && + II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg) { + PhysReg = Reg; + const TargetRegisterClass *RC = + TRI->getPhysicalRegisterRegClass(Reg, Def->getValueType(ResNo)); + Cost = RC->getCopyCost(); + } + } +} + +SUnit *ScheduleDAG::Clone(SUnit *Old) { + SUnit *SU = NewSUnit(Old->Node); + SU->OrigNode = Old->OrigNode; + SU->FlaggedNodes = Old->FlaggedNodes; + SU->Latency = Old->Latency; + SU->isTwoAddress = Old->isTwoAddress; + SU->isCommutable = Old->isCommutable; + SU->hasPhysRegDefs = Old->hasPhysRegDefs; + return SU; +} + + /// BuildSchedUnits - Build SUnits from the selection dag that we are input. /// This SUnit graph is similar to the SelectionDAG, but represents flagged /// together nodes with a single SUnit. @@ -34,17 +95,22 @@ void ScheduleDAG::BuildSchedUnits() { // Reserve entries in the vector for each of the SUnits we are creating. This // ensure that reallocation of the vector won't happen, so SUnit*'s won't get // invalidated. - SUnits.reserve(std::distance(DAG.allnodes_begin(), DAG.allnodes_end())); - - const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); + SUnits.reserve(DAG.allnodes_size()); + // During scheduling, the NodeId field of SDNode is used to map SDNodes + // to their associated SUnits by holding SUnits table indices. A value + // of -1 means the SDNode does not yet have an associated SUnit. + for (SelectionDAG::allnodes_iterator NI = DAG.allnodes_begin(), + E = DAG.allnodes_end(); NI != E; ++NI) + NI->setNodeId(-1); + for (SelectionDAG::allnodes_iterator NI = DAG.allnodes_begin(), E = DAG.allnodes_end(); NI != E; ++NI) { if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate. continue; // If this node has already been processed, stop now. - if (SUnitMap[NI]) continue; + if (NI->getNodeId() != -1) continue; SUnit *NodeSUnit = NewSUnit(NI); @@ -59,7 +125,8 @@ void ScheduleDAG::BuildSchedUnits() { do { N = N->getOperand(N->getNumOperands()-1).Val; NodeSUnit->FlaggedNodes.push_back(N); - SUnitMap[N] = NodeSUnit; + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); } while (N->getNumOperands() && N->getOperand(N->getNumOperands()-1).getValueType()== MVT::Flag); std::reverse(NodeSUnit->FlaggedNodes.begin(), @@ -76,11 +143,12 @@ void ScheduleDAG::BuildSchedUnits() { bool HasFlagUse = false; for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) - if (FlagVal.isOperand(*UI)) { + if (FlagVal.isOperandOf(UI->getUser())) { HasFlagUse = true; NodeSUnit->FlaggedNodes.push_back(N); - SUnitMap[N] = NodeSUnit; - N = *UI; + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); + N = UI->getUser(); break; } if (!HasFlagUse) break; @@ -89,33 +157,10 @@ void ScheduleDAG::BuildSchedUnits() { // Now all flagged nodes are in FlaggedNodes and N is the bottom-most node. // Update the SUnit NodeSUnit->Node = N; - SUnitMap[N] = NodeSUnit; - - // Compute the latency for the node. We use the sum of the latencies for - // all nodes flagged together into this SUnit. - if (InstrItins.isEmpty()) { - // No latency information. - NodeSUnit->Latency = 1; - } else { - NodeSUnit->Latency = 0; - if (N->isTargetOpcode()) { - unsigned SchedClass = TII->getSchedClass(N->getTargetOpcode()); - InstrStage *S = InstrItins.begin(SchedClass); - InstrStage *E = InstrItins.end(SchedClass); - for (; S != E; ++S) - NodeSUnit->Latency += S->Cycles; - } - for (unsigned i = 0, e = NodeSUnit->FlaggedNodes.size(); i != e; ++i) { - SDNode *FNode = NodeSUnit->FlaggedNodes[i]; - if (FNode->isTargetOpcode()) { - unsigned SchedClass = TII->getSchedClass(FNode->getTargetOpcode()); - InstrStage *S = InstrItins.begin(SchedClass); - InstrStage *E = InstrItins.end(SchedClass); - for (; S != E; ++S) - NodeSUnit->Latency += S->Cycles; - } - } - } + assert(N->getNodeId() == -1 && "Node already inserted!"); + N->setNodeId(NodeSUnit->NodeNum); + + ComputeLatency(NodeSUnit); } // Pass 2: add the preds, succs, etc. @@ -125,13 +170,14 @@ void ScheduleDAG::BuildSchedUnits() { if (MainNode->isTargetOpcode()) { unsigned Opc = MainNode->getTargetOpcode(); - for (unsigned i = 0, ee = TII->getNumOperands(Opc); i != ee; ++i) { - if (TII->getOperandConstraint(Opc, i, TOI::TIED_TO) != -1) { + const TargetInstrDesc &TID = TII->get(Opc); + for (unsigned i = 0; i != TID.getNumOperands(); ++i) { + if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { SU->isTwoAddress = true; break; } } - if (TII->isCommutableInstr(Opc)) + if (TID.isCommutable()) SU->isCommutable = true; } @@ -141,78 +187,178 @@ void ScheduleDAG::BuildSchedUnits() { for (unsigned n = 0, e = SU->FlaggedNodes.size(); n != e; ++n) { SDNode *N = SU->FlaggedNodes[n]; + if (N->isTargetOpcode() && + TII->get(N->getTargetOpcode()).getImplicitDefs() && + CountResults(N) > TII->get(N->getTargetOpcode()).getNumDefs()) + SU->hasPhysRegDefs = true; for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { SDNode *OpN = N->getOperand(i).Val; if (isPassiveNode(OpN)) continue; // Not scheduled. - SUnit *OpSU = SUnitMap[OpN]; + SUnit *OpSU = &SUnits[OpN->getNodeId()]; assert(OpSU && "Node has no SUnit!"); if (OpSU == SU) continue; // In the same group. - MVT::ValueType OpVT = N->getOperand(i).getValueType(); + MVT OpVT = N->getOperand(i).getValueType(); assert(OpVT != MVT::Flag && "Flagged nodes should be in same sunit!"); bool isChain = OpVT == MVT::Other; - - if (SU->addPred(OpSU, isChain)) { - if (!isChain) { - SU->NumPreds++; - SU->NumPredsLeft++; - } else { - SU->NumChainPredsLeft++; - } - } - if (OpSU->addSucc(SU, isChain)) { - if (!isChain) { - OpSU->NumSuccs++; - OpSU->NumSuccsLeft++; - } else { - OpSU->NumChainSuccsLeft++; - } - } + + unsigned PhysReg = 0; + int Cost = 1; + // Determine if this is a physical register dependency. + CheckForPhysRegDependency(OpN, N, i, TRI, TII, PhysReg, Cost); + SU->addPred(OpSU, isChain, false, PhysReg, Cost); } } // Remove MainNode from FlaggedNodes again. SU->FlaggedNodes.pop_back(); } - - return; } -static void CalculateDepths(SUnit &SU, unsigned Depth) { - if (SU.Depth == 0 || Depth > SU.Depth) { - SU.Depth = Depth; - for (SUnit::succ_iterator I = SU.Succs.begin(), E = SU.Succs.end(); - I != E; ++I) - CalculateDepths(*I->first, Depth+1); +void ScheduleDAG::ComputeLatency(SUnit *SU) { + const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); + + // Compute the latency for the node. We use the sum of the latencies for + // all nodes flagged together into this SUnit. + if (InstrItins.isEmpty()) { + // No latency information. + SU->Latency = 1; + } else { + SU->Latency = 0; + if (SU->Node->isTargetOpcode()) { + unsigned SchedClass = + TII->get(SU->Node->getTargetOpcode()).getSchedClass(); + const InstrStage *S = InstrItins.begin(SchedClass); + const InstrStage *E = InstrItins.end(SchedClass); + for (; S != E; ++S) + SU->Latency += S->Cycles; + } + for (unsigned i = 0, e = SU->FlaggedNodes.size(); i != e; ++i) { + SDNode *FNode = SU->FlaggedNodes[i]; + if (FNode->isTargetOpcode()) { + unsigned SchedClass =TII->get(FNode->getTargetOpcode()).getSchedClass(); + const InstrStage *S = InstrItins.begin(SchedClass); + const InstrStage *E = InstrItins.end(SchedClass); + for (; S != E; ++S) + SU->Latency += S->Cycles; + } + } } } +/// CalculateDepths - compute depths using algorithms for the longest +/// paths in the DAG void ScheduleDAG::CalculateDepths() { - SUnit *Entry = SUnitMap[DAG.getEntryNode().Val]; - ::CalculateDepths(*Entry, 0U); - for (unsigned i = 0, e = SUnits.size(); i != e; ++i) - if (SUnits[i].Preds.size() == 0 && &SUnits[i] != Entry) { - ::CalculateDepths(SUnits[i], 0U); + unsigned DAGSize = SUnits.size(); + std::vector InDegree(DAGSize); + std::vector WorkList; + WorkList.reserve(DAGSize); + + // Initialize the data structures + for (unsigned i = 0, e = DAGSize; i != e; ++i) { + SUnit *SU = &SUnits[i]; + int NodeNum = SU->NodeNum; + unsigned Degree = SU->Preds.size(); + InDegree[NodeNum] = Degree; + SU->Depth = 0; + + // Is it a node without dependencies? + if (Degree == 0) { + assert(SU->Preds.empty() && "SUnit should have no predecessors"); + // Collect leaf nodes + WorkList.push_back(SU); } -} + } -static void CalculateHeights(SUnit &SU, unsigned Height) { - if (SU.Height == 0 || Height > SU.Height) { - SU.Height = Height; - for (SUnit::pred_iterator I = SU.Preds.begin(), E = SU.Preds.end(); - I != E; ++I) - CalculateHeights(*I->first, Height+1); + // Process nodes in the topological order + while (!WorkList.empty()) { + SUnit *SU = WorkList.back(); + WorkList.pop_back(); + unsigned &SUDepth = SU->Depth; + + // Use dynamic programming: + // When current node is being processed, all of its dependencies + // are already processed. + // So, just iterate over all predecessors and take the longest path + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + unsigned PredDepth = I->Dep->Depth; + if (PredDepth+1 > SUDepth) { + SUDepth = PredDepth + 1; + } + } + + // Update InDegrees of all nodes depending on current SUnit + for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) { + SUnit *SU = I->Dep; + if (!--InDegree[SU->NodeNum]) + // If all dependencies of the node are processed already, + // then the longest path for the node can be computed now + WorkList.push_back(SU); + } } } + +/// CalculateHeights - compute heights using algorithms for the longest +/// paths in the DAG void ScheduleDAG::CalculateHeights() { - SUnit *Root = SUnitMap[DAG.getRoot().Val]; - ::CalculateHeights(*Root, 0U); + unsigned DAGSize = SUnits.size(); + std::vector InDegree(DAGSize); + std::vector WorkList; + WorkList.reserve(DAGSize); + + // Initialize the data structures + for (unsigned i = 0, e = DAGSize; i != e; ++i) { + SUnit *SU = &SUnits[i]; + int NodeNum = SU->NodeNum; + unsigned Degree = SU->Succs.size(); + InDegree[NodeNum] = Degree; + SU->Height = 0; + + // Is it a node without dependencies? + if (Degree == 0) { + assert(SU->Succs.empty() && "Something wrong"); + assert(WorkList.empty() && "Should be empty"); + // Collect leaf nodes + WorkList.push_back(SU); + } + } + + // Process nodes in the topological order + while (!WorkList.empty()) { + SUnit *SU = WorkList.back(); + WorkList.pop_back(); + unsigned &SUHeight = SU->Height; + + // Use dynamic programming: + // When current node is being processed, all of its dependencies + // are already processed. + // So, just iterate over all successors and take the longest path + for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); + I != E; ++I) { + unsigned SuccHeight = I->Dep->Height; + if (SuccHeight+1 > SUHeight) { + SUHeight = SuccHeight + 1; + } + } + + // Update InDegrees of all nodes depending on current SUnit + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + SUnit *SU = I->Dep; + if (!--InDegree[SU->NodeNum]) + // If all dependencies of the node are processed already, + // then the longest path for the node can be computed now + WorkList.push_back(SU); + } + } } /// CountResults - The results of target nodes have register or immediate /// operands first, then an optional chain, and optional flag operands (which do -/// not go into the machine instrs.) +/// not go into the resulting MachineInstr). unsigned ScheduleDAG::CountResults(SDNode *Node) { unsigned N = Node->getNumValues(); while (N && Node->getValueType(N - 1) == MVT::Flag) @@ -222,10 +368,20 @@ unsigned ScheduleDAG::CountResults(SDNode *Node) { return N; } -/// CountOperands The inputs to target nodes have any actual inputs first, -/// followed by an optional chain operand, then flag operands. Compute the -/// number of actual operands that will go into the machine instr. +/// CountOperands - The inputs to target nodes have any actual inputs first, +/// followed by special operands that describe memory references, then an +/// optional chain operand, then flag operands. Compute the number of +/// actual operands that will go into the resulting MachineInstr. unsigned ScheduleDAG::CountOperands(SDNode *Node) { + unsigned N = ComputeMemOperandsEnd(Node); + while (N && isa(Node->getOperand(N - 1).Val)) + --N; // Ignore MEMOPERAND nodes + return N; +} + +/// ComputeMemOperandsEnd - Find the index one past the last MemOperandSDNode +/// operand +unsigned ScheduleDAG::ComputeMemOperandsEnd(SDNode *Node) { unsigned N = Node->getNumOperands(); while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag) --N; @@ -235,44 +391,171 @@ unsigned ScheduleDAG::CountOperands(SDNode *Node) { } static const TargetRegisterClass *getInstrOperandRegClass( - const MRegisterInfo *MRI, + const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, - const TargetInstrDescriptor *II, + const TargetInstrDesc &II, unsigned Op) { - if (Op >= II->numOperands) { - assert((II->Flags & M_VARIABLE_OPS)&& "Invalid operand # of instruction"); + if (Op >= II.getNumOperands()) { + assert(II.isVariadic() && "Invalid operand # of instruction"); return NULL; } - const TargetOperandInfo &toi = II->OpInfo[Op]; - return (toi.Flags & M_LOOK_UP_PTR_REG_CLASS) - ? TII->getPointerRegClass() : MRI->getRegClass(toi.RegClass); + if (II.OpInfo[Op].isLookupPtrRegClass()) + return TII->getPointerRegClass(); + return TRI->getRegClass(II.OpInfo[Op].RegClass); } -static unsigned CreateVirtualRegisters(const MRegisterInfo *MRI, - MachineInstr *MI, - unsigned NumResults, - SSARegMap *RegMap, - const TargetInstrInfo *TII, - const TargetInstrDescriptor &II) { - // Create the result registers for this node and add the result regs to - // the machine instruction. - unsigned ResultReg = - RegMap->createVirtualRegister(getInstrOperandRegClass(MRI, TII, &II, 0)); - MI->addRegOperand(ResultReg, true); - for (unsigned i = 1; i != NumResults; ++i) { - const TargetRegisterClass *RC = getInstrOperandRegClass(MRI, TII, &II, i); - assert(RC && "Isn't a register operand!"); - MI->addRegOperand(RegMap->createVirtualRegister(RC), true); +void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo, + bool IsClone, unsigned SrcReg, + DenseMap &VRBaseMap) { + unsigned VRBase = 0; + if (TargetRegisterInfo::isVirtualRegister(SrcReg)) { + // Just use the input register directly! + if (IsClone) + VRBaseMap.erase(SDOperand(Node, ResNo)); + bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,ResNo),SrcReg)); + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); + return; + } + + // If the node is only used by a CopyToReg and the dest reg is a vreg, use + // the CopyToReg'd destination register instead of creating a new vreg. + bool MatchReg = true; + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *Use = UI->getUser(); + bool Match = true; + if (Use->getOpcode() == ISD::CopyToReg && + Use->getOperand(2).Val == Node && + Use->getOperand(2).ResNo == ResNo) { + unsigned DestReg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(DestReg)) { + VRBase = DestReg; + Match = false; + } else if (DestReg != SrcReg) + Match = false; + } else { + for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) { + SDOperand Op = Use->getOperand(i); + if (Op.Val != Node || Op.ResNo != ResNo) + continue; + MVT VT = Node->getValueType(Op.ResNo); + if (VT != MVT::Other && VT != MVT::Flag) + Match = false; + } + } + MatchReg &= Match; + if (VRBase) + break; + } + + const TargetRegisterClass *SrcRC = 0, *DstRC = 0; + SrcRC = TRI->getPhysicalRegisterRegClass(SrcReg, Node->getValueType(ResNo)); + + // Figure out the register class to create for the destreg. + if (VRBase) { + DstRC = MRI.getRegClass(VRBase); + } else { + DstRC = TLI->getRegClassFor(Node->getValueType(ResNo)); + } + + // If all uses are reading from the src physical register and copying the + // register is either impossible or very expensive, then don't create a copy. + if (MatchReg && SrcRC->getCopyCost() < 0) { + VRBase = SrcReg; + } else { + // Create the reg, emit the copy. + VRBase = MRI.createVirtualRegister(DstRC); + TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC); + } + + if (IsClone) + VRBaseMap.erase(SDOperand(Node, ResNo)); + bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,ResNo), VRBase)); + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); +} + +/// getDstOfCopyToRegUse - If the only use of the specified result number of +/// node is a CopyToReg, return its destination register. Return 0 otherwise. +unsigned ScheduleDAG::getDstOfOnlyCopyToRegUse(SDNode *Node, + unsigned ResNo) const { + if (!Node->hasOneUse()) + return 0; + + SDNode *Use = Node->use_begin()->getUser(); + if (Use->getOpcode() == ISD::CopyToReg && + Use->getOperand(2).Val == Node && + Use->getOperand(2).ResNo == ResNo) { + unsigned Reg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return Reg; + } + return 0; +} + +void ScheduleDAG::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, + const TargetInstrDesc &II, + DenseMap &VRBaseMap) { + assert(Node->getTargetOpcode() != TargetInstrInfo::IMPLICIT_DEF && + "IMPLICIT_DEF should have been handled as a special case elsewhere!"); + + for (unsigned i = 0; i < II.getNumDefs(); ++i) { + // If the specific node value is only used by a CopyToReg and the dest reg + // is a vreg, use the CopyToReg'd destination register instead of creating + // a new vreg. + unsigned VRBase = 0; + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *Use = UI->getUser(); + if (Use->getOpcode() == ISD::CopyToReg && + Use->getOperand(2).Val == Node && + Use->getOperand(2).ResNo == i) { + unsigned Reg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + VRBase = Reg; + MI->addOperand(MachineOperand::CreateReg(Reg, true)); + break; + } + } + } + + // Create the result registers for this node and add the result regs to + // the machine instruction. + if (VRBase == 0) { + const TargetRegisterClass *RC = getInstrOperandRegClass(TRI, TII, II, i); + assert(RC && "Isn't a register operand!"); + VRBase = MRI.createVirtualRegister(RC); + MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + } + + bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,i), VRBase)); + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); } - return ResultReg; } /// getVR - Return the virtual register corresponding to the specified result /// of the specified node. -static unsigned getVR(SDOperand Op, DenseMap &VRBaseMap) { - DenseMap::iterator I = VRBaseMap.find(Op.Val); +unsigned ScheduleDAG::getVR(SDOperand Op, + DenseMap &VRBaseMap) { + if (Op.isTargetOpcode() && + Op.getTargetOpcode() == TargetInstrInfo::IMPLICIT_DEF) { + // Add an IMPLICIT_DEF instruction before every use. + unsigned VReg = getDstOfOnlyCopyToRegUse(Op.Val, Op.ResNo); + // IMPLICIT_DEF can produce any type of result so its TargetInstrDesc + // does not include operand register class info. + if (!VReg) { + const TargetRegisterClass *RC = TLI->getRegClassFor(Op.getValueType()); + VReg = MRI.createVirtualRegister(RC); + } + BuildMI(BB, TII->get(TargetInstrInfo::IMPLICIT_DEF), VReg); + return VReg; + } + + DenseMap::iterator I = VRBaseMap.find(Op); assert(I != VRBaseMap.end() && "Node emitted out of order - late"); - return I->second + Op.ResNo; + return I->second; } @@ -282,8 +565,8 @@ static unsigned getVR(SDOperand Op, DenseMap &VRBaseMap) { /// assertions only. void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, unsigned IIOpNum, - const TargetInstrDescriptor *II, - DenseMap &VRBaseMap) { + const TargetInstrDesc *II, + DenseMap &VRBaseMap) { if (Op.isTargetOpcode()) { // Note that this case is redundant with the final else block, but we // include it because it is the most common and it makes the logic @@ -291,40 +574,55 @@ void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); - // Get/emit the operand. unsigned VReg = getVR(Op, VRBaseMap); - MI->addRegOperand(VReg, false); + const TargetInstrDesc &TID = MI->getDesc(); + bool isOptDef = IIOpNum < TID.getNumOperands() && + TID.OpInfo[IIOpNum].isOptionalDef(); + MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef)); // Verify that it is right. - assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); + assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); +#ifndef NDEBUG if (II) { + // There may be no register class for this operand if it is a variadic + // argument (RC will be NULL in this case). In this case, we just assume + // the regclass is ok. const TargetRegisterClass *RC = - getInstrOperandRegClass(MRI, TII, II, IIOpNum); - assert(RC && "Don't have operand info for this instruction!"); - assert(RegMap->getRegClass(VReg) == RC && - "Register class of operand and regclass of use don't agree!"); - } - } else if (ConstantSDNode *C = - dyn_cast(Op)) { - MI->addImmOperand(C->getValue()); - } else if (RegisterSDNode *R = - dyn_cast(Op)) { - MI->addRegOperand(R->getReg(), false); - } else if (GlobalAddressSDNode *TGA = - dyn_cast(Op)) { - MI->addGlobalAddressOperand(TGA->getGlobal(), TGA->getOffset()); - } else if (BasicBlockSDNode *BB = - dyn_cast(Op)) { - MI->addMachineBasicBlockOperand(BB->getBasicBlock()); - } else if (FrameIndexSDNode *FI = - dyn_cast(Op)) { - MI->addFrameIndexOperand(FI->getIndex()); - } else if (JumpTableSDNode *JT = - dyn_cast(Op)) { - MI->addJumpTableIndexOperand(JT->getIndex()); - } else if (ConstantPoolSDNode *CP = - dyn_cast(Op)) { + getInstrOperandRegClass(TRI, TII, *II, IIOpNum); + assert((RC || II->isVariadic()) && "Expected reg class info!"); + const TargetRegisterClass *VRC = MRI.getRegClass(VReg); + if (RC && VRC != RC) { + cerr << "Register class of operand and regclass of use don't agree!\n"; + cerr << "Operand = " << IIOpNum << "\n"; + cerr << "Op->Val = "; Op.Val->dump(&DAG); cerr << "\n"; + cerr << "MI = "; MI->print(cerr); + cerr << "VReg = " << VReg << "\n"; + cerr << "VReg RegClass size = " << VRC->getSize() + << ", align = " << VRC->getAlignment() << "\n"; + cerr << "Expected RegClass size = " << RC->getSize() + << ", align = " << RC->getAlignment() << "\n"; + cerr << "Fatal error, aborting.\n"; + abort(); + } + } +#endif + } else if (ConstantSDNode *C = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateImm(C->getValue())); + } else if (ConstantFPSDNode *F = dyn_cast(Op)) { + ConstantFP *CFP = ConstantFP::get(F->getValueAPF()); + MI->addOperand(MachineOperand::CreateFPImm(CFP)); + } else if (RegisterSDNode *R = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateReg(R->getReg(), false)); + } else if (GlobalAddressSDNode *TGA = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateGA(TGA->getGlobal(),TGA->getOffset())); + } else if (BasicBlockSDNode *BB = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateMBB(BB->getBasicBlock())); + } else if (FrameIndexSDNode *FI = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateFI(FI->getIndex())); + } else if (JumpTableSDNode *JT = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateJTI(JT->getIndex())); + } else if (ConstantPoolSDNode *CP = dyn_cast(Op)) { int Offset = CP->getOffset(); unsigned Align = CP->getAlignment(); const Type *Type = CP->getType(); @@ -332,8 +630,8 @@ void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, if (Align == 0) { Align = TM.getTargetData()->getPreferredTypeAlignmentShift(Type); if (Align == 0) { - // Alignment of packed types. FIXME! - Align = TM.getTargetData()->getTypeSize(Type); + // Alignment of vector types. FIXME! + Align = TM.getTargetData()->getABITypeSize(Type); Align = Log2_64(Align); } } @@ -343,83 +641,201 @@ void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op, Idx = ConstPool->getConstantPoolIndex(CP->getMachineCPVal(), Align); else Idx = ConstPool->getConstantPoolIndex(CP->getConstVal(), Align); - MI->addConstantPoolIndexOperand(Idx, Offset); - } else if (ExternalSymbolSDNode *ES = - dyn_cast(Op)) { - MI->addExternalSymbolOperand(ES->getSymbol()); + MI->addOperand(MachineOperand::CreateCPI(Idx, Offset)); + } else if (ExternalSymbolSDNode *ES = dyn_cast(Op)) { + MI->addOperand(MachineOperand::CreateES(ES->getSymbol())); } else { assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); unsigned VReg = getVR(Op, VRBaseMap); - MI->addRegOperand(VReg, false); + MI->addOperand(MachineOperand::CreateReg(VReg, false)); - // Verify that it is right. - assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); - if (II) { - const TargetRegisterClass *RC = - getInstrOperandRegClass(MRI, TII, II, IIOpNum); - assert(RC && "Don't have operand info for this instruction!"); - assert(RegMap->getRegClass(VReg) == RC && - "Register class of operand and regclass of use don't agree!"); + // Verify that it is right. Note that the reg class of the physreg and the + // vreg don't necessarily need to match, but the target copy insertion has + // to be able to handle it. This handles things like copies from ST(0) to + // an FP vreg on x86. + assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); + if (II && !II->isVariadic()) { + assert(getInstrOperandRegClass(TRI, TII, *II, IIOpNum) && + "Don't have operand info for this instruction!"); } } } +void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) { + MI->addMemOperand(MO); +} -/// EmitNode - Generate machine code for an node and needed dependencies. +// Returns the Register Class of a subregister +static const TargetRegisterClass *getSubRegisterRegClass( + const TargetRegisterClass *TRC, + unsigned SubIdx) { + // Pick the register class of the subregister + TargetRegisterInfo::regclass_iterator I = + TRC->subregclasses_begin() + SubIdx-1; + assert(I < TRC->subregclasses_end() && + "Invalid subregister index for register class"); + return *I; +} + +static const TargetRegisterClass *getSuperregRegisterClass( + const TargetRegisterClass *TRC, + unsigned SubIdx, + MVT VT) { + // Pick the register class of the superegister for this type + for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(), + E = TRC->superregclasses_end(); I != E; ++I) + if ((*I)->hasType(VT) && getSubRegisterRegClass(*I, SubIdx) == TRC) + return *I; + assert(false && "Couldn't find the register class"); + return 0; +} + +/// EmitSubregNode - Generate machine code for subreg nodes. /// -void ScheduleDAG::EmitNode(SDNode *Node, - DenseMap &VRBaseMap) { - unsigned VRBase = 0; // First virtual register for node +void ScheduleDAG::EmitSubregNode(SDNode *Node, + DenseMap &VRBaseMap) { + unsigned VRBase = 0; + unsigned Opc = Node->getTargetOpcode(); + // If the node is only used by a CopyToReg and the dest reg is a vreg, use + // the CopyToReg'd destination register instead of creating a new vreg. + for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); + UI != E; ++UI) { + SDNode *Use = UI->getUser(); + if (Use->getOpcode() == ISD::CopyToReg && + Use->getOperand(2).Val == Node) { + unsigned DestReg = cast(Use->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(DestReg)) { + VRBase = DestReg; + break; + } + } + } + + if (Opc == TargetInstrInfo::EXTRACT_SUBREG) { + unsigned SubIdx = cast(Node->getOperand(1))->getValue(); + + // Create the extract_subreg machine instruction. + MachineInstr *MI = BuildMI(TII->get(TargetInstrInfo::EXTRACT_SUBREG)); + + // Figure out the register class to create for the destreg. + unsigned VReg = getVR(Node->getOperand(0), VRBaseMap); + const TargetRegisterClass *TRC = MRI.getRegClass(VReg); + const TargetRegisterClass *SRC = getSubRegisterRegClass(TRC, SubIdx); + + if (VRBase) { + // Grab the destination register +#ifndef NDEBUG + const TargetRegisterClass *DRC = MRI.getRegClass(VRBase); + assert(SRC && DRC && SRC == DRC && + "Source subregister and destination must have the same class"); +#endif + } else { + // Create the reg + assert(SRC && "Couldn't find source register class"); + VRBase = MRI.createVirtualRegister(SRC); + } + + // Add def, source, and subreg index + MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap); + MI->addOperand(MachineOperand::CreateImm(SubIdx)); + BB->push_back(MI); + } else if (Opc == TargetInstrInfo::INSERT_SUBREG || + Opc == TargetInstrInfo::SUBREG_TO_REG) { + SDOperand N0 = Node->getOperand(0); + SDOperand N1 = Node->getOperand(1); + SDOperand N2 = Node->getOperand(2); + unsigned SubReg = getVR(N1, VRBaseMap); + unsigned SubIdx = cast(N2)->getValue(); + + + // Figure out the register class to create for the destreg. + const TargetRegisterClass *TRC = 0; + if (VRBase) { + TRC = MRI.getRegClass(VRBase); + } else { + TRC = getSuperregRegisterClass(MRI.getRegClass(SubReg), SubIdx, + Node->getValueType(0)); + assert(TRC && "Couldn't determine register class for insert_subreg"); + VRBase = MRI.createVirtualRegister(TRC); // Create the reg + } + + // Create the insert_subreg or subreg_to_reg machine instruction. + MachineInstr *MI = BuildMI(TII->get(Opc)); + MI->addOperand(MachineOperand::CreateReg(VRBase, true)); + + // If creating a subreg_to_reg, then the first input operand + // is an implicit value immediate, otherwise it's a register + if (Opc == TargetInstrInfo::SUBREG_TO_REG) { + const ConstantSDNode *SD = cast(N0); + MI->addOperand(MachineOperand::CreateImm(SD->getValue())); + } else + AddOperand(MI, N0, 0, 0, VRBaseMap); + // Add the subregster being inserted + AddOperand(MI, N1, 0, 0, VRBaseMap); + MI->addOperand(MachineOperand::CreateImm(SubIdx)); + BB->push_back(MI); + } else + assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg"); + + bool isNew = VRBaseMap.insert(std::make_pair(SDOperand(Node,0), VRBase)); + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); +} + +/// EmitNode - Generate machine code for an node and needed dependencies. +/// +void ScheduleDAG::EmitNode(SDNode *Node, bool IsClone, + DenseMap &VRBaseMap) { // If machine instruction if (Node->isTargetOpcode()) { unsigned Opc = Node->getTargetOpcode(); - const TargetInstrDescriptor &II = TII->get(Opc); + + // Handle subreg insert/extract specially + if (Opc == TargetInstrInfo::EXTRACT_SUBREG || + Opc == TargetInstrInfo::INSERT_SUBREG || + Opc == TargetInstrInfo::SUBREG_TO_REG) { + EmitSubregNode(Node, VRBaseMap); + return; + } + if (Opc == TargetInstrInfo::IMPLICIT_DEF) + // We want a unique VR for each IMPLICIT_DEF use. + return; + + const TargetInstrDesc &II = TII->get(Opc); unsigned NumResults = CountResults(Node); unsigned NodeOperands = CountOperands(Node); - unsigned NumMIOperands = NodeOperands + NumResults; + unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node); + bool HasPhysRegOuts = (NumResults > II.getNumDefs()) && + II.getImplicitDefs() != 0; #ifndef NDEBUG - assert((unsigned(II.numOperands) == NumMIOperands || - (II.Flags & M_VARIABLE_OPS)) && + unsigned NumMIOperands = NodeOperands + NumResults; + assert((II.getNumOperands() == NumMIOperands || + HasPhysRegOuts || II.isVariadic()) && "#operands for dag node doesn't match .td file!"); #endif // Create the new machine instruction. - MachineInstr *MI = new MachineInstr(II); + MachineInstr *MI = BuildMI(II); // Add result register values for things that are defined by this // instruction. - - // If the node is only used by a CopyToReg and the dest reg is a vreg, use - // the CopyToReg'd destination register instead of creating a new vreg. - if (NumResults == 1) { - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *Use = *UI; - if (Use->getOpcode() == ISD::CopyToReg && - Use->getOperand(2).Val == Node) { - unsigned Reg = cast(Use->getOperand(1))->getReg(); - if (MRegisterInfo::isVirtualRegister(Reg)) { - VRBase = Reg; - MI->addRegOperand(Reg, true); - break; - } - } - } - } - - // Otherwise, create new virtual registers. - if (NumResults && VRBase == 0) - VRBase = CreateVirtualRegisters(MRI, MI, NumResults, RegMap, TII, II); + if (NumResults) + CreateVirtualRegisters(Node, MI, II, VRBaseMap); // Emit all of the actual operands of this instruction, adding them to the // instruction as appropriate. for (unsigned i = 0; i != NodeOperands; ++i) - AddOperand(MI, Node->getOperand(i), i+NumResults, &II, VRBaseMap); + AddOperand(MI, Node->getOperand(i), i+II.getNumDefs(), &II, VRBaseMap); + + // Emit all of the memory operands of this instruction + for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i) + AddMemOperand(MI, cast(Node->getOperand(i))->MO); // Commute node if it has been determined to be profitable. if (CommuteSet.count(Node)) { @@ -432,82 +848,71 @@ void ScheduleDAG::EmitNode(SDNode *Node, delete MI; MI = NewMI; } + ++NumCommutes; } } - // Now that we have emitted all operands, emit this instruction itself. - if ((II.Flags & M_USES_CUSTOM_DAG_SCHED_INSERTION) == 0) { - BB->insert(BB->end(), MI); - } else { - // Insert this instruction into the end of the basic block, potentially - // taking some custom action. - BB = DAG.getTargetLoweringInfo().InsertAtEndOfBasicBlock(MI, BB); + if (II.usesCustomDAGSchedInsertionHook()) + // Insert this instruction into the basic block using a target + // specific inserter which may returns a new basic block. + BB = TLI->EmitInstrWithCustomInserter(MI, BB); + else + BB->push_back(MI); + + // Additional results must be an physical register def. + if (HasPhysRegOuts) { + for (unsigned i = II.getNumDefs(); i < NumResults; ++i) { + unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()]; + if (Node->hasAnyUseOfValue(i)) + EmitCopyFromReg(Node, i, IsClone, Reg, VRBaseMap); + } } } else { switch (Node->getOpcode()) { default: #ifndef NDEBUG - Node->dump(); + Node->dump(&DAG); #endif assert(0 && "This target-independent node should have been selected!"); - case ISD::EntryToken: // fall thru - case ISD::TokenFactor: + break; + case ISD::EntryToken: + assert(0 && "EntryToken should have been excluded from the schedule!"); + break; + case ISD::TokenFactor: // fall thru case ISD::LABEL: + case ISD::DECLARE: + case ISD::SRCVALUE: break; case ISD::CopyToReg: { - unsigned InReg; - if (RegisterSDNode *R = dyn_cast(Node->getOperand(2))) - InReg = R->getReg(); + unsigned SrcReg; + SDOperand SrcVal = Node->getOperand(2); + if (RegisterSDNode *R = dyn_cast(SrcVal)) + SrcReg = R->getReg(); else - InReg = getVR(Node->getOperand(2), VRBaseMap); + SrcReg = getVR(SrcVal, VRBaseMap); + unsigned DestReg = cast(Node->getOperand(1))->getReg(); - if (InReg != DestReg) // Coalesced away the copy? - MRI->copyRegToReg(*BB, BB->end(), DestReg, InReg, - RegMap->getRegClass(InReg)); + if (SrcReg == DestReg) // Coalesced away the copy? Ignore. + break; + + const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0; + // Get the register classes of the src/dst. + if (TargetRegisterInfo::isVirtualRegister(SrcReg)) + SrcTRC = MRI.getRegClass(SrcReg); + else + SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType()); + + if (TargetRegisterInfo::isVirtualRegister(DestReg)) + DstTRC = MRI.getRegClass(DestReg); + else + DstTRC = TRI->getPhysicalRegisterRegClass(DestReg, + Node->getOperand(1).getValueType()); + TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC); break; } case ISD::CopyFromReg: { unsigned SrcReg = cast(Node->getOperand(1))->getReg(); - if (MRegisterInfo::isVirtualRegister(SrcReg)) { - VRBase = SrcReg; // Just use the input register directly! - break; - } - - // If the node is only used by a CopyToReg and the dest reg is a vreg, use - // the CopyToReg'd destination register instead of creating a new vreg. - for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end(); - UI != E; ++UI) { - SDNode *Use = *UI; - if (Use->getOpcode() == ISD::CopyToReg && - Use->getOperand(2).Val == Node) { - unsigned DestReg = cast(Use->getOperand(1))->getReg(); - if (MRegisterInfo::isVirtualRegister(DestReg)) { - VRBase = DestReg; - break; - } - } - } - - // Figure out the register class to create for the destreg. - const TargetRegisterClass *TRC = 0; - if (VRBase) { - TRC = RegMap->getRegClass(VRBase); - } else { - - // Pick the register class of the right type that contains this physreg. - for (MRegisterInfo::regclass_iterator I = MRI->regclass_begin(), - E = MRI->regclass_end(); I != E; ++I) - if ((*I)->hasType(Node->getValueType(0)) && - (*I)->contains(SrcReg)) { - TRC = *I; - break; - } - assert(TRC && "Couldn't find register class for reg copy!"); - - // Create the reg, emit the copy. - VRBase = RegMap->createVirtualRegister(TRC); - } - MRI->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, TRC); + EmitCopyFromReg(Node, 0, IsClone, SrcReg, VRBaseMap); break; } case ISD::INLINEASM: { @@ -516,20 +921,19 @@ void ScheduleDAG::EmitNode(SDNode *Node, --NumOps; // Ignore the flag operand. // Create the inline asm machine instruction. - MachineInstr *MI = - new MachineInstr(BB, TII->get(TargetInstrInfo::INLINEASM)); + MachineInstr *MI = BuildMI(TII->get(TargetInstrInfo::INLINEASM)); // Add the asm string as an external symbol operand. const char *AsmStr = cast(Node->getOperand(1))->getSymbol(); - MI->addExternalSymbolOperand(AsmStr); + MI->addOperand(MachineOperand::CreateES(AsmStr)); // Add all of the operand registers to the instruction. for (unsigned i = 2; i != NumOps;) { unsigned Flags = cast(Node->getOperand(i))->getValue(); unsigned NumVals = Flags >> 3; - MI->addImmOperand(Flags); + MI->addOperand(MachineOperand::CreateImm(Flags)); ++i; // Skip the ID value. switch (Flags & 7) { @@ -537,25 +941,29 @@ void ScheduleDAG::EmitNode(SDNode *Node, case 1: // Use of register. for (; NumVals; --NumVals, ++i) { unsigned Reg = cast(Node->getOperand(i))->getReg(); - MI->addRegOperand(Reg, false); + MI->addOperand(MachineOperand::CreateReg(Reg, false)); } break; case 2: // Def of register. for (; NumVals; --NumVals, ++i) { unsigned Reg = cast(Node->getOperand(i))->getReg(); - MI->addRegOperand(Reg, true); + MI->addOperand(MachineOperand::CreateReg(Reg, true)); } break; case 3: { // Immediate. - assert(NumVals == 1 && "Unknown immediate value!"); - if (ConstantSDNode *CS=dyn_cast(Node->getOperand(i))){ - MI->addImmOperand(CS->getValue()); - } else { - GlobalAddressSDNode *GA = - cast(Node->getOperand(i)); - MI->addGlobalAddressOperand(GA->getGlobal(), GA->getOffset()); + for (; NumVals; --NumVals, ++i) { + if (ConstantSDNode *CS = + dyn_cast(Node->getOperand(i))) { + MI->addOperand(MachineOperand::CreateImm(CS->getValue())); + } else if (GlobalAddressSDNode *GA = + dyn_cast(Node->getOperand(i))) { + MI->addOperand(MachineOperand::CreateGA(GA->getGlobal(), + GA->getOffset())); + } else { + BasicBlockSDNode *BB =cast(Node->getOperand(i)); + MI->addOperand(MachineOperand::CreateMBB(BB->getBasicBlock())); + } } - ++i; break; } case 4: // Addressing mode. @@ -566,46 +974,165 @@ void ScheduleDAG::EmitNode(SDNode *Node, break; } } + BB->push_back(MI); break; } } } - - assert(!VRBaseMap.count(Node) && "Node emitted out of order - early"); - VRBaseMap[Node] = VRBase; } void ScheduleDAG::EmitNoop() { TII->insertNoop(*BB, BB->end()); } +void ScheduleDAG::EmitCrossRCCopy(SUnit *SU, + DenseMap &VRBaseMap) { + for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end(); + I != E; ++I) { + if (I->isCtrl) continue; // ignore chain preds + if (!I->Dep->Node) { + // Copy to physical register. + DenseMap::iterator VRI = VRBaseMap.find(I->Dep); + assert(VRI != VRBaseMap.end() && "Node emitted out of order - late"); + // Find the destination physical register. + unsigned Reg = 0; + for (SUnit::const_succ_iterator II = SU->Succs.begin(), + EE = SU->Succs.end(); II != EE; ++II) { + if (I->Reg) { + Reg = I->Reg; + break; + } + } + assert(I->Reg && "Unknown physical register!"); + TII->copyRegToReg(*BB, BB->end(), Reg, VRI->second, + SU->CopyDstRC, SU->CopySrcRC); + } else { + // Copy from physical register. + assert(I->Reg && "Unknown physical register!"); + unsigned VRBase = MRI.createVirtualRegister(SU->CopyDstRC); + bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)); + isNew = isNew; // Silence compiler warning. + assert(isNew && "Node emitted out of order - early"); + TII->copyRegToReg(*BB, BB->end(), VRBase, I->Reg, + SU->CopyDstRC, SU->CopySrcRC); + } + break; + } +} + +/// EmitLiveInCopy - Emit a copy for a live in physical register. If the +/// physical register has only a single copy use, then coalesced the copy +/// if possible. +void ScheduleDAG::EmitLiveInCopy(MachineBasicBlock *MBB, + MachineBasicBlock::iterator &InsertPos, + unsigned VirtReg, unsigned PhysReg, + const TargetRegisterClass *RC, + DenseMap &CopyRegMap){ + unsigned NumUses = 0; + MachineInstr *UseMI = NULL; + for (MachineRegisterInfo::use_iterator UI = MRI.use_begin(VirtReg), + UE = MRI.use_end(); UI != UE; ++UI) { + UseMI = &*UI; + if (++NumUses > 1) + break; + } + + // If the number of uses is not one, or the use is not a move instruction, + // don't coalesce. Also, only coalesce away a virtual register to virtual + // register copy. + bool Coalesced = false; + unsigned SrcReg, DstReg; + if (NumUses == 1 && + TII->isMoveInstr(*UseMI, SrcReg, DstReg) && + TargetRegisterInfo::isVirtualRegister(DstReg)) { + VirtReg = DstReg; + Coalesced = true; + } + + // Now find an ideal location to insert the copy. + MachineBasicBlock::iterator Pos = InsertPos; + while (Pos != MBB->begin()) { + MachineInstr *PrevMI = prior(Pos); + DenseMap::iterator RI = CopyRegMap.find(PrevMI); + // copyRegToReg might emit multiple instructions to do a copy. + unsigned CopyDstReg = (RI == CopyRegMap.end()) ? 0 : RI->second; + if (CopyDstReg && !TRI->regsOverlap(CopyDstReg, PhysReg)) + // This is what the BB looks like right now: + // r1024 = mov r0 + // ... + // r1 = mov r1024 + // + // We want to insert "r1025 = mov r1". Inserting this copy below the + // move to r1024 makes it impossible for that move to be coalesced. + // + // r1025 = mov r1 + // r1024 = mov r0 + // ... + // r1 = mov 1024 + // r2 = mov 1025 + break; // Woot! Found a good location. + --Pos; + } + + TII->copyRegToReg(*MBB, Pos, VirtReg, PhysReg, RC, RC); + CopyRegMap.insert(std::make_pair(prior(Pos), VirtReg)); + if (Coalesced) { + if (&*InsertPos == UseMI) ++InsertPos; + MBB->erase(UseMI); + } +} + +/// EmitLiveInCopies - If this is the first basic block in the function, +/// and if it has live ins that need to be copied into vregs, emit the +/// copies into the top of the block. +void ScheduleDAG::EmitLiveInCopies(MachineBasicBlock *MBB) { + DenseMap CopyRegMap; + MachineBasicBlock::iterator InsertPos = MBB->begin(); + for (MachineRegisterInfo::livein_iterator LI = MRI.livein_begin(), + E = MRI.livein_end(); LI != E; ++LI) + if (LI->second) { + const TargetRegisterClass *RC = MRI.getRegClass(LI->second); + EmitLiveInCopy(MBB, InsertPos, LI->second, LI->first, RC, CopyRegMap); + } +} + /// EmitSchedule - Emit the machine code in scheduled order. void ScheduleDAG::EmitSchedule() { - // If this is the first basic block in the function, and if it has live ins - // that need to be copied into vregs, emit the copies into the top of the - // block before emitting the code for the block. - MachineFunction &MF = DAG.getMachineFunction(); - if (&MF.front() == BB && MF.livein_begin() != MF.livein_end()) { - for (MachineFunction::livein_iterator LI = MF.livein_begin(), - E = MF.livein_end(); LI != E; ++LI) - if (LI->second) - MRI->copyRegToReg(*MF.begin(), MF.begin()->end(), LI->second, - LI->first, RegMap->getRegClass(LI->second)); + bool isEntryBB = &MF->front() == BB; + + if (isEntryBB && !SchedLiveInCopies) { + // If this is the first basic block in the function, and if it has live ins + // that need to be copied into vregs, emit the copies into the top of the + // block before emitting the code for the block. + for (MachineRegisterInfo::livein_iterator LI = MRI.livein_begin(), + E = MRI.livein_end(); LI != E; ++LI) + if (LI->second) { + const TargetRegisterClass *RC = MRI.getRegClass(LI->second); + TII->copyRegToReg(*MF->begin(), MF->begin()->end(), LI->second, + LI->first, RC, RC); + } } - - + // Finally, emit the code for all of the scheduled instructions. - DenseMap VRBaseMap; + DenseMap VRBaseMap; + DenseMap CopyVRBaseMap; for (unsigned i = 0, e = Sequence.size(); i != e; i++) { - if (SUnit *SU = Sequence[i]) { - for (unsigned j = 0, ee = SU->FlaggedNodes.size(); j != ee; j++) - EmitNode(SU->FlaggedNodes[j], VRBaseMap); - EmitNode(SU->Node, VRBaseMap); - } else { + SUnit *SU = Sequence[i]; + if (!SU) { // Null SUnit* is a noop. EmitNoop(); + continue; } + for (unsigned j = 0, ee = SU->FlaggedNodes.size(); j != ee; ++j) + EmitNode(SU->FlaggedNodes[j], SU->OrigNode != SU, VRBaseMap); + if (!SU->Node) + EmitCrossRCCopy(SU, CopyVRBaseMap); + else + EmitNode(SU->Node, SU->OrigNode != SU, VRBaseMap); } + + if (isEntryBB && SchedLiveInCopies) + EmitLiveInCopies(MF->begin()); } /// dump - dump the schedule. @@ -622,11 +1149,6 @@ void ScheduleDAG::dumpSchedule() const { /// Run - perform scheduling. /// MachineBasicBlock *ScheduleDAG::Run() { - TII = TM.getInstrInfo(); - MRI = TM.getRegisterInfo(); - RegMap = BB->getParent()->getSSARegMap(); - ConstPool = BB->getParent()->getConstantPool(); - Schedule(); return BB; } @@ -635,7 +1157,10 @@ MachineBasicBlock *ScheduleDAG::Run() { /// a group of nodes flagged together. void SUnit::dump(const SelectionDAG *G) const { cerr << "SU(" << NodeNum << "): "; - Node->dump(G); + if (Node) + Node->dump(G); + else + cerr << "CROSS RC COPY "; cerr << "\n"; if (FlaggedNodes.size() != 0) { for (unsigned i = 0, e = FlaggedNodes.size(); i != e; i++) { @@ -651,8 +1176,6 @@ void SUnit::dumpAll(const SelectionDAG *G) const { cerr << " # preds left : " << NumPredsLeft << "\n"; cerr << " # succs left : " << NumSuccsLeft << "\n"; - cerr << " # chain preds left : " << NumChainPredsLeft << "\n"; - cerr << " # chain succs left : " << NumChainSuccsLeft << "\n"; cerr << " Latency : " << Latency << "\n"; cerr << " Depth : " << Depth << "\n"; cerr << " Height : " << Height << "\n"; @@ -661,22 +1184,28 @@ void SUnit::dumpAll(const SelectionDAG *G) const { cerr << " Predecessors:\n"; for (SUnit::const_succ_iterator I = Preds.begin(), E = Preds.end(); I != E; ++I) { - if (I->second) + if (I->isCtrl) cerr << " ch #"; else cerr << " val #"; - cerr << I->first << " - SU(" << I->first->NodeNum << ")\n"; + cerr << I->Dep << " - SU(" << I->Dep->NodeNum << ")"; + if (I->isSpecial) + cerr << " *"; + cerr << "\n"; } } if (Succs.size() != 0) { cerr << " Successors:\n"; for (SUnit::const_succ_iterator I = Succs.begin(), E = Succs.end(); I != E; ++I) { - if (I->second) + if (I->isCtrl) cerr << " ch #"; else cerr << " val #"; - cerr << I->first << " - SU(" << I->first->NodeNum << ")\n"; + cerr << I->Dep << " - SU(" << I->Dep->NodeNum << ")"; + if (I->isSpecial) + cerr << " *"; + cerr << "\n"; } } cerr << "\n";