//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "pre-RA-sched"
+#include "llvm/Constants.h"
#include "llvm/Type.h"
#include "llvm/CodeGen/ScheduleDAG.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetInstrInfo.h"
#include "llvm/Target/TargetLowering.h"
+#include "llvm/ADT/Statistic.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
using namespace llvm;
+STATISTIC(NumCommutes, "Number of instructions commuted");
+
ScheduleDAG::ScheduleDAG(SelectionDAG &dag, MachineBasicBlock *bb,
const TargetMachine &tm)
: DAG(dag), BB(bb), TM(tm), RegInfo(BB->getParent()->getRegInfo()) {
TII = TM.getInstrInfo();
- MRI = TM.getRegisterInfo();
+ MF = &DAG.getMachineFunction();
+ TRI = TM.getRegisterInfo();
ConstPool = BB->getParent()->getConstantPool();
}
/// a specified operand is a physical register dependency. If so, returns the
/// register and the cost of copying the register.
static void CheckForPhysRegDependency(SDNode *Def, SDNode *Use, unsigned Op,
- const MRegisterInfo *MRI,
+ const TargetRegisterInfo *TRI,
const TargetInstrInfo *TII,
unsigned &PhysReg, int &Cost) {
if (Op != 2 || Use->getOpcode() != ISD::CopyToReg)
return;
unsigned Reg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
- if (MRegisterInfo::isVirtualRegister(Reg))
+ if (TargetRegisterInfo::isVirtualRegister(Reg))
return;
unsigned ResNo = Use->getOperand(2).ResNo;
if (Def->isTargetOpcode()) {
- const TargetInstrDescriptor &II = TII->get(Def->getTargetOpcode());
+ const TargetInstrDesc &II = TII->get(Def->getTargetOpcode());
if (ResNo >= II.getNumDefs() &&
II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg) {
PhysReg = Reg;
const TargetRegisterClass *RC =
- MRI->getPhysicalRegisterRegClass(Def->getValueType(ResNo), Reg);
+ TRI->getPhysicalRegisterRegClass(Def->getValueType(ResNo), Reg);
Cost = RC->getCopyCost();
}
}
bool HasFlagUse = false;
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI)
- if (FlagVal.isOperand(*UI)) {
+ if (FlagVal.isOperandOf(*UI)) {
HasFlagUse = true;
NodeSUnit->FlaggedNodes.push_back(N);
SUnitMap[N].push_back(NodeSUnit);
if (MainNode->isTargetOpcode()) {
unsigned Opc = MainNode->getTargetOpcode();
- const TargetInstrDescriptor &TID = TII->get(Opc);
+ const TargetInstrDesc &TID = TII->get(Opc);
for (unsigned i = 0; i != TID.getNumOperands(); ++i) {
if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) {
SU->isTwoAddress = true;
break;
}
}
- if (TID.Flags & M_COMMUTABLE)
+ if (TID.isCommutable())
SU->isCommutable = true;
}
unsigned PhysReg = 0;
int Cost = 1;
// Determine if this is a physical register dependency.
- CheckForPhysRegDependency(OpN, N, i, MRI, TII, PhysReg, Cost);
+ CheckForPhysRegDependency(OpN, N, i, TRI, TII, PhysReg, Cost);
SU->addPred(OpSU, isChain, false, PhysReg, Cost);
}
}
}
}
+/// CalculateDepths - compute depths using algorithms for the longest
+/// paths in the DAG
void ScheduleDAG::CalculateDepths() {
- std::vector<std::pair<SUnit*, unsigned> > WorkList;
- for (unsigned i = 0, e = SUnits.size(); i != e; ++i)
- if (SUnits[i].Preds.size() == 0)
- WorkList.push_back(std::make_pair(&SUnits[i], 0U));
+ unsigned DAGSize = SUnits.size();
+ std::vector<unsigned> InDegree(DAGSize);
+ std::vector<SUnit*> WorkList;
+ WorkList.reserve(DAGSize);
+
+ // Initialize the data structures
+ for (unsigned i = 0, e = DAGSize; i != e; ++i) {
+ SUnit *SU = &SUnits[i];
+ int NodeNum = SU->NodeNum;
+ unsigned Degree = SU->Preds.size();
+ InDegree[NodeNum] = Degree;
+ SU->Depth = 0;
+
+ // Is it a node without dependencies?
+ if (Degree == 0) {
+ assert(SU->Preds.empty() && "SUnit should have no predecessors");
+ // Collect leaf nodes
+ WorkList.push_back(SU);
+ }
+ }
+ // Process nodes in the topological order
while (!WorkList.empty()) {
- SUnit *SU = WorkList.back().first;
- unsigned Depth = WorkList.back().second;
+ SUnit *SU = WorkList.back();
WorkList.pop_back();
- if (SU->Depth == 0 || Depth > SU->Depth) {
- SU->Depth = Depth;
- for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
- I != E; ++I)
- WorkList.push_back(std::make_pair(I->Dep, Depth+1));
+ unsigned &SUDepth = SU->Depth;
+
+ // Use dynamic programming:
+ // When current node is being processed, all of its dependencies
+ // are already processed.
+ // So, just iterate over all predecessors and take the longest path
+ for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ unsigned PredDepth = I->Dep->Depth;
+ if (PredDepth+1 > SUDepth) {
+ SUDepth = PredDepth + 1;
+ }
+ }
+
+ // Update InDegrees of all nodes depending on current SUnit
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I) {
+ SUnit *SU = I->Dep;
+ if (!--InDegree[SU->NodeNum])
+ // If all dependencies of the node are processed already,
+ // then the longest path for the node can be computed now
+ WorkList.push_back(SU);
}
}
}
+/// CalculateHeights - compute heights using algorithms for the longest
+/// paths in the DAG
void ScheduleDAG::CalculateHeights() {
- std::vector<std::pair<SUnit*, unsigned> > WorkList;
- SUnit *Root = SUnitMap[DAG.getRoot().Val].front();
- WorkList.push_back(std::make_pair(Root, 0U));
+ unsigned DAGSize = SUnits.size();
+ std::vector<unsigned> InDegree(DAGSize);
+ std::vector<SUnit*> WorkList;
+ WorkList.reserve(DAGSize);
+
+ // Initialize the data structures
+ for (unsigned i = 0, e = DAGSize; i != e; ++i) {
+ SUnit *SU = &SUnits[i];
+ int NodeNum = SU->NodeNum;
+ unsigned Degree = SU->Succs.size();
+ InDegree[NodeNum] = Degree;
+ SU->Height = 0;
+
+ // Is it a node without dependencies?
+ if (Degree == 0) {
+ assert(SU->Succs.empty() && "Something wrong");
+ assert(WorkList.empty() && "Should be empty");
+ // Collect leaf nodes
+ WorkList.push_back(SU);
+ }
+ }
+ // Process nodes in the topological order
while (!WorkList.empty()) {
- SUnit *SU = WorkList.back().first;
- unsigned Height = WorkList.back().second;
+ SUnit *SU = WorkList.back();
WorkList.pop_back();
- if (SU->Height == 0 || Height > SU->Height) {
- SU->Height = Height;
- for (SUnit::pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
- I != E; ++I)
- WorkList.push_back(std::make_pair(I->Dep, Height+1));
+ unsigned &SUHeight = SU->Height;
+
+ // Use dynamic programming:
+ // When current node is being processed, all of its dependencies
+ // are already processed.
+ // So, just iterate over all successors and take the longest path
+ for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+ I != E; ++I) {
+ unsigned SuccHeight = I->Dep->Height;
+ if (SuccHeight+1 > SUHeight) {
+ SUHeight = SuccHeight + 1;
+ }
+ }
+
+ // Update InDegrees of all nodes depending on current SUnit
+ for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+ I != E; ++I) {
+ SUnit *SU = I->Dep;
+ if (!--InDegree[SU->NodeNum])
+ // If all dependencies of the node are processed already,
+ // then the longest path for the node can be computed now
+ WorkList.push_back(SU);
}
}
}
/// CountResults - The results of target nodes have register or immediate
/// operands first, then an optional chain, and optional flag operands (which do
-/// not go into the machine instrs.)
+/// not go into the resulting MachineInstr).
unsigned ScheduleDAG::CountResults(SDNode *Node) {
unsigned N = Node->getNumValues();
while (N && Node->getValueType(N - 1) == MVT::Flag)
return N;
}
-/// CountOperands The inputs to target nodes have any actual inputs first,
-/// followed by an optional chain operand, then flag operands. Compute the
-/// number of actual operands that will go into the machine instr.
+/// CountOperands - The inputs to target nodes have any actual inputs first,
+/// followed by special operands that describe memory references, then an
+/// optional chain operand, then flag operands. Compute the number of
+/// actual operands that will go into the resulting MachineInstr.
unsigned ScheduleDAG::CountOperands(SDNode *Node) {
+ unsigned N = ComputeMemOperandsEnd(Node);
+ while (N && isa<MemOperandSDNode>(Node->getOperand(N - 1).Val))
+ --N; // Ignore MemOperand nodes
+ return N;
+}
+
+/// ComputeMemOperandsEnd - Find the index one past the last MemOperandSDNode
+/// operand
+unsigned ScheduleDAG::ComputeMemOperandsEnd(SDNode *Node) {
unsigned N = Node->getNumOperands();
while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag)
--N;
}
static const TargetRegisterClass *getInstrOperandRegClass(
- const MRegisterInfo *MRI,
+ const TargetRegisterInfo *TRI,
const TargetInstrInfo *TII,
- const TargetInstrDescriptor *II,
+ const TargetInstrDesc &II,
unsigned Op) {
- if (Op >= II->getNumOperands()) {
- assert(II->isVariadic() && "Invalid operand # of instruction");
+ if (Op >= II.getNumOperands()) {
+ assert(II.isVariadic() && "Invalid operand # of instruction");
return NULL;
}
- if (II->OpInfo[Op].isLookupPtrRegClass())
+ if (II.OpInfo[Op].isLookupPtrRegClass())
return TII->getPointerRegClass();
- return MRI->getRegClass(II->OpInfo[Op].RegClass);
+ return TRI->getRegClass(II.OpInfo[Op].RegClass);
}
void ScheduleDAG::EmitCopyFromReg(SDNode *Node, unsigned ResNo,
unsigned InstanceNo, unsigned SrcReg,
DenseMap<SDOperand, unsigned> &VRBaseMap) {
unsigned VRBase = 0;
- if (MRegisterInfo::isVirtualRegister(SrcReg)) {
+ if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
// Just use the input register directly!
if (InstanceNo > 0)
VRBaseMap.erase(SDOperand(Node, ResNo));
Use->getOperand(2).Val == Node &&
Use->getOperand(2).ResNo == ResNo) {
unsigned DestReg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
- if (MRegisterInfo::isVirtualRegister(DestReg)) {
+ if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
VRBase = DestReg;
Match = false;
} else if (DestReg != SrcReg)
break;
}
- const TargetRegisterClass *TRC = 0;
+ const TargetRegisterClass *SrcRC = 0, *DstRC = 0;
+ SrcRC = TRI->getPhysicalRegisterRegClass(Node->getValueType(ResNo), SrcReg);
+
// Figure out the register class to create for the destreg.
- if (VRBase)
- TRC = RegInfo.getRegClass(VRBase);
- else
- TRC = MRI->getPhysicalRegisterRegClass(Node->getValueType(ResNo), SrcReg);
+ if (VRBase) {
+ DstRC = RegInfo.getRegClass(VRBase);
+ } else {
+ DstRC = DAG.getTargetLoweringInfo()
+ .getRegClassFor(Node->getValueType(ResNo));
+ }
// If all uses are reading from the src physical register and copying the
// register is either impossible or very expensive, then don't create a copy.
- if (MatchReg && TRC->getCopyCost() < 0) {
+ if (MatchReg && SrcRC->getCopyCost() < 0) {
VRBase = SrcReg;
} else {
// Create the reg, emit the copy.
- VRBase = RegInfo.createVirtualRegister(TRC);
- TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, TRC, TRC);
+ VRBase = RegInfo.createVirtualRegister(DstRC);
+ TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC);
}
if (InstanceNo > 0)
void ScheduleDAG::CreateVirtualRegisters(SDNode *Node,
MachineInstr *MI,
- const TargetInstrDescriptor &II,
+ const TargetInstrDesc &II,
DenseMap<SDOperand, unsigned> &VRBaseMap) {
for (unsigned i = 0; i < II.getNumDefs(); ++i) {
// If the specific node value is only used by a CopyToReg and the dest reg
Use->getOperand(2).Val == Node &&
Use->getOperand(2).ResNo == i) {
unsigned Reg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
- if (MRegisterInfo::isVirtualRegister(Reg)) {
+ if (TargetRegisterInfo::isVirtualRegister(Reg)) {
VRBase = Reg;
MI->addOperand(MachineOperand::CreateReg(Reg, true));
break;
// Create the result registers for this node and add the result regs to
// the machine instruction.
if (VRBase == 0) {
- const TargetRegisterClass *RC = getInstrOperandRegClass(MRI, TII, &II, i);
+ const TargetRegisterClass *RC = getInstrOperandRegClass(TRI, TII, II, i);
assert(RC && "Isn't a register operand!");
VRBase = RegInfo.createVirtualRegister(RC);
MI->addOperand(MachineOperand::CreateReg(VRBase, true));
/// assertions only.
void ScheduleDAG::AddOperand(MachineInstr *MI, SDOperand Op,
unsigned IIOpNum,
- const TargetInstrDescriptor *II,
+ const TargetInstrDesc *II,
DenseMap<SDOperand, unsigned> &VRBaseMap) {
if (Op.isTargetOpcode()) {
// Note that this case is redundant with the final else block, but we
// Get/emit the operand.
unsigned VReg = getVR(Op, VRBaseMap);
- const TargetInstrDescriptor *TID = MI->getDesc();
- bool isOptDef = (IIOpNum < TID->getNumOperands())
- ? (TID->OpInfo[IIOpNum].isOptionalDef()) : false;
+ const TargetInstrDesc &TID = MI->getDesc();
+ bool isOptDef = (IIOpNum < TID.getNumOperands())
+ ? (TID.OpInfo[IIOpNum].isOptionalDef()) : false;
MI->addOperand(MachineOperand::CreateReg(VReg, isOptDef));
// Verify that it is right.
- assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
+ assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
if (II) {
const TargetRegisterClass *RC =
- getInstrOperandRegClass(MRI, TII, II, IIOpNum);
+ getInstrOperandRegClass(TRI, TII, *II, IIOpNum);
assert(RC && "Don't have operand info for this instruction!");
const TargetRegisterClass *VRC = RegInfo.getRegClass(VReg);
if (VRC != RC) {
}
} else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
MI->addOperand(MachineOperand::CreateImm(C->getValue()));
+ } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) {
+ const Type *FType = MVT::getTypeForValueType(Op.getValueType());
+ ConstantFP *CFP = ConstantFP::get(FType, F->getValueAPF());
+ MI->addOperand(MachineOperand::CreateFPImm(CFP));
} else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) {
MI->addOperand(MachineOperand::CreateReg(R->getReg(), false));
} else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) {
unsigned VReg = getVR(Op, VRBaseMap);
MI->addOperand(MachineOperand::CreateReg(VReg, false));
- // Verify that it is right.
- assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
+ // Verify that it is right. Note that the reg class of the physreg and the
+ // vreg don't necessarily need to match, but the target copy insertion has
+ // to be able to handle it. This handles things like copies from ST(0) to
+ // an FP vreg on x86.
+ assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
if (II) {
- const TargetRegisterClass *RC =
- getInstrOperandRegClass(MRI, TII, II, IIOpNum);
- assert(RC && "Don't have operand info for this instruction!");
- assert(RegInfo.getRegClass(VReg) == RC &&
- "Register class of operand and regclass of use don't agree!");
+ assert(getInstrOperandRegClass(TRI, TII, *II, IIOpNum) &&
+ "Don't have operand info for this instruction!");
}
}
}
+void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MemOperand &MO) {
+ MI->addMemOperand(MO);
+}
+
// Returns the Register Class of a subregister
static const TargetRegisterClass *getSubRegisterRegClass(
const TargetRegisterClass *TRC,
unsigned SubIdx) {
// Pick the register class of the subregister
- MRegisterInfo::regclass_iterator I = TRC->subregclasses_begin() + SubIdx-1;
+ TargetRegisterInfo::regclass_iterator I =
+ TRC->subregclasses_begin() + SubIdx-1;
assert(I < TRC->subregclasses_end() &&
"Invalid subregister index for register class");
return *I;
unsigned SubIdx,
MVT::ValueType VT) {
// Pick the register class of the superegister for this type
- for (MRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(),
+ for (TargetRegisterInfo::regclass_iterator I = TRC->superregclasses_begin(),
E = TRC->superregclasses_end(); I != E; ++I)
if ((*I)->hasType(VT) && getSubRegisterRegClass(*I, SubIdx) == TRC)
return *I;
if (Use->getOpcode() == ISD::CopyToReg &&
Use->getOperand(2).Val == Node) {
unsigned DestReg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
- if (MRegisterInfo::isVirtualRegister(DestReg)) {
+ if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
VRBase = DestReg;
break;
}
if (VRBase) {
// Grab the destination register
- const TargetRegisterClass *DRC = 0;
- DRC = RegInfo.getRegClass(VRBase);
- assert(SRC == DRC &&
+ const TargetRegisterClass *DRC = RegInfo.getRegClass(VRBase);
+ assert(SRC && DRC && SRC == DRC &&
"Source subregister and destination must have the same class");
} else {
// Create the reg
+ assert(SRC && "Couldn't find source register class");
VRBase = RegInfo.createVirtualRegister(SRC);
}
if (Use->getOpcode() == ISD::CopyToReg &&
Use->getOperand(2).Val == Node) {
unsigned DestReg = cast<RegisterSDNode>(Use->getOperand(1))->getReg();
- if (MRegisterInfo::isVirtualRegister(DestReg)) {
+ if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
VRBase = DestReg;
break;
}
return;
}
- const TargetInstrDescriptor &II = TII->get(Opc);
+ const TargetInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NodeOperands = CountOperands(Node);
+ unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node);
unsigned NumMIOperands = NodeOperands + NumResults;
bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
II.getImplicitDefs() != 0;
for (unsigned i = 0; i != NodeOperands; ++i)
AddOperand(MI, Node->getOperand(i), i+II.getNumDefs(), &II, VRBaseMap);
+ // Emit all of the memory operands of this instruction
+ for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i)
+ AddMemOperand(MI, cast<MemOperandSDNode>(Node->getOperand(i))->MO);
+
// Commute node if it has been determined to be profitable.
if (CommuteSet.count(Node)) {
MachineInstr *NewMI = TII->commuteInstruction(MI);
delete MI;
MI = NewMI;
}
+ ++NumCommutes;
}
}
- // Now that we have emitted all operands, emit this instruction itself.
- if (!II.usesCustomDAGSchedInsertionHook()) {
- BB->insert(BB->end(), MI);
- } else {
- // Insert this instruction into the end of the basic block, potentially
- // taking some custom action.
- BB = DAG.getTargetLoweringInfo().InsertAtEndOfBasicBlock(MI, BB);
- }
+ if (II.usesCustomDAGSchedInsertionHook())
+ // Insert this instruction into the basic block using a target
+ // specific inserter which may returns a new basic block.
+ BB = DAG.getTargetLoweringInfo().EmitInstrWithCustomInserter(MI, BB);
+ else
+ BB->push_back(MI);
// Additional results must be an physical register def.
if (HasPhysRegOuts) {
case ISD::EntryToken: // fall thru
case ISD::TokenFactor:
case ISD::LABEL:
+ case ISD::DECLARE:
+ case ISD::SRCVALUE:
break;
case ISD::CopyToReg: {
unsigned InReg;
if (InReg != DestReg) {// Coalesced away the copy?
const TargetRegisterClass *TRC = 0;
// Get the target register class
- if (MRegisterInfo::isVirtualRegister(InReg))
+ if (TargetRegisterInfo::isVirtualRegister(InReg))
TRC = RegInfo.getRegClass(InReg);
else
TRC =
- MRI->getPhysicalRegisterRegClass(Node->getOperand(2).getValueType(),
+ TRI->getPhysicalRegisterRegClass(Node->getOperand(2).getValueType(),
InReg);
TII->copyRegToReg(*BB, BB->end(), DestReg, InReg, TRC, TRC);
}
TII->insertNoop(*BB, BB->end());
}
-void ScheduleDAG::EmitCrossRCCopy(SUnit *SU, DenseMap<SUnit*, unsigned> &VRBaseMap) {
+void ScheduleDAG::EmitCrossRCCopy(SUnit *SU,
+ DenseMap<SUnit*, unsigned> &VRBaseMap) {
for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
I != E; ++I) {
if (I->isCtrl) continue; // ignore chain preds
// If this is the first basic block in the function, and if it has live ins
// that need to be copied into vregs, emit the copies into the top of the
// block before emitting the code for the block.
- MachineFunction &MF = DAG.getMachineFunction();
- if (&MF.front() == BB) {
+ if (&MF->front() == BB) {
for (MachineRegisterInfo::livein_iterator LI = RegInfo.livein_begin(),
E = RegInfo.livein_end(); LI != E; ++LI)
if (LI->second) {
const TargetRegisterClass *RC = RegInfo.getRegClass(LI->second);
- TII->copyRegToReg(*MF.begin(), MF.begin()->end(), LI->second,
+ TII->copyRegToReg(*MF->begin(), MF->begin()->end(), LI->second,
LI->first, RC, RC);
}
}
projects / oota-llvm.git / blobdiff