#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/DwarfWriter.h"
+#include "llvm/Analysis/DebugInfo.h"
#include "llvm/CodeGen/PseudoSourceValue.h"
#include "llvm/Target/TargetFrameInfo.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/CallingConv.h"
#include "llvm/Constants.h"
#include "llvm/DerivedTypes.h"
+#include "llvm/GlobalVariable.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MathExtras.h"
class VISIBILITY_HIDDEN SelectionDAGLegalize {
TargetLowering &TLI;
SelectionDAG &DAG;
+ bool TypesNeedLegalizing;
// Libcall insertion helpers.
DenseMap<SDValue, SDValue> PromotedNodes;
/// ExpandedNodes - For nodes that need to be expanded this map indicates
- /// which which operands are the expanded version of the input. This allows
+ /// which operands are the expanded version of the input. This allows
/// us to avoid expanding the same node more than once.
DenseMap<SDValue, std::pair<SDValue, SDValue> > ExpandedNodes;
/// SplitNodes - For vector nodes that need to be split, this map indicates
- /// which which operands are the split version of the input. This allows us
+ /// which operands are the split version of the input. This allows us
/// to avoid splitting the same node more than once.
std::map<SDValue, std::pair<SDValue, SDValue> > SplitNodes;
/// processed to the result.
std::map<SDValue, SDValue> ScalarizedNodes;
+ /// WidenNodes - For nodes that need to be widened from one vector type to
+ /// another, this contains the mapping of those that we have already widen.
+ /// This allows us to avoid widening more than once.
+ std::map<SDValue, SDValue> WidenNodes;
+
void AddLegalizedOperand(SDValue From, SDValue To) {
LegalizedNodes.insert(std::make_pair(From, To));
// If someone requests legalization of the new node, return itself.
void AddPromotedOperand(SDValue From, SDValue To) {
bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second;
assert(isNew && "Got into the map somehow?");
+ isNew = isNew;
+ // If someone requests legalization of the new node, return itself.
+ LegalizedNodes.insert(std::make_pair(To, To));
+ }
+ void AddWidenedOperand(SDValue From, SDValue To) {
+ bool isNew = WidenNodes.insert(std::make_pair(From, To)).second;
+ assert(isNew && "Got into the map somehow?");
+ isNew = isNew;
// If someone requests legalization of the new node, return itself.
LegalizedNodes.insert(std::make_pair(To, To));
}
public:
- explicit SelectionDAGLegalize(SelectionDAG &DAG);
+ explicit SelectionDAGLegalize(SelectionDAG &DAG, bool TypesNeedLegalizing);
/// getTypeAction - Return how we should legalize values of this type, either
/// it is already legal or we need to expand it into multiple registers of
/// ExpandOp - Expand the specified SDValue into its two component pieces
/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this,
- /// the LegalizeNodes map is filled in for any results that are not expanded,
+ /// the LegalizedNodes map is filled in for any results that are not expanded,
/// the ExpandedNodes map is filled in for any results that are expanded, and
/// the Lo/Hi values are returned. This applies to integer types and Vector
/// types.
void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi);
+ /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT
+ /// (e.g., v3i32 to v4i32). The produced value will have the correct value
+ /// for the existing elements but no guarantee is made about the new elements
+ /// at the end of the vector: it may be zero, ones, or garbage. This is useful
+ /// when we have an instruction operating on an illegal vector type and we
+ /// want to widen it to do the computation on a legal wider vector type.
+ SDValue WidenVectorOp(SDValue Op, MVT WidenVT);
+
/// SplitVectorOp - Given an operand of vector type, break it down into
/// two smaller values.
void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi);
/// scalar (e.g. f32) value.
SDValue ScalarizeVectorOp(SDValue O);
+ /// Useful 16 element vector type that is used to pass operands for widening.
+ typedef SmallVector<SDValue, 16> SDValueVector;
+
+ /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if
+ /// the LdChain contains a single load and false if it contains a token
+ /// factor for multiple loads. It takes
+ /// Result: location to return the result
+ /// LdChain: location to return the load chain
+ /// Op: load operation to widen
+ /// NVT: widen vector result type we want for the load
+ bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain,
+ SDValue Op, MVT NVT);
+
+ /// Helper genWidenVectorLoads - Helper function to generate a set of
+ /// loads to load a vector with a resulting wider type. It takes
+ /// LdChain: list of chains for the load we have generated
+ /// Chain: incoming chain for the ld vector
+ /// BasePtr: base pointer to load from
+ /// SV: memory disambiguation source value
+ /// SVOffset: memory disambiugation offset
+ /// Alignment: alignment of the memory
+ /// isVolatile: volatile load
+ /// LdWidth: width of memory that we want to load
+ /// ResType: the wider result result type for the resulting loaded vector
+ SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain,
+ SDValue BasePtr, const Value *SV,
+ int SVOffset, unsigned Alignment,
+ bool isVolatile, unsigned LdWidth,
+ MVT ResType);
+
+ /// StoreWidenVectorOp - Stores a widen vector into non widen memory
+ /// location. It takes
+ /// ST: store node that we want to replace
+ /// Chain: incoming store chain
+ /// BasePtr: base address of where we want to store into
+ SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain,
+ SDValue BasePtr);
+
+ /// Helper genWidenVectorStores - Helper function to generate a set of
+ /// stores to store a widen vector into non widen memory
+ // It takes
+ // StChain: list of chains for the stores we have generated
+ // Chain: incoming chain for the ld vector
+ // BasePtr: base pointer to load from
+ // SV: memory disambiguation source value
+ // SVOffset: memory disambiugation offset
+ // Alignment: alignment of the memory
+ // isVolatile: volatile lod
+ // ValOp: value to store
+ // StWidth: width of memory that we want to store
+ void genWidenVectorStores(SDValueVector& StChain, SDValue Chain,
+ SDValue BasePtr, const Value *SV,
+ int SVOffset, unsigned Alignment,
+ bool isVolatile, SDValue ValOp,
+ unsigned StWidth);
+
/// isShuffleLegal - Return non-null if a vector shuffle is legal with the
/// specified mask and type. Targets can specify exactly which masks they
/// support and the code generator is tasked with not creating illegal masks.
SmallPtrSet<SDNode*, 32> &NodesLeadingTo);
void LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC);
+ void LegalizeSetCCCondCode(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC);
+ void LegalizeSetCC(MVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC) {
+ LegalizeSetCCOperands(LHS, RHS, CC);
+ LegalizeSetCCCondCode(VT, LHS, RHS, CC);
+ }
SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned,
SDValue &Hi);
SDValue ExpandEXTRACT_SUBVECTOR(SDValue Op);
SDValue ExpandEXTRACT_VECTOR_ELT(SDValue Op);
+
+ // Returns the legalized (truncated or extended) shift amount.
+ SDValue LegalizeShiftAmount(SDValue ShiftAmt);
};
}
return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.getNode() : 0;
}
-SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
- : TLI(dag.getTargetLoweringInfo()), DAG(dag),
+SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, bool types)
+ : TLI(dag.getTargetLoweringInfo()), DAG(dag), TypesNeedLegalizing(types),
ValueTypeActions(TLI.getValueTypeActions()) {
assert(MVT::LAST_VALUETYPE <= 32 &&
"Too many value types for ValueTypeActions to hold!");
// practice however, this causes us to run out of stack space on large basic
// blocks. To avoid this problem, compute an ordering of the nodes where each
// node is only legalized after all of its operands are legalized.
- std::vector<SDNode *> TopOrder;
- unsigned N = DAG.AssignTopologicalOrder(TopOrder);
- for (unsigned i = N; i != 0; --i)
- HandleOp(SDValue(TopOrder[i-1], 0));
- TopOrder.clear();
+ DAG.AssignTopologicalOrder();
+ for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
+ E = prior(DAG.allnodes_end()); I != next(E); ++I)
+ HandleOp(SDValue(I, 0));
// Finally, it's possible the root changed. Get the new root.
SDValue OldRoot = DAG.getRoot();
PromotedNodes.clear();
SplitNodes.clear();
ScalarizedNodes.clear();
+ WidenNodes.clear();
// Remove dead nodes now.
DAG.RemoveDeadNodes();
return false;
}
-/// HandleOp - Legalize, Promote, or Expand the specified operand as
+/// HandleOp - Legalize, Promote, Widen, or Expand the specified operand as
/// appropriate for its type.
void SelectionDAGLegalize::HandleOp(SDValue Op) {
MVT VT = Op.getValueType();
+ // If the type legalizer was run then we should never see any illegal result
+ // types here except for target constants (the type legalizer does not touch
+ // those) or for build vector used as a mask for a vector shuffle.
+ // FIXME: We can removed the BUILD_VECTOR case when we fix PR2957.
+ assert((TypesNeedLegalizing || getTypeAction(VT) == Legal ||
+ Op.getOpcode() == ISD::TargetConstant ||
+ Op.getOpcode() == ISD::BUILD_VECTOR) &&
+ "Illegal type introduced after type legalization?");
switch (getTypeAction(VT)) {
default: assert(0 && "Bad type action!");
case Legal: (void)LegalizeOp(Op); break;
- case Promote: (void)PromoteOp(Op); break;
+ case Promote:
+ if (!VT.isVector()) {
+ (void)PromoteOp(Op);
+ break;
+ }
+ else {
+ // See if we can widen otherwise use Expand to either scalarize or split
+ MVT WidenVT = TLI.getWidenVectorType(VT);
+ if (WidenVT != MVT::Other) {
+ (void) WidenVectorOp(Op, WidenVT);
+ break;
+ }
+ // else fall thru to expand since we can't widen the vector
+ }
case Expand:
if (!VT.isVector()) {
// If this is an illegal scalar, expand it into its two component
// scalar operation.
(void)ScalarizeVectorOp(Op);
} else {
- // Otherwise, this is an illegal multiple element vector.
+ // This is an illegal multiple element vector.
// Split it in half and legalize both parts.
SDValue X, Y;
SplitVectorOp(Op, X, Y);
/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or
/// a load from the constant pool.
static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP,
- SelectionDAG &DAG, TargetLowering &TLI) {
+ SelectionDAG &DAG, const TargetLowering &TLI) {
bool Extend = false;
// If a FP immediate is precise when represented as a float and if the
if (!UseCP) {
if (VT!=MVT::f64 && VT!=MVT::f32)
assert(0 && "Invalid type expansion");
- return DAG.getConstant(LLVMC->getValueAPF().convertToAPInt(),
+ return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(),
(VT == MVT::f64) ? MVT::i64 : MVT::i32);
}
if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) &&
// Only do this if the target has a native EXTLOAD instruction from
// smaller type.
- TLI.isLoadXLegal(ISD::EXTLOAD, SVT) &&
+ TLI.isLoadExtLegal(ISD::EXTLOAD, SVT) &&
TLI.ShouldShrinkFPConstant(OrigVT)) {
const Type *SType = SVT.getTypeForMVT();
LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
/// operations.
static
SDValue ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT,
- SelectionDAG &DAG, TargetLowering &TLI) {
+ SelectionDAG &DAG,
+ const TargetLowering &TLI) {
MVT VT = Node->getValueType(0);
MVT SrcVT = Node->getOperand(1).getValueType();
assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) &&
/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores.
static
SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
- TargetLowering &TLI) {
+ const TargetLowering &TLI) {
SDValue Chain = ST->getChain();
SDValue Ptr = ST->getBasePtr();
SDValue Val = ST->getValue();
int SVOffset = ST->getSrcValueOffset();
if (ST->getMemoryVT().isFloatingPoint() ||
ST->getMemoryVT().isVector()) {
- // Expand to a bitconvert of the value to the integer type of the
- // same size, then a (misaligned) int store.
- MVT intVT;
- if (VT.is128BitVector() || VT == MVT::ppcf128 || VT == MVT::f128)
- intVT = MVT::i128;
- else if (VT.is64BitVector() || VT==MVT::f64)
- intVT = MVT::i64;
- else if (VT==MVT::f32)
- intVT = MVT::i32;
- else
- assert(0 && "Unaligned store of unsupported type");
+ MVT intVT = MVT::getIntegerVT(VT.getSizeInBits());
+ if (TLI.isTypeLegal(intVT)) {
+ // Expand to a bitconvert of the value to the integer type of the
+ // same size, then a (misaligned) int store.
+ // FIXME: Does not handle truncating floating point stores!
+ SDValue Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val);
+ return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(),
+ SVOffset, ST->isVolatile(), Alignment);
+ } else {
+ // Do a (aligned) store to a stack slot, then copy from the stack slot
+ // to the final destination using (unaligned) integer loads and stores.
+ MVT StoredVT = ST->getMemoryVT();
+ MVT RegVT =
+ TLI.getRegisterType(MVT::getIntegerVT(StoredVT.getSizeInBits()));
+ unsigned StoredBytes = StoredVT.getSizeInBits() / 8;
+ unsigned RegBytes = RegVT.getSizeInBits() / 8;
+ unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
+
+ // Make sure the stack slot is also aligned for the register type.
+ SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT);
+
+ // Perform the original store, only redirected to the stack slot.
+ SDValue Store = DAG.getTruncStore(Chain, Val, StackPtr, NULL, 0,StoredVT);
+ SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy());
+ SmallVector<SDValue, 8> Stores;
+ unsigned Offset = 0;
+
+ // Do all but one copies using the full register width.
+ for (unsigned i = 1; i < NumRegs; i++) {
+ // Load one integer register's worth from the stack slot.
+ SDValue Load = DAG.getLoad(RegVT, Store, StackPtr, NULL, 0);
+ // Store it to the final location. Remember the store.
+ Stores.push_back(DAG.getStore(Load.getValue(1), Load, Ptr,
+ ST->getSrcValue(), SVOffset + Offset,
+ ST->isVolatile(),
+ MinAlign(ST->getAlignment(), Offset)));
+ // Increment the pointers.
+ Offset += RegBytes;
+ StackPtr = DAG.getNode(ISD::ADD, StackPtr.getValueType(), StackPtr,
+ Increment);
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, Increment);
+ }
- SDValue Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val);
- return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(),
- SVOffset, ST->isVolatile(), Alignment);
+ // The last store may be partial. Do a truncating store. On big-endian
+ // machines this requires an extending load from the stack slot to ensure
+ // that the bits are in the right place.
+ MVT MemVT = MVT::getIntegerVT(8 * (StoredBytes - Offset));
+
+ // Load from the stack slot.
+ SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, RegVT, Store, StackPtr,
+ NULL, 0, MemVT);
+
+ Stores.push_back(DAG.getTruncStore(Load.getValue(1), Load, Ptr,
+ ST->getSrcValue(), SVOffset + Offset,
+ MemVT, ST->isVolatile(),
+ MinAlign(ST->getAlignment(), Offset)));
+ // The order of the stores doesn't matter - say it with a TokenFactor.
+ return DAG.getNode(ISD::TokenFactor, MVT::Other, &Stores[0],
+ Stores.size());
+ }
}
assert(ST->getMemoryVT().isInteger() &&
!ST->getMemoryVT().isVector() &&
/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads.
static
SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
- TargetLowering &TLI) {
+ const TargetLowering &TLI) {
int SVOffset = LD->getSrcValueOffset();
SDValue Chain = LD->getChain();
SDValue Ptr = LD->getBasePtr();
MVT VT = LD->getValueType(0);
MVT LoadedVT = LD->getMemoryVT();
if (VT.isFloatingPoint() || VT.isVector()) {
- // Expand to a (misaligned) integer load of the same size,
- // then bitconvert to floating point or vector.
- MVT intVT;
- if (LoadedVT.is128BitVector() ||
- LoadedVT == MVT::ppcf128 || LoadedVT == MVT::f128)
- intVT = MVT::i128;
- else if (LoadedVT.is64BitVector() || LoadedVT == MVT::f64)
- intVT = MVT::i64;
- else if (LoadedVT == MVT::f32)
- intVT = MVT::i32;
- else
- assert(0 && "Unaligned load of unsupported type");
-
- SDValue newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(),
- SVOffset, LD->isVolatile(),
+ MVT intVT = MVT::getIntegerVT(LoadedVT.getSizeInBits());
+ if (TLI.isTypeLegal(intVT)) {
+ // Expand to a (misaligned) integer load of the same size,
+ // then bitconvert to floating point or vector.
+ SDValue newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(),
+ SVOffset, LD->isVolatile(),
LD->getAlignment());
- SDValue Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad);
- if (VT.isFloatingPoint() && LoadedVT != VT)
- Result = DAG.getNode(ISD::FP_EXTEND, VT, Result);
+ SDValue Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad);
+ if (VT.isFloatingPoint() && LoadedVT != VT)
+ Result = DAG.getNode(ISD::FP_EXTEND, VT, Result);
+
+ SDValue Ops[] = { Result, Chain };
+ return DAG.getMergeValues(Ops, 2);
+ } else {
+ // Copy the value to a (aligned) stack slot using (unaligned) integer
+ // loads and stores, then do a (aligned) load from the stack slot.
+ MVT RegVT = TLI.getRegisterType(intVT);
+ unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8;
+ unsigned RegBytes = RegVT.getSizeInBits() / 8;
+ unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
+
+ // Make sure the stack slot is also aligned for the register type.
+ SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
+
+ SDValue Increment = DAG.getConstant(RegBytes, TLI.getPointerTy());
+ SmallVector<SDValue, 8> Stores;
+ SDValue StackPtr = StackBase;
+ unsigned Offset = 0;
+
+ // Do all but one copies using the full register width.
+ for (unsigned i = 1; i < NumRegs; i++) {
+ // Load one integer register's worth from the original location.
+ SDValue Load = DAG.getLoad(RegVT, Chain, Ptr, LD->getSrcValue(),
+ SVOffset + Offset, LD->isVolatile(),
+ MinAlign(LD->getAlignment(), Offset));
+ // Follow the load with a store to the stack slot. Remember the store.
+ Stores.push_back(DAG.getStore(Load.getValue(1), Load, StackPtr,
+ NULL, 0));
+ // Increment the pointers.
+ Offset += RegBytes;
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr, Increment);
+ StackPtr = DAG.getNode(ISD::ADD, StackPtr.getValueType(), StackPtr,
+ Increment);
+ }
+
+ // The last copy may be partial. Do an extending load.
+ MVT MemVT = MVT::getIntegerVT(8 * (LoadedBytes - Offset));
+ SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, RegVT, Chain, Ptr,
+ LD->getSrcValue(), SVOffset + Offset,
+ MemVT, LD->isVolatile(),
+ MinAlign(LD->getAlignment(), Offset));
+ // Follow the load with a store to the stack slot. Remember the store.
+ // On big-endian machines this requires a truncating store to ensure
+ // that the bits end up in the right place.
+ Stores.push_back(DAG.getTruncStore(Load.getValue(1), Load, StackPtr,
+ NULL, 0, MemVT));
+
+ // The order of the stores doesn't matter - say it with a TokenFactor.
+ SDValue TF = DAG.getNode(ISD::TokenFactor, MVT::Other, &Stores[0],
+ Stores.size());
+
+ // Finally, perform the original load only redirected to the stack slot.
+ Load = DAG.getExtLoad(LD->getExtensionType(), VT, TF, StackBase,
+ NULL, 0, LoadedVT);
- SDValue Ops[] = { Result, Chain };
- return DAG.getMergeValues(Ops, 2);
+ // Callers expect a MERGE_VALUES node.
+ SDValue Ops[] = { Load, TF };
+ return DAG.getMergeValues(Ops, 2);
+ }
}
assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
"Unaligned load of unsupported type.");
Operands[j] = Operand;
}
}
- Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT,
- &Operands[0], Operands.size()));
+
+ switch (Op.getOpcode()) {
+ default:
+ Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT,
+ &Operands[0], Operands.size()));
+ break;
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL:
+ Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT, Operands[0],
+ LegalizeShiftAmount(Operands[1])));
+ break;
+ }
}
return DAG.getNode(ISD::BUILD_VECTOR, VT, &Scalars[0], Scalars.size());
// Store the vector.
SDValue Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr,
- PseudoSourceValue::getFixedStack(SPFI), 0);
+ PseudoSourceValue::getFixedStack(SPFI), 0);
// Truncate or zero extend offset to target pointer type.
unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
PseudoSourceValue::getFixedStack(SPFI), 0);
}
+SDValue SelectionDAGLegalize::LegalizeShiftAmount(SDValue ShiftAmt) {
+ if (TLI.getShiftAmountTy().bitsLT(ShiftAmt.getValueType()))
+ return DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), ShiftAmt);
+
+ if (TLI.getShiftAmountTy().bitsGT(ShiftAmt.getValueType()))
+ return DAG.getNode(ISD::ZERO_EXTEND, TLI.getShiftAmountTy(), ShiftAmt);
+
+ return ShiftAmt;
+}
+
+
/// LegalizeOp - We know that the specified value has a legal type, and
/// that its operands are legal. Now ensure that the operation itself
/// is legal, recursively ensuring that the operands' operations remain
case TargetLowering::Promote:
default: assert(0 && "This action is not supported yet!");
case TargetLowering::Expand: {
- MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
- bool useDEBUG_LOC = TLI.isOperationLegal(ISD::DEBUG_LOC, MVT::Other);
- bool useLABEL = TLI.isOperationLegal(ISD::DBG_LABEL, MVT::Other);
+ DwarfWriter *DW = DAG.getDwarfWriter();
+ bool useDEBUG_LOC = TLI.isOperationLegalOrCustom(ISD::DEBUG_LOC,
+ MVT::Other);
+ bool useLABEL = TLI.isOperationLegalOrCustom(ISD::DBG_LABEL, MVT::Other);
const DbgStopPointSDNode *DSP = cast<DbgStopPointSDNode>(Node);
- if (MMI && (useDEBUG_LOC || useLABEL)) {
- const CompileUnitDesc *CompileUnit = DSP->getCompileUnit();
- unsigned SrcFile = MMI->RecordSource(CompileUnit);
-
+ GlobalVariable *CU_GV = cast<GlobalVariable>(DSP->getCompileUnit());
+ if (DW && (useDEBUG_LOC || useLABEL) && !CU_GV->isDeclaration()) {
+ DICompileUnit CU(cast<GlobalVariable>(DSP->getCompileUnit()));
+ unsigned SrcFile = DW->RecordSource(CU.getDirectory(),
+ CU.getFilename());
+
unsigned Line = DSP->getLine();
unsigned Col = DSP->getColumn();
DAG.getConstant(SrcFile, MVT::i32) };
Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, Ops, 4);
} else {
- unsigned ID = MMI->RecordSourceLine(Line, Col, SrcFile);
+ unsigned ID = DW->RecordSourceLine(Line, Col, SrcFile);
Result = DAG.getLabel(ISD::DBG_LABEL, Tmp1, ID);
}
} else {
break;
}
- case ISD::ATOMIC_CMP_SWAP_8:
- case ISD::ATOMIC_CMP_SWAP_16:
- case ISD::ATOMIC_CMP_SWAP_32:
- case ISD::ATOMIC_CMP_SWAP_64: {
+ case ISD::ATOMIC_CMP_SWAP: {
unsigned int num_operands = 4;
assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!");
SDValue Ops[4];
AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1));
return Result.getValue(Op.getResNo());
}
- case ISD::ATOMIC_LOAD_ADD_8:
- case ISD::ATOMIC_LOAD_SUB_8:
- case ISD::ATOMIC_LOAD_AND_8:
- case ISD::ATOMIC_LOAD_OR_8:
- case ISD::ATOMIC_LOAD_XOR_8:
- case ISD::ATOMIC_LOAD_NAND_8:
- case ISD::ATOMIC_LOAD_MIN_8:
- case ISD::ATOMIC_LOAD_MAX_8:
- case ISD::ATOMIC_LOAD_UMIN_8:
- case ISD::ATOMIC_LOAD_UMAX_8:
- case ISD::ATOMIC_SWAP_8:
- case ISD::ATOMIC_LOAD_ADD_16:
- case ISD::ATOMIC_LOAD_SUB_16:
- case ISD::ATOMIC_LOAD_AND_16:
- case ISD::ATOMIC_LOAD_OR_16:
- case ISD::ATOMIC_LOAD_XOR_16:
- case ISD::ATOMIC_LOAD_NAND_16:
- case ISD::ATOMIC_LOAD_MIN_16:
- case ISD::ATOMIC_LOAD_MAX_16:
- case ISD::ATOMIC_LOAD_UMIN_16:
- case ISD::ATOMIC_LOAD_UMAX_16:
- case ISD::ATOMIC_SWAP_16:
- case ISD::ATOMIC_LOAD_ADD_32:
- case ISD::ATOMIC_LOAD_SUB_32:
- case ISD::ATOMIC_LOAD_AND_32:
- case ISD::ATOMIC_LOAD_OR_32:
- case ISD::ATOMIC_LOAD_XOR_32:
- case ISD::ATOMIC_LOAD_NAND_32:
- case ISD::ATOMIC_LOAD_MIN_32:
- case ISD::ATOMIC_LOAD_MAX_32:
- case ISD::ATOMIC_LOAD_UMIN_32:
- case ISD::ATOMIC_LOAD_UMAX_32:
- case ISD::ATOMIC_SWAP_32:
- case ISD::ATOMIC_LOAD_ADD_64:
- case ISD::ATOMIC_LOAD_SUB_64:
- case ISD::ATOMIC_LOAD_AND_64:
- case ISD::ATOMIC_LOAD_OR_64:
- case ISD::ATOMIC_LOAD_XOR_64:
- case ISD::ATOMIC_LOAD_NAND_64:
- case ISD::ATOMIC_LOAD_MIN_64:
- case ISD::ATOMIC_LOAD_MAX_64:
- case ISD::ATOMIC_LOAD_UMIN_64:
- case ISD::ATOMIC_LOAD_UMAX_64:
- case ISD::ATOMIC_SWAP_64: {
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_LOAD_MIN:
+ case ISD::ATOMIC_LOAD_MAX:
+ case ISD::ATOMIC_LOAD_UMIN:
+ case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_SWAP: {
unsigned int num_operands = 3;
assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!");
SDValue Ops[3];
case TargetLowering::Custom:
Result = TLI.LowerOperation(Result, DAG);
break;
- case TargetLowering::Expand:
- Result = SDValue(TLI.ReplaceNodeResults(Op.getNode(), DAG),0);
- break;
case TargetLowering::Legal:
break;
}
default: assert(0 && "Cannot expand insert element operand");
case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break;
case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break;
+ case Expand:
+ // FIXME: An alternative would be to check to see if the target is not
+ // going to custom lower this operation, we could bitcast to half elt
+ // width and perform two inserts at that width, if that is legal.
+ Tmp2 = Node->getOperand(1);
+ break;
}
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
break;
}
// FALLTHROUGH
+ case TargetLowering::Promote:
+ // Fall thru for vector case
case TargetLowering::Expand: {
// If the insert index is a constant, codegen this as a scalar_to_vector,
// then a shuffle that inserts it into the right position in the vector.
Result = ExpandEXTRACT_SUBVECTOR(Result);
break;
+ case ISD::CONCAT_VECTORS: {
+ // Use extract/insert/build vector for now. We might try to be
+ // more clever later.
+ MVT PtrVT = TLI.getPointerTy();
+ SmallVector<SDValue, 8> Ops;
+ unsigned NumOperands = Node->getNumOperands();
+ for (unsigned i=0; i < NumOperands; ++i) {
+ SDValue SubOp = Node->getOperand(i);
+ MVT VVT = SubOp.getNode()->getValueType(0);
+ MVT EltVT = VVT.getVectorElementType();
+ unsigned NumSubElem = VVT.getVectorNumElements();
+ for (unsigned j=0; j < NumSubElem; ++j) {
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, SubOp,
+ DAG.getConstant(j, PtrVT)));
+ }
+ }
+ return LegalizeOp(DAG.getNode(ISD::BUILD_VECTOR, Node->getValueType(0),
+ &Ops[0], Ops.size()));
+ }
+
case ISD::CALLSEQ_START: {
SDNode *CallEnd = FindCallEndFromCallStart(Node);
// Chain the dynamic stack allocation so that it doesn't modify the stack
// pointer when other instructions are using the stack.
- Chain = DAG.getCALLSEQ_START(Chain,
- DAG.getConstant(0, TLI.getPointerTy()));
+ Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true));
SDValue Size = Tmp2.getOperand(1);
SDValue SP = DAG.getCopyFromReg(Chain, SPReg, VT);
Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size); // Value
Chain = DAG.getCopyToReg(Chain, SPReg, Tmp1); // Output chain
- Tmp2 =
- DAG.getCALLSEQ_END(Chain,
- DAG.getConstant(0, TLI.getPointerTy()),
- DAG.getConstant(0, TLI.getPointerTy()),
- SDValue());
+ Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true),
+ DAG.getIntPtrConstant(0, true), SDValue());
Tmp1 = LegalizeOp(Tmp1);
Tmp2 = LegalizeOp(Tmp2);
unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize();
Index= DAG.getNode(ISD::MUL, PTy, Index, DAG.getConstant(EntrySize, PTy));
SDValue Addr = DAG.getNode(ISD::ADD, PTy, Index, Table);
-
- SDValue LD;
- switch (EntrySize) {
- default: assert(0 && "Size of jump table not supported yet."); break;
- case 4: LD = DAG.getLoad(MVT::i32, Chain, Addr,
- PseudoSourceValue::getJumpTable(), 0); break;
- case 8: LD = DAG.getLoad(MVT::i64, Chain, Addr,
- PseudoSourceValue::getJumpTable(), 0); break;
- }
+ MVT MemVT = MVT::getIntegerVT(EntrySize * 8);
+ SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, PTy, Chain, Addr,
+ PseudoSourceValue::getJumpTable(), 0, MemVT);
Addr = LD;
if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) {
// For PIC, the sequence is:
// BRIND(load(Jumptable + index) + RelocBase)
// RelocBase can be JumpTable, GOT or some sort of global base.
- if (PTy != MVT::i32)
- Addr = DAG.getNode(ISD::SIGN_EXTEND, PTy, Addr);
Addr = DAG.getNode(ISD::ADD, PTy, Addr,
TLI.getPICJumpTableRelocBase(Table, DAG));
}
Tmp3 = Node->getOperand(3); // RHS
Tmp4 = Node->getOperand(1); // CC
- LegalizeSetCCOperands(Tmp2, Tmp3, Tmp4);
+ LegalizeSetCC(TLI.getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3,Tmp4);
LastCALLSEQ_END = DAG.getEntryNode();
- // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
+ // If we didn't get both a LHS and RHS back from LegalizeSetCC,
// the LHS is a legal SETCC itself. In this case, we need to compare
// the result against zero to select between true and false values.
if (Tmp3.getNode() == 0) {
// nice to have an effective generic way of getting these benefits...
// Until such a way is found, don't insist on promoting i1 here.
(SrcVT != MVT::i1 ||
- TLI.getLoadXAction(ExtType, MVT::i1) == TargetLowering::Promote)) {
+ TLI.getLoadExtAction(ExtType, MVT::i1) == TargetLowering::Promote)) {
// Promote to a byte-sized load if not loading an integral number of
// bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
unsigned NewWidth = SrcVT.getStoreSizeInBits();
Tmp1 = LegalizeOp(Result);
Tmp2 = LegalizeOp(Ch);
} else {
- switch (TLI.getLoadXAction(ExtType, SrcVT)) {
+ switch (TLI.getLoadExtAction(ExtType, SrcVT)) {
default: assert(0 && "This action is not supported yet!");
case TargetLowering::Custom:
isCustom = true;
if (CFP->getValueType(0) == MVT::f32 &&
getTypeAction(MVT::i32) == Legal) {
Tmp3 = DAG.getConstant(CFP->getValueAPF().
- convertToAPInt().zextOrTrunc(32),
+ bitcastToAPInt().zextOrTrunc(32),
MVT::i32);
Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
SVOffset, isVolatile, Alignment);
} else if (CFP->getValueType(0) == MVT::f64) {
// If this target supports 64-bit registers, do a single 64-bit store.
if (getTypeAction(MVT::i64) == Legal) {
- Tmp3 = DAG.getConstant(CFP->getValueAPF().convertToAPInt().
+ Tmp3 = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
zextOrTrunc(64), MVT::i64);
Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
SVOffset, isVolatile, Alignment);
// Otherwise, if the target supports 32-bit registers, use 2 32-bit
// stores. If the target supports neither 32- nor 64-bits, this
// xform is certainly not worth it.
- const APInt &IntVal =CFP->getValueAPF().convertToAPInt();
+ const APInt &IntVal =CFP->getValueAPF().bitcastToAPInt();
SDValue Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32);
SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32);
if (TLI.isBigEndian()) std::swap(Lo, Hi);
break;
}
case Promote:
- // Truncate the value and store the result.
- Tmp3 = PromoteOp(ST->getValue());
- Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
- SVOffset, ST->getMemoryVT(),
- isVolatile, Alignment);
- break;
-
- case Expand:
+ if (!ST->getMemoryVT().isVector()) {
+ // Truncate the value and store the result.
+ Tmp3 = PromoteOp(ST->getValue());
+ Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, ST->getMemoryVT(),
+ isVolatile, Alignment);
+ break;
+ }
+ // Fall thru to expand for vector
+ case Expand: {
unsigned IncrementSize = 0;
SDValue Lo, Hi;
Result = LegalizeOp(Result);
break;
} else {
- SplitVectorOp(ST->getValue(), Lo, Hi);
- IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() *
- EVT.getSizeInBits()/8;
+ // Check if we have widen this node with another value
+ std::map<SDValue, SDValue>::iterator I =
+ WidenNodes.find(ST->getValue());
+ if (I != WidenNodes.end()) {
+ Result = StoreWidenVectorOp(ST, Tmp1, Tmp2);
+ break;
+ }
+ else {
+ SplitVectorOp(ST->getValue(), Lo, Hi);
+ IncrementSize = Lo.getNode()->getValueType(0).getVectorNumElements() *
+ EVT.getSizeInBits()/8;
+ }
}
} else {
ExpandOp(ST->getValue(), Lo, Hi);
SVOffset, isVolatile, Alignment);
Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
break;
+ } // case Expand
}
} else {
switch (getTypeAction(ST->getValue().getValueType())) {
Tmp3 = LegalizeOp(ST->getValue());
break;
case Promote:
- // We can promote the value, the truncstore will still take care of it.
- Tmp3 = PromoteOp(ST->getValue());
- break;
+ if (!ST->getValue().getValueType().isVector()) {
+ // We can promote the value, the truncstore will still take care of it.
+ Tmp3 = PromoteOp(ST->getValue());
+ break;
+ }
+ // Vector case falls through to expand
case Expand:
// Just store the low part. This may become a non-trunc store, so make
// sure to use getTruncStore, not UpdateNodeOperands below.
Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition.
break;
case Promote: {
+ assert(!Node->getOperand(0).getValueType().isVector() && "not possible");
Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition.
// Make sure the condition is either zero or one.
unsigned BitWidth = Tmp1.getValueSizeInBits();
Tmp4 = LegalizeOp(Node->getOperand(3)); // False
SDValue CC = Node->getOperand(4);
- LegalizeSetCCOperands(Tmp1, Tmp2, CC);
+ LegalizeSetCC(TLI.getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC);
- // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
+ // If we didn't get both a LHS and RHS back from LegalizeSetCC,
// the LHS is a legal SETCC itself. In this case, we need to compare
// the result against zero to select between true and false values.
if (Tmp2.getNode() == 0) {
Tmp1 = Node->getOperand(0);
Tmp2 = Node->getOperand(1);
Tmp3 = Node->getOperand(2);
- LegalizeSetCCOperands(Tmp1, Tmp2, Tmp3);
+ LegalizeSetCC(Node->getValueType(0), Tmp1, Tmp2, Tmp3);
// If we had to Expand the SetCC operands into a SELECT node, then it may
// not always be possible to return a true LHS & RHS. In this case, just
"Fell off of the edge of the floating point world");
// If the target supports SETCC of this type, use it.
- if (TLI.isOperationLegal(ISD::SETCC, NewInTy))
+ if (TLI.isOperationLegalOrCustom(ISD::SETCC, NewInTy))
break;
}
if (NewInTy.isInteger())
Tmp1 = TLI.LowerOperation(Result, DAG);
if (Tmp1.getNode()) Result = Tmp1;
break;
+ case TargetLowering::Expand: {
+ // Unroll into a nasty set of scalar code for now.
+ MVT VT = Node->getValueType(0);
+ unsigned NumElems = VT.getVectorNumElements();
+ MVT EltVT = VT.getVectorElementType();
+ MVT TmpEltVT = Tmp1.getValueType().getVectorElementType();
+ SmallVector<SDValue, 8> Ops(NumElems);
+ for (unsigned i = 0; i < NumElems; ++i) {
+ SDValue In1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, TmpEltVT,
+ Tmp1, DAG.getIntPtrConstant(i));
+ Ops[i] = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(TmpEltVT), In1,
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, TmpEltVT,
+ Tmp2, DAG.getIntPtrConstant(i)),
+ CC);
+ Ops[i] = DAG.getNode(ISD::SELECT, EltVT, Ops[i],
+ DAG.getConstant(EltVT.getIntegerVTBitMask(),EltVT),
+ DAG.getConstant(0, EltVT));
+ }
+ Result = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], NumElems);
+ break;
+ }
}
break;
}
Node->getOpcode() == ISD::SRL ||
Node->getOpcode() == ISD::SRA) &&
!Node->getValueType(0).isVector()) {
- if (TLI.getShiftAmountTy().bitsLT(Tmp2.getValueType()))
- Tmp2 = DAG.getNode(ISD::TRUNCATE, TLI.getShiftAmountTy(), Tmp2);
- else if (TLI.getShiftAmountTy().bitsGT(Tmp2.getValueType()))
- Tmp2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Tmp2);
+ Tmp2 = LegalizeShiftAmount(Tmp2);
}
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
-
+
switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
default: assert(0 && "BinOp legalize operation not supported");
case TargetLowering::Legal: break;
// Fall through if the custom lower can't deal with the operation
case TargetLowering::Expand: {
MVT VT = Op.getValueType();
-
+
// See if multiply or divide can be lowered using two-result operations.
SDVTList VTs = DAG.getVTList(VT, VT);
if (Node->getOpcode() == ISD::MUL) {
// and unsigned forms. If the target supports both SMUL_LOHI and
// UMUL_LOHI, form a preference by checking which forms of plain
// MULH it supports.
- bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, VT);
- bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, VT);
- bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, VT);
- bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, VT);
+ bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
+ bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
+ bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
+ bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
unsigned OpToUse = 0;
if (HasSMUL_LOHI && !HasMULHS) {
OpToUse = ISD::SMUL_LOHI;
}
}
if (Node->getOpcode() == ISD::MULHS &&
- TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) {
- Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), 1);
+ TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) {
+ Result = SDValue(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).getNode(),
+ 1);
break;
}
if (Node->getOpcode() == ISD::MULHU &&
- TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) {
- Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).getNode(), 1);
+ TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) {
+ Result = SDValue(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).getNode(),
+ 1);
break;
}
if (Node->getOpcode() == ISD::SDIV &&
- TLI.isOperationLegal(ISD::SDIVREM, VT)) {
- Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), 0);
+ TLI.isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
+ Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(),
+ 0);
break;
}
if (Node->getOpcode() == ISD::UDIV &&
- TLI.isOperationLegal(ISD::UDIVREM, VT)) {
- Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), 0);
+ TLI.isOperationLegalOrCustom(ISD::UDIVREM, VT)) {
+ Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(),
+ 0);
break;
}
case ISD::SDIV:
if (VT == MVT::i32) {
LC = Node->getOpcode() == ISD::UDIV
- ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32;
+ ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32;
isSigned = Node->getOpcode() == ISD::SDIV;
}
break;
+ case ISD::MUL:
+ if (VT == MVT::i32)
+ LC = RTLIB::MUL_I32;
+ else if (VT == MVT::i64)
+ LC = RTLIB::MUL_I64;
+ break;
case ISD::FPOW:
LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
RTLIB::POW_PPCF128);
break;
+ case ISD::FDIV:
+ LC = GetFPLibCall(VT, RTLIB::DIV_F32, RTLIB::DIV_F64, RTLIB::DIV_F80,
+ RTLIB::DIV_PPCF128);
+ break;
default: break;
}
if (LC != RTLIB::UNKNOWN_LIBCALL) {
Result = ExpandLibCall(LC, Node, isSigned, Dummy);
break;
}
-
+
assert(Node->getValueType(0).isVector() &&
"Cannot expand this binary operator!");
// Expand the operation into a bunch of nasty scalar code.
MVT IVT =
Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64;
SDValue SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2);
- SignBit = DAG.getSetCC(TLI.getSetCCResultType(SignBit),
+ SignBit = DAG.getSetCC(TLI.getSetCCResultType(IVT),
SignBit, DAG.getConstant(0, IVT), ISD::SETLT);
// Get the absolute value of the result.
SDValue AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1);
Tmp1 = LegalizeOp(Node->getOperand(0));
Tmp2 = LegalizeOp(Node->getOperand(1));
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ Tmp3 = Result.getValue(0);
+ Tmp4 = Result.getValue(1);
+
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Tmp3, DAG);
+ if (Tmp1.getNode() != NULL) {
+ Tmp3 = LegalizeOp(Tmp1);
+ Tmp4 = LegalizeOp(Tmp1.getValue(1));
+ }
+ break;
+ }
// Since this produces two values, make sure to remember that we legalized
// both of them.
- AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0));
- AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1));
- return Result;
+ AddLegalizedOperand(SDValue(Node, 0), Tmp3);
+ AddLegalizedOperand(SDValue(Node, 1), Tmp4);
+ return Op.getResNo() ? Tmp4 : Tmp3;
case ISD::ADDE:
case ISD::SUBE:
Tmp2 = LegalizeOp(Node->getOperand(1));
Tmp3 = LegalizeOp(Node->getOperand(2));
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ Tmp3 = Result.getValue(0);
+ Tmp4 = Result.getValue(1);
+
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Tmp3, DAG);
+ if (Tmp1.getNode() != NULL) {
+ Tmp3 = LegalizeOp(Tmp1);
+ Tmp4 = LegalizeOp(Tmp1.getValue(1));
+ }
+ break;
+ }
// Since this produces two values, make sure to remember that we legalized
// both of them.
- AddLegalizedOperand(SDValue(Node, 0), Result.getValue(0));
- AddLegalizedOperand(SDValue(Node, 1), Result.getValue(1));
- return Result;
+ AddLegalizedOperand(SDValue(Node, 0), Tmp3);
+ AddLegalizedOperand(SDValue(Node, 1), Tmp4);
+ return Op.getResNo() ? Tmp4 : Tmp3;
case ISD::BUILD_PAIR: {
MVT PairTy = Node->getValueType(0);
// See if remainder can be lowered using two-result operations.
SDVTList VTs = DAG.getVTList(VT, VT);
if (Node->getOpcode() == ISD::SREM &&
- TLI.isOperationLegal(ISD::SDIVREM, VT)) {
+ TLI.isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
Result = SDValue(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).getNode(), 1);
break;
}
if (Node->getOpcode() == ISD::UREM &&
- TLI.isOperationLegal(ISD::UDIVREM, VT)) {
+ TLI.isOperationLegalOrCustom(ISD::UDIVREM, VT)) {
Result = SDValue(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).getNode(), 1);
break;
}
const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
SDValue VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0);
// Increment the pointer, VAList, to the next vaarg
- Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
- DAG.getConstant(VT.getSizeInBits()/8,
+ Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
+ DAG.getConstant(TLI.getTargetData()->
+ getTypePaddedSize(VT.getTypeForMVT()),
TLI.getPointerTy()));
// Store the incremented VAList to the legalized pointer
Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0);
break;
case ISD::CTTZ:
//if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
- Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1,
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1.getValueType()), Tmp1,
DAG.getConstant(NVT.getSizeInBits(), NVT),
ISD::SETEQ);
Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
// Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
MVT VT = Node->getValueType(0);
Tmp2 = DAG.getConstantFP(0.0, VT);
- Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2,
- ISD::SETUGT);
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1.getValueType()),
+ Tmp1, Tmp2, ISD::SETUGT);
Tmp3 = DAG.getNode(ISD::FNEG, VT, Tmp1);
Result = DAG.getNode(ISD::SELECT, VT, Tmp2, Tmp1, Tmp3);
break;
}
}
break;
-
+ case ISD::CONVERT_RNDSAT: {
+ ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode();
+ switch (CvtCode) {
+ default: assert(0 && "Unknown cvt code!");
+ case ISD::CVT_SF:
+ case ISD::CVT_UF:
+ case ISD::CVT_FF:
+ break;
+ case ISD::CVT_FS:
+ case ISD::CVT_FU:
+ case ISD::CVT_SS:
+ case ISD::CVT_SU:
+ case ISD::CVT_US:
+ case ISD::CVT_UU: {
+ SDValue DTyOp = Node->getOperand(1);
+ SDValue STyOp = Node->getOperand(2);
+ SDValue RndOp = Node->getOperand(3);
+ SDValue SatOp = Node->getOperand(4);
+ switch (getTypeAction(Node->getOperand(0).getValueType())) {
+ case Expand: assert(0 && "Shouldn't need to expand other operators here!");
+ case Legal:
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, DTyOp, STyOp,
+ RndOp, SatOp);
+ if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) ==
+ TargetLowering::Custom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.getNode()) Result = Tmp1;
+ }
+ break;
+ case Promote:
+ Result = PromoteOp(Node->getOperand(0));
+ // For FP, make Op1 a i32
+
+ Result = DAG.getConvertRndSat(Op.getValueType(), Result,
+ DTyOp, STyOp, RndOp, SatOp, CvtCode);
+ break;
+ }
+ break;
+ }
+ } // end switch CvtCode
+ break;
+ }
// Conversion operators. The source and destination have different types.
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP: {
switch (getTypeAction(Node->getOperand(0).getValueType())) {
case Legal:
Tmp1 = LegalizeOp(Node->getOperand(0));
- Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ default: assert(0 && "Unknown TRUNCATE legalization operation action!");
+ case TargetLowering::Custom:
+ isCustom = true;
+ // FALLTHROUGH
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ if (isCustom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.getNode()) Result = Tmp1;
+ }
+ break;
+ case TargetLowering::Expand:
+ assert(Result.getValueType().isVector() && "must be vector type");
+ // Unroll the truncate. We should do better.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ }
break;
case Expand:
ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
APInt x = APInt::getSignBit(NVT.getSizeInBits());
(void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
Tmp2 = DAG.getConstantFP(apf, VT);
- Tmp3 = DAG.getSetCC(TLI.getSetCCResultType(Node->getOperand(0)),
- Node->getOperand(0), Tmp2, ISD::SETLT);
+ Tmp3 = DAG.getSetCC(TLI.getSetCCResultType(VT), Node->getOperand(0),
+ Tmp2, ISD::SETLT);
True = DAG.getNode(ISD::FP_TO_SINT, NVT, Node->getOperand(0));
False = DAG.getNode(ISD::FP_TO_SINT, NVT,
DAG.getNode(ISD::FSUB, VT, Node->getOperand(0),
}
break;
}
+
+ case ISD::SADDO:
+ case ISD::SSUBO: {
+ MVT VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action not supported for this op yet!");
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.getNode()) break;
+ // FALLTHROUGH
+ case TargetLowering::Legal: {
+ SDValue LHS = LegalizeOp(Node->getOperand(0));
+ SDValue RHS = LegalizeOp(Node->getOperand(1));
+
+ SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
+ ISD::ADD : ISD::SUB, LHS.getValueType(),
+ LHS, RHS);
+ MVT OType = Node->getValueType(1);
+
+ SDValue Zero = DAG.getConstant(0, LHS.getValueType());
+
+ // LHSSign -> LHS >= 0
+ // RHSSign -> RHS >= 0
+ // SumSign -> Sum >= 0
+ //
+ // Add:
+ // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
+ // Sub:
+ // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
+ //
+ SDValue LHSSign = DAG.getSetCC(OType, LHS, Zero, ISD::SETGE);
+ SDValue RHSSign = DAG.getSetCC(OType, RHS, Zero, ISD::SETGE);
+ SDValue SignsMatch = DAG.getSetCC(OType, LHSSign, RHSSign,
+ Node->getOpcode() == ISD::SADDO ?
+ ISD::SETEQ : ISD::SETNE);
+
+ SDValue SumSign = DAG.getSetCC(OType, Sum, Zero, ISD::SETGE);
+ SDValue SumSignNE = DAG.getSetCC(OType, LHSSign, SumSign, ISD::SETNE);
+
+ SDValue Cmp = DAG.getNode(ISD::AND, OType, SignsMatch, SumSignNE);
+
+ MVT ValueVTs[] = { LHS.getValueType(), OType };
+ SDValue Ops[] = { Sum, Cmp };
+
+ Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(&ValueVTs[0], 2),
+ &Ops[0], 2);
+ SDNode *RNode = Result.getNode();
+ DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), SDValue(RNode, 0));
+ DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(RNode, 1));
+ break;
+ }
+ }
+
+ break;
+ }
+ case ISD::UADDO:
+ case ISD::USUBO: {
+ MVT VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action not supported for this op yet!");
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.getNode()) break;
+ // FALLTHROUGH
+ case TargetLowering::Legal: {
+ SDValue LHS = LegalizeOp(Node->getOperand(0));
+ SDValue RHS = LegalizeOp(Node->getOperand(1));
+
+ SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
+ ISD::ADD : ISD::SUB, LHS.getValueType(),
+ LHS, RHS);
+ MVT OType = Node->getValueType(1);
+ SDValue Cmp = DAG.getSetCC(OType, Sum, LHS,
+ Node->getOpcode () == ISD::UADDO ?
+ ISD::SETULT : ISD::SETUGT);
+
+ MVT ValueVTs[] = { LHS.getValueType(), OType };
+ SDValue Ops[] = { Sum, Cmp };
+
+ Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(&ValueVTs[0], 2),
+ &Ops[0], 2);
+ SDNode *RNode = Result.getNode();
+ DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), SDValue(RNode, 0));
+ DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(RNode, 1));
+ break;
+ }
+ }
+
+ break;
+ }
+ case ISD::SMULO:
+ case ISD::UMULO: {
+ MVT VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported at all!");
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.getNode()) break;
+ // Fall Thru
+ case TargetLowering::Legal:
+ // FIXME: According to Hacker's Delight, this can be implemented in
+ // target independent lowering, but it would be inefficient, since it
+ // requires a division + a branch.
+ assert(0 && "Target independent lowering is not supported for SMULO/UMULO!");
+ break;
+ }
+ break;
+ }
+
}
assert(Result.getValueType() == Op.getValueType() &&
assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?");
break;
- case ISD::SETCC:
- assert(isTypeLegal(TLI.getSetCCResultType(Node->getOperand(0)))
+ case ISD::SETCC: {
+ MVT VT0 = Node->getOperand(0).getValueType();
+ assert(isTypeLegal(TLI.getSetCCResultType(VT0))
&& "SetCC type is not legal??");
- Result = DAG.getNode(ISD::SETCC,
- TLI.getSetCCResultType(Node->getOperand(0)),
+ Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(VT0),
Node->getOperand(0), Node->getOperand(1),
Node->getOperand(2));
break;
-
+ }
case ISD::TRUNCATE:
switch (getTypeAction(Node->getOperand(0).getValueType())) {
case Legal:
break;
}
break;
+ case ISD::CONVERT_RNDSAT: {
+ ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode();
+ assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
+ CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
+ CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) &&
+ "can only promote integers");
+ Result = DAG.getConvertRndSat(NVT, Node->getOperand(0),
+ Node->getOperand(1), Node->getOperand(2),
+ Node->getOperand(3), Node->getOperand(4),
+ CvtCode);
+ break;
+
+ }
case ISD::BIT_CONVERT:
Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
Node->getValueType(0));
// FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not
// legal, such as PowerPC.
if (Node->getOpcode() == ISD::FP_TO_UINT &&
- !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
- (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) ||
+ !TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NVT) &&
+ (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT) ||
TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){
Result = DAG.getNode(ISD::FP_TO_SINT, NVT, Tmp1);
} else {
break;
}
- case ISD::ATOMIC_CMP_SWAP_8:
- case ISD::ATOMIC_CMP_SWAP_16:
- case ISD::ATOMIC_CMP_SWAP_32:
- case ISD::ATOMIC_CMP_SWAP_64: {
+ case ISD::ATOMIC_CMP_SWAP: {
AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node);
Tmp2 = PromoteOp(Node->getOperand(2));
Tmp3 = PromoteOp(Node->getOperand(3));
- Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(),
+ Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getMemoryVT(),
+ AtomNode->getChain(),
AtomNode->getBasePtr(), Tmp2, Tmp3,
AtomNode->getSrcValue(),
AtomNode->getAlignment());
AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
break;
}
- case ISD::ATOMIC_LOAD_ADD_8:
- case ISD::ATOMIC_LOAD_SUB_8:
- case ISD::ATOMIC_LOAD_AND_8:
- case ISD::ATOMIC_LOAD_OR_8:
- case ISD::ATOMIC_LOAD_XOR_8:
- case ISD::ATOMIC_LOAD_NAND_8:
- case ISD::ATOMIC_LOAD_MIN_8:
- case ISD::ATOMIC_LOAD_MAX_8:
- case ISD::ATOMIC_LOAD_UMIN_8:
- case ISD::ATOMIC_LOAD_UMAX_8:
- case ISD::ATOMIC_SWAP_8:
- case ISD::ATOMIC_LOAD_ADD_16:
- case ISD::ATOMIC_LOAD_SUB_16:
- case ISD::ATOMIC_LOAD_AND_16:
- case ISD::ATOMIC_LOAD_OR_16:
- case ISD::ATOMIC_LOAD_XOR_16:
- case ISD::ATOMIC_LOAD_NAND_16:
- case ISD::ATOMIC_LOAD_MIN_16:
- case ISD::ATOMIC_LOAD_MAX_16:
- case ISD::ATOMIC_LOAD_UMIN_16:
- case ISD::ATOMIC_LOAD_UMAX_16:
- case ISD::ATOMIC_SWAP_16:
- case ISD::ATOMIC_LOAD_ADD_32:
- case ISD::ATOMIC_LOAD_SUB_32:
- case ISD::ATOMIC_LOAD_AND_32:
- case ISD::ATOMIC_LOAD_OR_32:
- case ISD::ATOMIC_LOAD_XOR_32:
- case ISD::ATOMIC_LOAD_NAND_32:
- case ISD::ATOMIC_LOAD_MIN_32:
- case ISD::ATOMIC_LOAD_MAX_32:
- case ISD::ATOMIC_LOAD_UMIN_32:
- case ISD::ATOMIC_LOAD_UMAX_32:
- case ISD::ATOMIC_SWAP_32:
- case ISD::ATOMIC_LOAD_ADD_64:
- case ISD::ATOMIC_LOAD_SUB_64:
- case ISD::ATOMIC_LOAD_AND_64:
- case ISD::ATOMIC_LOAD_OR_64:
- case ISD::ATOMIC_LOAD_XOR_64:
- case ISD::ATOMIC_LOAD_NAND_64:
- case ISD::ATOMIC_LOAD_MIN_64:
- case ISD::ATOMIC_LOAD_MAX_64:
- case ISD::ATOMIC_LOAD_UMIN_64:
- case ISD::ATOMIC_LOAD_UMAX_64:
- case ISD::ATOMIC_SWAP_64: {
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_LOAD_MIN:
+ case ISD::ATOMIC_LOAD_MAX:
+ case ISD::ATOMIC_LOAD_UMIN:
+ case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_SWAP: {
AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node);
Tmp2 = PromoteOp(Node->getOperand(2));
- Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(),
+ Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getMemoryVT(),
+ AtomNode->getChain(),
AtomNode->getBasePtr(), Tmp2,
AtomNode->getSrcValue(),
AtomNode->getAlignment());
break;
case ISD::CTTZ:
// if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
- Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1,
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1.getValueType()), Tmp1,
DAG.getConstant(NVT.getSizeInBits(), NVT),
ISD::SETEQ);
Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
return Op;
}
break;
+ case TargetLowering::Promote:
+ assert(TVT.isVector() && "not vector type");
+ // fall thru to expand since vectors are by default are promote
case TargetLowering::Expand:
break;
}
DAG.getValueType(VT));
Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
DAG.getValueType(VT));
+ Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes.
+ Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes.
break;
}
}
Tmp2 = DAG.getConstant(0, MVT::i32);
CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1));
if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
- Tmp1 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2,
- CC);
+ Tmp1 = DAG.getNode(ISD::SETCC,
+ TLI.getSetCCResultType(Tmp1.getValueType()),
+ Tmp1, Tmp2, CC);
LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2).getNode(),
false /*sign irrelevant*/, Dummy);
- Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHS), LHS, Tmp2,
- DAG.getCondCode(TLI.getCmpLibcallCC(LC2)));
+ Tmp2 = DAG.getNode(ISD::SETCC,
+ TLI.getSetCCResultType(LHS.getValueType()), LHS,
+ Tmp2, DAG.getCondCode(TLI.getCmpLibcallCC(LC2)));
Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
Tmp2 = SDValue();
}
// BNE crN, L:
// FCMPU crN, lo1, lo2
// The following can be improved, but not that much.
- Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
- ISD::SETOEQ);
- Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, CCCode);
+ Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi, ISD::SETOEQ);
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo.getValueType()),
+ LHSLo, RHSLo, CCCode);
Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
- Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
- ISD::SETUNE);
- Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, CCCode);
+ Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi, ISD::SETUNE);
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi, CCCode);
Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3);
Tmp2 = SDValue();
// NOTE: on targets without efficient SELECT of bools, we can always use
// this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL);
- Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo,
- LowCC, false, DagCombineInfo);
+ Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()),
+ LHSLo, RHSLo, LowCC, false, DagCombineInfo);
if (!Tmp1.getNode())
- Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC);
- Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
- CCCode, false, DagCombineInfo);
+ Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo.getValueType()),
+ LHSLo, RHSLo, LowCC);
+ Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi, CCCode, false, DagCombineInfo);
if (!Tmp2.getNode())
- Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi,
- RHSHi,CC);
+ Tmp2 = DAG.getNode(ISD::SETCC,
+ TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi,CC);
ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode());
ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode());
Tmp1 = Tmp2;
Tmp2 = SDValue();
} else {
- Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
- ISD::SETEQ, false, DagCombineInfo);
+ Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi, ISD::SETEQ, false,
+ DagCombineInfo);
if (!Result.getNode())
- Result=DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
- ISD::SETEQ);
+ Result=DAG.getSetCC(TLI.getSetCCResultType(LHSHi.getValueType()),
+ LHSHi, RHSHi, ISD::SETEQ);
Result = LegalizeOp(DAG.getNode(ISD::SELECT, Tmp1.getValueType(),
Result, Tmp1, Tmp2));
Tmp1 = Result;
RHS = Tmp2;
}
+/// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and
+/// condition code CC on the current target. This routine assumes LHS and rHS
+/// have already been legalized by LegalizeSetCCOperands. It expands SETCC with
+/// illegal condition code into AND / OR of multiple SETCC values.
+void SelectionDAGLegalize::LegalizeSetCCCondCode(MVT VT,
+ SDValue &LHS, SDValue &RHS,
+ SDValue &CC) {
+ MVT OpVT = LHS.getValueType();
+ ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
+ switch (TLI.getCondCodeAction(CCCode, OpVT)) {
+ default: assert(0 && "Unknown condition code action!");
+ case TargetLowering::Legal:
+ // Nothing to do.
+ break;
+ case TargetLowering::Expand: {
+ ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
+ unsigned Opc = 0;
+ switch (CCCode) {
+ default: assert(0 && "Don't know how to expand this condition!"); abort();
+ case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO; Opc = ISD::AND; break;
+ case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO; Opc = ISD::AND; break;
+ case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO; Opc = ISD::AND; break;
+ case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO; Opc = ISD::AND; break;
+ case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO; Opc = ISD::AND; break;
+ case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO; Opc = ISD::AND; break;
+ case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR; break;
+ case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR; break;
+ case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
+ case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR; break;
+ case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
+ case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR; break;
+ // FIXME: Implement more expansions.
+ }
+
+ SDValue SetCC1 = DAG.getSetCC(VT, LHS, RHS, CC1);
+ SDValue SetCC2 = DAG.getSetCC(VT, LHS, RHS, CC2);
+ LHS = DAG.getNode(Opc, VT, SetCC1, SetCC2);
+ RHS = SDValue();
+ CC = SDValue();
+ break;
+ }
+ }
+}
+
/// EmitStackConvert - Emit a store/load combination to the stack. This stores
/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
/// a load from the stack slot to DestVT, extending it if needed.
FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
int SPFI = StackPtrFI->getIndex();
-
+ const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
+
unsigned SrcSize = SrcOp.getValueType().getSizeInBits();
unsigned SlotSize = SlotVT.getSizeInBits();
unsigned DestSize = DestVT.getSizeInBits();
if (SrcSize > SlotSize)
Store = DAG.getTruncStore(DAG.getEntryNode(), SrcOp, FIPtr,
- PseudoSourceValue::getFixedStack(SPFI), 0,
- SlotVT, false, SrcAlign);
+ SV, 0, SlotVT, false, SrcAlign);
else {
assert(SrcSize == SlotSize && "Invalid store");
Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr,
- PseudoSourceValue::getFixedStack(SPFI), 0,
- false, SrcAlign);
+ SV, 0, false, SrcAlign);
}
// Result is a load from the stack slot.
if (SlotSize == DestSize)
- return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0, false, DestAlign);
+ return DAG.getLoad(DestVT, Store, FIPtr, SV, 0, false, DestAlign);
assert(SlotSize < DestSize && "Unknown extension!");
- return DAG.getExtLoad(ISD::EXTLOAD, DestVT, Store, FIPtr, NULL, 0, SlotVT,
+ return DAG.getExtLoad(ISD::EXTLOAD, DestVT, Store, FIPtr, SV, 0, SlotVT,
false, DestAlign);
}
&MaskVec[0], MaskVec.size());
// If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it.
- if (TLI.isOperationLegal(ISD::SCALAR_TO_VECTOR, Node->getValueType(0)) &&
+ if (TLI.isOperationLegalOrCustom(ISD::SCALAR_TO_VECTOR,
+ Node->getValueType(0)) &&
isShuffleLegal(Node->getValueType(0), ShuffleMask)) {
Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val1);
Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val2);
MVT VT = Node->getValueType(0);
// Create the stack frame object.
SDValue FIPtr = DAG.CreateStackTemporary(VT);
-
+ int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
+ const Value *SV = PseudoSourceValue::getFixedStack(FI);
+
// Emit a store of each element to the stack slot.
SmallVector<SDValue, 8> Stores;
unsigned TypeByteSize = Node->getOperand(0).getValueType().getSizeInBits()/8;
Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx);
Stores.push_back(DAG.getStore(DAG.getEntryNode(), Node->getOperand(i), Idx,
- NULL, 0));
+ SV, Offset));
}
SDValue StoreChain;
StoreChain = DAG.getEntryNode();
// Result is a load from the stack slot.
- return DAG.getLoad(VT, StoreChain, FIPtr, NULL, 0);
+ return DAG.getLoad(VT, StoreChain, FIPtr, SV, 0);
}
void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp,
Args.push_back(Entry);
}
SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
- TLI.getPointerTy());
+ TLI.getPointerTy());
// Splice the libcall in wherever FindInputOutputChains tells us to.
const Type *RetTy = Node->getValueType(0).getTypeForMVT();
DestTy.getVectorNumElements() / 2);
SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Lo);
SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, Hi);
- return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult, HiResult));
+ return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, DestTy, LoResult,
+ HiResult));
}
// Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc.
Hi = Source;
}
+ // Check to see if the target has a custom way to lower this. If so, use
+ // it. (Note we've already expanded the operand in this case.)
+ switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) {
+ default: assert(0 && "This action not implemented for this operation!");
+ case TargetLowering::Legal:
+ case TargetLowering::Expand:
+ break; // This case is handled below.
+ case TargetLowering::Custom: {
+ SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, DestTy,
+ Source), DAG);
+ if (NV.getNode())
+ return LegalizeOp(NV);
+ break; // The target decided this was legal after all
+ }
+ }
+
// If this is unsigned, and not supported, first perform the conversion to
// signed, then adjust the result if the sign bit is set.
SDValue SignedConv = ExpandIntToFP(true, DestTy, Source);
- SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi,
- DAG.getConstant(0, Hi.getValueType()),
- ISD::SETLT);
+ SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi.getValueType()),
+ Hi, DAG.getConstant(0, Hi.getValueType()),
+ ISD::SETLT);
SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
SDValue CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
SignSet, Four, Zero);
assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0);
- SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Op0), Op0,
- DAG.getConstant(0, Op0.getValueType()),
- ISD::SETLT);
+ SDValue SignSet = DAG.getSetCC(TLI.getSetCCResultType(Op0.getValueType()),
+ Op0, DAG.getConstant(0, Op0.getValueType()),
+ ISD::SETLT);
SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
SDValue CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
SignSet, Four, Zero);
// the target lowering hooks to expand it. Just keep the low part of the
// expanded operation, we know that we're truncating anyway.
if (getTypeAction(NewOutTy) == Expand) {
- Operation = SDValue(TLI.ReplaceNodeResults(Operation.getNode(), DAG), 0);
- assert(Operation.getNode() && "Didn't return anything");
+ SmallVector<SDValue, 2> Results;
+ TLI.ReplaceNodeResults(Operation.getNode(), Results, DAG);
+ assert(Results.size() == 1 && "Incorrect FP_TO_XINT lowering!");
+ Operation = Results[0];
}
// Truncate the result of the extended FP_TO_*INT operation to the desired
unsigned len = VT.getSizeInBits();
for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
//x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8])
- SDValue Tmp2 = DAG.getConstant(mask[i], VT);
+ SDValue Tmp2 = DAG.getConstant(VT.getIntegerVTBitMask() & mask[i], VT);
SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT);
Op = DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2),
DAG.getNode(ISD::AND, VT,
SDValue Tmp3 = DAG.getConstant(1ULL << i, ShVT);
Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3));
}
- Op = DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(~0ULL, VT));
+ Op = DAG.getNOT(Op, VT);
return DAG.getNode(ISD::CTPOP, VT, Op);
}
case ISD::CTTZ: {
// { return 32 - nlz(~x & (x-1)); }
// see also http://www.hackersdelight.org/HDcode/ntz.cc
MVT VT = Op.getValueType();
- SDValue Tmp2 = DAG.getConstant(~0ULL, VT);
- SDValue Tmp3 = DAG.getNode(ISD::AND, VT,
- DAG.getNode(ISD::XOR, VT, Op, Tmp2),
+ SDValue Tmp3 = DAG.getNode(ISD::AND, VT, DAG.getNOT(Op, VT),
DAG.getNode(ISD::SUB, VT, Op, DAG.getConstant(1, VT)));
// If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
- if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
- TLI.isOperationLegal(ISD::CTLZ, VT))
+ if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
+ TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
return DAG.getNode(ISD::SUB, VT,
DAG.getConstant(VT.getSizeInBits(), VT),
DAG.getNode(ISD::CTLZ, VT, Tmp3));
/// ExpandOp - Expand the specified SDValue into its two component pieces
/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the
-/// LegalizeNodes map is filled in for any results that are not expanded, the
+/// LegalizedNodes map is filled in for any results that are not expanded, the
/// ExpandedNodes map is filled in for any results that are expanded, and the
/// Lo/Hi values are returned.
void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){
return ExpandOp(Hi, Lo, Hi);
return ExpandOp(Lo, Lo, Hi);
case ISD::EXTRACT_VECTOR_ELT:
- assert(VT==MVT::i64 && "Do not know how to expand this operator!");
// ExpandEXTRACT_VECTOR_ELT tolerates invalid result types.
Lo = ExpandEXTRACT_VECTOR_ELT(Op);
return ExpandOp(Lo, Lo, Hi);
case ISD::ConstantFP: {
ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
if (CFP->getValueType(0) == MVT::ppcf128) {
- APInt api = CFP->getValueAPF().convertToAPInt();
+ APInt api = CFP->getValueAPF().bitcastToAPInt();
Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])),
MVT::f64);
Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])),
ExpandOp(Node->getOperand(0), Lo, Hi);
SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT);
SDValue HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi);
- SDValue TopNotZero = DAG.getSetCC(TLI.getSetCCResultType(HLZ), HLZ, BitsC,
- ISD::SETNE);
+ SDValue TopNotZero = DAG.getSetCC(TLI.getSetCCResultType(NVT), HLZ, BitsC,
+ ISD::SETNE);
SDValue LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo);
LowPart = DAG.getNode(ISD::ADD, NVT, LowPart, BitsC);
ExpandOp(Node->getOperand(0), Lo, Hi);
SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT);
SDValue LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo);
- SDValue BotNotZero = DAG.getSetCC(TLI.getSetCCResultType(LTZ), LTZ, BitsC,
- ISD::SETNE);
+ SDValue BotNotZero = DAG.getSetCC(TLI.getSetCCResultType(NVT), LTZ, BitsC,
+ ISD::SETNE);
SDValue HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi);
HiPart = DAG.getNode(ISD::ADD, NVT, HiPart, BitsC);
break;
}
- // FIXME: should the LOAD_BIN and SWAP atomics get here too? Probably.
- case ISD::ATOMIC_CMP_SWAP_8:
- case ISD::ATOMIC_CMP_SWAP_16:
- case ISD::ATOMIC_CMP_SWAP_32:
- case ISD::ATOMIC_CMP_SWAP_64: {
+ case ISD::ATOMIC_CMP_SWAP: {
+ // This operation does not need a loop.
SDValue Tmp = TLI.LowerOperation(Op, DAG);
assert(Tmp.getNode() && "Node must be custom expanded!");
ExpandOp(Tmp.getValue(0), Lo, Hi);
break;
}
-
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_SWAP: {
+ // These operations require a loop to be generated. We can't do that yet,
+ // so substitute a target-dependent pseudo and expand that later.
+ SDValue In2Lo, In2Hi, In2;
+ ExpandOp(Op.getOperand(2), In2Lo, In2Hi);
+ In2 = DAG.getNode(ISD::BUILD_PAIR, VT, In2Lo, In2Hi);
+ AtomicSDNode* Anode = cast<AtomicSDNode>(Node);
+ SDValue Replace =
+ DAG.getAtomic(Op.getOpcode(), Anode->getMemoryVT(),
+ Op.getOperand(0), Op.getOperand(1), In2,
+ Anode->getSrcValue(), Anode->getAlignment());
+ SDValue Result = TLI.LowerOperation(Replace, DAG);
+ ExpandOp(Result.getValue(0), Lo, Hi);
+ // Remember that we legalized the chain.
+ AddLegalizedOperand(SDValue(Node,1), LegalizeOp(Result.getValue(1)));
+ break;
+ }
// These operators cannot be expanded directly, emit them as calls to
// library functions.
// If ADDC/ADDE are supported and if the shift amount is a constant 1, emit
// this X << 1 as X+X.
if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) {
- if (ShAmt->getAPIntValue() == 1 && TLI.isOperationLegal(ISD::ADDC, NVT) &&
- TLI.isOperationLegal(ISD::ADDE, NVT)) {
+ if (ShAmt->getAPIntValue() == 1 &&
+ TLI.isOperationLegalOrCustom(ISD::ADDC, NVT) &&
+ TLI.isOperationLegalOrCustom(ISD::ADDE, NVT)) {
SDValue LoOps[2], HiOps[3];
ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]);
SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag);
break;
}
}
-
// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
ExpandOp(Node->getOperand(0), LHSL, LHSH);
ExpandOp(Node->getOperand(1), RHSL, RHSH);
- SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
SDValue LoOps[2], HiOps[3];
LoOps[0] = LHSL;
LoOps[1] = RHSL;
HiOps[0] = LHSH;
HiOps[1] = RHSH;
- if (Node->getOpcode() == ISD::ADD) {
- Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
- HiOps[2] = Lo.getValue(1);
- Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
+
+ //cascaded check to see if any smaller size has a a carry flag.
+ unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC;
+ bool hasCarry = false;
+ for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) {
+ MVT AVT = MVT::getIntegerVT(BitSize);
+ if (TLI.isOperationLegalOrCustom(OpV, AVT)) {
+ hasCarry = true;
+ break;
+ }
+ }
+
+ if(hasCarry) {
+ SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
+ if (Node->getOpcode() == ISD::ADD) {
+ Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
+ } else {
+ Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
+ }
+ break;
} else {
- Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
- HiOps[2] = Lo.getValue(1);
- Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
+ if (Node->getOpcode() == ISD::ADD) {
+ Lo = DAG.getNode(ISD::ADD, NVT, LoOps, 2);
+ Hi = DAG.getNode(ISD::ADD, NVT, HiOps, 2);
+ SDValue Cmp1 = DAG.getSetCC(TLI.getSetCCResultType(NVT),
+ Lo, LoOps[0], ISD::SETULT);
+ SDValue Carry1 = DAG.getNode(ISD::SELECT, NVT, Cmp1,
+ DAG.getConstant(1, NVT),
+ DAG.getConstant(0, NVT));
+ SDValue Cmp2 = DAG.getSetCC(TLI.getSetCCResultType(NVT),
+ Lo, LoOps[1], ISD::SETULT);
+ SDValue Carry2 = DAG.getNode(ISD::SELECT, NVT, Cmp2,
+ DAG.getConstant(1, NVT),
+ Carry1);
+ Hi = DAG.getNode(ISD::ADD, NVT, Hi, Carry2);
+ } else {
+ Lo = DAG.getNode(ISD::SUB, NVT, LoOps, 2);
+ Hi = DAG.getNode(ISD::SUB, NVT, HiOps, 2);
+ SDValue Cmp = DAG.getSetCC(NVT, LoOps[0], LoOps[1], ISD::SETULT);
+ SDValue Borrow = DAG.getNode(ISD::SELECT, NVT, Cmp,
+ DAG.getConstant(1, NVT),
+ DAG.getConstant(0, NVT));
+ Hi = DAG.getNode(ISD::SUB, NVT, Hi, Borrow);
+ }
+ break;
}
- break;
}
case ISD::ADDC:
}
}
- bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
- bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
- bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT);
- bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT);
+ bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT);
+ bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT);
+ bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT);
+ bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT);
if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
SDValue LL, LH, RL, RH;
ExpandOp(Node->getOperand(0), LL, LH);
bool isNew =
ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
assert(isNew && "Value already expanded?!?");
+ isNew = isNew;
}
/// SplitVectorOp - Given an operand of vector type, break it down into
Lo = Node->getOperand(0);
Hi = Node->getOperand(1);
} else {
- SmallVector<SDValue, 8> LoOps(Node->op_begin(),
- Node->op_begin()+NewNumSubvectors);
+ SmallVector<SDValue, 8> LoOps(Node->op_begin(),
+ Node->op_begin()+NewNumSubvectors);
Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Lo, &LoOps[0], LoOps.size());
- SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumSubvectors,
+ SmallVector<SDValue, 8> HiOps(Node->op_begin()+NewNumSubvectors,
Node->op_end());
Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Hi, &HiOps[0], HiOps.size());
}
break;
}
+ case ISD::EXTRACT_SUBVECTOR: {
+ SDValue Vec = Op.getOperand(0);
+ SDValue Idx = Op.getOperand(1);
+ MVT IdxVT = Idx.getValueType();
+
+ Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Lo, Vec, Idx);
+ ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx);
+ if (CIdx) {
+ Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec,
+ DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo,
+ IdxVT));
+ } else {
+ Idx = DAG.getNode(ISD::ADD, IdxVT, Idx,
+ DAG.getConstant(NewNumElts_Lo, IdxVT));
+ Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, NewVT_Hi, Vec, Idx);
+ }
+ break;
+ }
case ISD::SELECT: {
SDValue Cond = Node->getOperand(0);
case ISD::XOR:
case ISD::UREM:
case ISD::SREM:
- case ISD::FREM: {
+ case ISD::FREM:
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL: {
SDValue LL, LH, RL, RH;
SplitVectorOp(Node->getOperand(0), LL, LH);
SplitVectorOp(Node->getOperand(1), RL, RH);
Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H);
break;
}
+ case ISD::CONVERT_RNDSAT: {
+ ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode();
+ SDValue L, H;
+ SplitVectorOp(Node->getOperand(0), L, H);
+ SDValue DTyOpL = DAG.getValueType(NewVT_Lo);
+ SDValue DTyOpH = DAG.getValueType(NewVT_Hi);
+ SDValue STyOpL = DAG.getValueType(L.getValueType());
+ SDValue STyOpH = DAG.getValueType(H.getValueType());
+
+ SDValue RndOp = Node->getOperand(3);
+ SDValue SatOp = Node->getOperand(4);
+
+ Lo = DAG.getConvertRndSat(NewVT_Lo, L, DTyOpL, STyOpL,
+ RndOp, SatOp, CvtCode);
+ Hi = DAG.getConvertRndSat(NewVT_Hi, H, DTyOpH, STyOpH,
+ RndOp, SatOp, CvtCode);
+ break;
+ }
case ISD::LOAD: {
LoadSDNode *LD = cast<LoadSDNode>(Node);
SDValue Ch = LD->getChain();
bool isNew =
SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
assert(isNew && "Value already split?!?");
+ isNew = isNew;
}
NewVT,
ScalarizeVectorOp(Node->getOperand(0)));
break;
+ case ISD::CONVERT_RNDSAT: {
+ SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0));
+ Result = DAG.getConvertRndSat(NewVT, Op0,
+ DAG.getValueType(NewVT),
+ DAG.getValueType(Op0.getValueType()),
+ Node->getOperand(3),
+ Node->getOperand(4),
+ cast<CvtRndSatSDNode>(Node)->getCvtCode());
+ break;
+ }
case ISD::FPOWI:
case ISD::FP_ROUND:
Result = DAG.getNode(Node->getOpcode(),
break;
}
case ISD::EXTRACT_SUBVECTOR:
- Result = Node->getOperand(0);
- assert(Result.getValueType() == NewVT);
+ Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewVT, Node->getOperand(0),
+ Node->getOperand(1));
break;
case ISD::BIT_CONVERT: {
SDValue Op0 = Op.getOperand(0);
case ISD::VSETCC: {
SDValue Op0 = ScalarizeVectorOp(Op.getOperand(0));
SDValue Op1 = ScalarizeVectorOp(Op.getOperand(1));
- Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Op0), Op0, Op1,
- Op.getOperand(2));
+ Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Op0.getValueType()),
+ Op0, Op1, Op.getOperand(2));
Result = DAG.getNode(ISD::SELECT, NewVT, Result,
DAG.getConstant(-1ULL, NewVT),
DAG.getConstant(0ULL, NewVT));
Result = LegalizeOp(Result);
bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second;
assert(isNew && "Value already scalarized?");
+ isNew = isNew;
+ return Result;
+}
+
+
+SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
+ std::map<SDValue, SDValue>::iterator I = WidenNodes.find(Op);
+ if (I != WidenNodes.end()) return I->second;
+
+ MVT VT = Op.getValueType();
+ assert(VT.isVector() && "Cannot widen non-vector type!");
+
+ SDValue Result;
+ SDNode *Node = Op.getNode();
+ MVT EVT = VT.getVectorElementType();
+
+ unsigned NumElts = VT.getVectorNumElements();
+ unsigned NewNumElts = WidenVT.getVectorNumElements();
+ assert(NewNumElts > NumElts && "Cannot widen to smaller type!");
+ assert(NewNumElts < 17);
+
+ // When widen is called, it is assumed that it is more efficient to use a
+ // wide type. The default action is to widen to operation to a wider legal
+ // vector type and then do the operation if it is legal by calling LegalizeOp
+ // again. If there is no vector equivalent, we will unroll the operation, do
+ // it, and rebuild the vector. If most of the operations are vectorizible to
+ // the legal type, the resulting code will be more efficient. If this is not
+ // the case, the resulting code will preform badly as we end up generating
+ // code to pack/unpack the results. It is the function that calls widen
+ // that is responsible for seeing this doesn't happen.
+ switch (Node->getOpcode()) {
+ default:
+#ifndef NDEBUG
+ Node->dump(&DAG);
+#endif
+ assert(0 && "Unexpected operation in WidenVectorOp!");
+ break;
+ case ISD::CopyFromReg:
+ assert(0 && "CopyFromReg doesn't need widening!");
+ case ISD::Constant:
+ case ISD::ConstantFP:
+ // To build a vector of these elements, clients should call BuildVector
+ // and with each element instead of creating a node with a vector type
+ assert(0 && "Unexpected operation in WidenVectorOp!");
+ case ISD::VAARG:
+ // Variable Arguments with vector types doesn't make any sense to me
+ assert(0 && "Unexpected operation in WidenVectorOp!");
+ break;
+ case ISD::UNDEF:
+ Result = DAG.getNode(ISD::UNDEF, WidenVT);
+ break;
+ case ISD::BUILD_VECTOR: {
+ // Build a vector with undefined for the new nodes
+ SDValueVector NewOps(Node->op_begin(), Node->op_end());
+ for (unsigned i = NumElts; i < NewNumElts; ++i) {
+ NewOps.push_back(DAG.getNode(ISD::UNDEF,EVT));
+ }
+ Result = DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &NewOps[0], NewOps.size());
+ break;
+ }
+ case ISD::INSERT_VECTOR_ELT: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, WidenVT, Tmp1,
+ Node->getOperand(1), Node->getOperand(2));
+ break;
+ }
+ case ISD::VECTOR_SHUFFLE: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT);
+ // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is
+ // used as permutation array. We build the vector here instead of widening
+ // because we don't want to legalize and have it turned to something else.
+ SDValue PermOp = Node->getOperand(2);
+ SDValueVector NewOps;
+ MVT PVT = PermOp.getValueType().getVectorElementType();
+ for (unsigned i = 0; i < NumElts; ++i) {
+ if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) {
+ NewOps.push_back(PermOp.getOperand(i));
+ } else {
+ unsigned Idx =
+ cast<ConstantSDNode>(PermOp.getOperand(i))->getZExtValue();
+ if (Idx < NumElts) {
+ NewOps.push_back(PermOp.getOperand(i));
+ }
+ else {
+ NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts,
+ PermOp.getOperand(i).getValueType()));
+ }
+ }
+ }
+ for (unsigned i = NumElts; i < NewNumElts; ++i) {
+ NewOps.push_back(DAG.getNode(ISD::UNDEF,PVT));
+ }
+
+ SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR,
+ MVT::getVectorVT(PVT, NewOps.size()),
+ &NewOps[0], NewOps.size());
+
+ Result = DAG.getNode(ISD::VECTOR_SHUFFLE, WidenVT, Tmp1, Tmp2, Tmp3);
+ break;
+ }
+ case ISD::LOAD: {
+ // If the load widen returns true, we can use a single load for the
+ // vector. Otherwise, it is returning a token factor for multiple
+ // loads.
+ SDValue TFOp;
+ if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT))
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1)));
+ else
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0)));
+ break;
+ }
+
+ case ISD::BIT_CONVERT: {
+ SDValue Tmp1 = Node->getOperand(0);
+ // Converts between two different types so we need to determine
+ // the correct widen type for the input operand.
+ MVT InVT = Tmp1.getValueType();
+ unsigned WidenSize = WidenVT.getSizeInBits();
+ if (InVT.isVector()) {
+ MVT InEltVT = InVT.getVectorElementType();
+ unsigned InEltSize = InEltVT.getSizeInBits();
+ assert(WidenSize % InEltSize == 0 &&
+ "can not widen bit convert that are not multiple of element type");
+ MVT NewInWidenVT = MVT::getVectorVT(InEltVT, WidenSize / InEltSize);
+ Tmp1 = WidenVectorOp(Tmp1, NewInWidenVT);
+ assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits());
+ Result = DAG.getNode(ISD::BIT_CONVERT, WidenVT, Tmp1);
+ } else {
+ // If the result size is a multiple of the input size, widen the input
+ // and then convert.
+ unsigned InSize = InVT.getSizeInBits();
+ assert(WidenSize % InSize == 0 &&
+ "can not widen bit convert that are not multiple of element type");
+ unsigned NewNumElts = WidenSize / InSize;
+ SmallVector<SDValue, 16> Ops(NewNumElts);
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, InVT);
+ Ops[0] = Tmp1;
+ for (unsigned i = 1; i < NewNumElts; ++i)
+ Ops[i] = UndefVal;
+
+ MVT NewInVT = MVT::getVectorVT(InVT, NewNumElts);
+ Result = DAG.getNode(ISD::BUILD_VECTOR, NewInVT, &Ops[0], NewNumElts);
+ Result = DAG.getNode(ISD::BIT_CONVERT, WidenVT, Result);
+ }
+ break;
+ }
+
+ case ISD::SINT_TO_FP:
+ case ISD::UINT_TO_FP:
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT:
+ case ISD::FP_ROUND: {
+ SDValue Tmp1 = Node->getOperand(0);
+ // Converts between two different types so we need to determine
+ // the correct widen type for the input operand.
+ MVT TVT = Tmp1.getValueType();
+ assert(TVT.isVector() && "can not widen non vector type");
+ MVT TEVT = TVT.getVectorElementType();
+ MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts);
+ Tmp1 = WidenVectorOp(Tmp1, TWidenVT);
+ assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
+ break;
+ }
+
+ case ISD::FP_EXTEND:
+ assert(0 && "Case not implemented. Dynamically dead with 2 FP types!");
+ case ISD::TRUNCATE:
+ case ISD::SIGN_EXTEND:
+ case ISD::ZERO_EXTEND:
+ case ISD::ANY_EXTEND:
+ case ISD::SIGN_EXTEND_INREG:
+ case ISD::FABS:
+ case ISD::FNEG:
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS:
+ case ISD::CTPOP:
+ case ISD::CTTZ:
+ case ISD::CTLZ: {
+ // Unary op widening
+ SDValue Tmp1;
+ Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
+ break;
+ }
+ case ISD::CONVERT_RNDSAT: {
+ SDValue RndOp = Node->getOperand(3);
+ SDValue SatOp = Node->getOperand(4);
+ SDValue SrcOp = Node->getOperand(0);
+
+ // Converts between two different types so we need to determine
+ // the correct widen type for the input operand.
+ MVT SVT = SrcOp.getValueType();
+ assert(SVT.isVector() && "can not widen non vector type");
+ MVT SEVT = SVT.getVectorElementType();
+ MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts);
+
+ SrcOp = WidenVectorOp(SrcOp, SWidenVT);
+ assert(SrcOp.getValueType() == WidenVT);
+ SDValue DTyOp = DAG.getValueType(WidenVT);
+ SDValue STyOp = DAG.getValueType(SrcOp.getValueType());
+ ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode();
+
+ Result = DAG.getConvertRndSat(WidenVT, SrcOp, DTyOp, STyOp,
+ RndOp, SatOp, CvtCode);
+ break;
+ }
+ case ISD::FPOW:
+ case ISD::FPOWI:
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ case ISD::MULHS:
+ case ISD::MULHU:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::SDIV:
+ case ISD::SREM:
+ case ISD::FDIV:
+ case ISD::FREM:
+ case ISD::FCOPYSIGN:
+ case ISD::UDIV:
+ case ISD::UREM:
+ case ISD::BSWAP: {
+ // Binary op widening
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2);
+ break;
+ }
+
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT);
+ SDValue ShOp = Node->getOperand(1);
+ MVT ShVT = ShOp.getValueType();
+ MVT NewShVT = MVT::getVectorVT(ShVT.getVectorElementType(),
+ WidenVT.getVectorNumElements());
+ ShOp = WidenVectorOp(ShOp, NewShVT);
+ assert(ShOp.getValueType() == NewShVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, ShOp);
+ break;
+ }
+
+ case ISD::EXTRACT_VECTOR_ELT: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), EVT, Tmp1, Node->getOperand(1));
+ break;
+ }
+ case ISD::CONCAT_VECTORS: {
+ // We concurrently support only widen on a multiple of the incoming vector.
+ // We could widen on a multiple of the incoming operand if necessary.
+ unsigned NumConcat = NewNumElts / NumElts;
+ assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector");
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT);
+ SmallVector<SDValue, 8> MOps;
+ MOps.push_back(Op);
+ for (unsigned i = 1; i != NumConcat; ++i) {
+ MOps.push_back(UndefVal);
+ }
+ Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT,
+ &MOps[0], MOps.size()));
+ break;
+ }
+ case ISD::EXTRACT_SUBVECTOR: {
+ SDValue Tmp1 = Node->getOperand(0);
+ SDValue Idx = Node->getOperand(1);
+ ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx);
+ if (CIdx && CIdx->getZExtValue() == 0) {
+ // Since we are access the start of the vector, the incoming
+ // vector type might be the proper.
+ MVT Tmp1VT = Tmp1.getValueType();
+ if (Tmp1VT == WidenVT)
+ return Tmp1;
+ else {
+ unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements();
+ if (Tmp1VTNumElts < NewNumElts)
+ Result = WidenVectorOp(Tmp1, WidenVT);
+ else
+ Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, WidenVT, Tmp1, Idx);
+ }
+ } else if (NewNumElts % NumElts == 0) {
+ // Widen the extracted subvector.
+ unsigned NumConcat = NewNumElts / NumElts;
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT);
+ SmallVector<SDValue, 8> MOps;
+ MOps.push_back(Op);
+ for (unsigned i = 1; i != NumConcat; ++i) {
+ MOps.push_back(UndefVal);
+ }
+ Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT,
+ &MOps[0], MOps.size()));
+ } else {
+ assert(0 && "can not widen extract subvector");
+ // This could be implemented using insert and build vector but I would
+ // like to see when this happens.
+ }
+ break;
+ }
+
+ case ISD::SELECT: {
+ // Determine new condition widen type and widen
+ SDValue Cond1 = Node->getOperand(0);
+ MVT CondVT = Cond1.getValueType();
+ assert(CondVT.isVector() && "can not widen non vector type");
+ MVT CondEVT = CondVT.getVectorElementType();
+ MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts);
+ Cond1 = WidenVectorOp(Cond1, CondWidenVT);
+ assert(Cond1.getValueType() == CondWidenVT && "Condition not widen");
+
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Tmp1, Tmp2);
+ break;
+ }
+
+ case ISD::SELECT_CC: {
+ // Determine new condition widen type and widen
+ SDValue Cond1 = Node->getOperand(0);
+ SDValue Cond2 = Node->getOperand(1);
+ MVT CondVT = Cond1.getValueType();
+ assert(CondVT.isVector() && "can not widen non vector type");
+ assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs");
+ MVT CondEVT = CondVT.getVectorElementType();
+ MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts);
+ Cond1 = WidenVectorOp(Cond1, CondWidenVT);
+ Cond2 = WidenVectorOp(Cond2, CondWidenVT);
+ assert(Cond1.getValueType() == CondWidenVT &&
+ Cond2.getValueType() == CondWidenVT && "condition not widen");
+
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT &&
+ "operands not widen");
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Cond2, Tmp1,
+ Tmp2, Node->getOperand(4));
+ break;
+ }
+ case ISD::VSETCC: {
+ // Determine widen for the operand
+ SDValue Tmp1 = Node->getOperand(0);
+ MVT TmpVT = Tmp1.getValueType();
+ assert(TmpVT.isVector() && "can not widen non vector type");
+ MVT TmpEVT = TmpVT.getVectorElementType();
+ MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts);
+ Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2,
+ Node->getOperand(2));
+ break;
+ }
+ case ISD::ATOMIC_CMP_SWAP:
+ case ISD::ATOMIC_LOAD_ADD:
+ case ISD::ATOMIC_LOAD_SUB:
+ case ISD::ATOMIC_LOAD_AND:
+ case ISD::ATOMIC_LOAD_OR:
+ case ISD::ATOMIC_LOAD_XOR:
+ case ISD::ATOMIC_LOAD_NAND:
+ case ISD::ATOMIC_LOAD_MIN:
+ case ISD::ATOMIC_LOAD_MAX:
+ case ISD::ATOMIC_LOAD_UMIN:
+ case ISD::ATOMIC_LOAD_UMAX:
+ case ISD::ATOMIC_SWAP: {
+ // For now, we assume that using vectors for these operations don't make
+ // much sense so we just split it. We return an empty result
+ SDValue X, Y;
+ SplitVectorOp(Op, X, Y);
+ return Result;
+ break;
+ }
+
+ } // end switch (Node->getOpcode())
+
+ assert(Result.getNode() && "Didn't set a result!");
+ if (Result != Op)
+ Result = LegalizeOp(Result);
+
+ AddWidenedOperand(Op, Result);
return Result;
}
+// Utility function to find a legal vector type and its associated element
+// type from a preferred width and whose vector type must be the same size
+// as the VVT.
+// TLI: Target lowering used to determine legal types
+// Width: Preferred width of element type
+// VVT: Vector value type whose size we must match.
+// Returns VecEVT and EVT - the vector type and its associated element type
+static void FindWidenVecType(const TargetLowering &TLI, unsigned Width, MVT VVT,
+ MVT& EVT, MVT& VecEVT) {
+ // We start with the preferred width, make it a power of 2 and see if
+ // we can find a vector type of that width. If not, we reduce it by
+ // another power of 2. If we have widen the type, a vector of bytes should
+ // always be legal.
+ assert(TLI.isTypeLegal(VVT));
+ unsigned EWidth = Width + 1;
+ do {
+ assert(EWidth > 0);
+ EWidth = (1 << Log2_32(EWidth-1));
+ EVT = MVT::getIntegerVT(EWidth);
+ unsigned NumEVT = VVT.getSizeInBits()/EWidth;
+ VecEVT = MVT::getVectorVT(EVT, NumEVT);
+ } while (!TLI.isTypeLegal(VecEVT) ||
+ VVT.getSizeInBits() != VecEVT.getSizeInBits());
+}
+
+SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain,
+ SDValue Chain,
+ SDValue BasePtr,
+ const Value *SV,
+ int SVOffset,
+ unsigned Alignment,
+ bool isVolatile,
+ unsigned LdWidth,
+ MVT ResType) {
+ // We assume that we have good rules to handle loading power of two loads so
+ // we break down the operations to power of 2 loads. The strategy is to
+ // load the largest power of 2 that we can easily transform to a legal vector
+ // and then insert into that vector, and the cast the result into the legal
+ // vector that we want. This avoids unnecessary stack converts.
+ // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and
+ // the load is nonvolatile, we an use a wider load for the value.
+ // Find a vector length we can load a large chunk
+ MVT EVT, VecEVT;
+ unsigned EVTWidth;
+ FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+
+ SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, SVOffset,
+ isVolatile, Alignment);
+ SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecEVT, LdOp);
+ LdChain.push_back(LdOp.getValue(1));
+
+ // Check if we can load the element with one instruction
+ if (LdWidth == EVTWidth) {
+ return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp);
+ }
+
+ // The vector element order is endianness dependent.
+ unsigned Idx = 1;
+ LdWidth -= EVTWidth;
+ unsigned Offset = 0;
+
+ while (LdWidth > 0) {
+ unsigned Increment = EVTWidth / 8;
+ Offset += Increment;
+ BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr,
+ DAG.getIntPtrConstant(Increment));
+
+ if (LdWidth < EVTWidth) {
+ // Our current type we are using is too large, use a smaller size by
+ // using a smaller power of 2
+ unsigned oEVTWidth = EVTWidth;
+ FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+ // Readjust position and vector position based on new load type
+ Idx = Idx * (oEVTWidth/EVTWidth);
+ VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp);
+ }
+
+ SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV,
+ SVOffset+Offset, isVolatile,
+ MinAlign(Alignment, Offset));
+ LdChain.push_back(LdOp.getValue(1));
+ VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, VecEVT, VecOp, LdOp,
+ DAG.getIntPtrConstant(Idx++));
+
+ LdWidth -= EVTWidth;
+ }
+
+ return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp);
+}
+
+bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result,
+ SDValue& TFOp,
+ SDValue Op,
+ MVT NVT) {
+ // TODO: Add support for ConcatVec and the ability to load many vector
+ // types (e.g., v4i8). This will not work when a vector register
+ // to memory mapping is strange (e.g., vector elements are not
+ // stored in some sequential order).
+
+ // It must be true that the widen vector type is bigger than where
+ // we need to load from.
+ LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
+ MVT LdVT = LD->getMemoryVT();
+ assert(LdVT.isVector() && NVT.isVector());
+ assert(LdVT.getVectorElementType() == NVT.getVectorElementType());
+
+ // Load information
+ SDValue Chain = LD->getChain();
+ SDValue BasePtr = LD->getBasePtr();
+ int SVOffset = LD->getSrcValueOffset();
+ unsigned Alignment = LD->getAlignment();
+ bool isVolatile = LD->isVolatile();
+ const Value *SV = LD->getSrcValue();
+ unsigned int LdWidth = LdVT.getSizeInBits();
+
+ // Load value as a large register
+ SDValueVector LdChain;
+ Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset,
+ Alignment, isVolatile, LdWidth, NVT);
+
+ if (LdChain.size() == 1) {
+ TFOp = LdChain[0];
+ return true;
+ }
+ else {
+ TFOp=DAG.getNode(ISD::TokenFactor, MVT::Other, &LdChain[0], LdChain.size());
+ return false;
+ }
+}
+
+
+void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain,
+ SDValue Chain,
+ SDValue BasePtr,
+ const Value *SV,
+ int SVOffset,
+ unsigned Alignment,
+ bool isVolatile,
+ SDValue ValOp,
+ unsigned StWidth) {
+ // Breaks the stores into a series of power of 2 width stores. For any
+ // width, we convert the vector to the vector of element size that we
+ // want to store. This avoids requiring a stack convert.
+
+ // Find a width of the element type we can store with
+ MVT VVT = ValOp.getValueType();
+ MVT EVT, VecEVT;
+ unsigned EVTWidth;
+ FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+
+ SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, ValOp);
+ SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp,
+ DAG.getIntPtrConstant(0));
+ SDValue StOp = DAG.getStore(Chain, EOp, BasePtr, SV, SVOffset,
+ isVolatile, Alignment);
+ StChain.push_back(StOp);
+
+ // Check if we are done
+ if (StWidth == EVTWidth) {
+ return;
+ }
+
+ unsigned Idx = 1;
+ StWidth -= EVTWidth;
+ unsigned Offset = 0;
+
+ while (StWidth > 0) {
+ unsigned Increment = EVTWidth / 8;
+ Offset += Increment;
+ BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr,
+ DAG.getIntPtrConstant(Increment));
+
+ if (StWidth < EVTWidth) {
+ // Our current type we are using is too large, use a smaller size by
+ // using a smaller power of 2
+ unsigned oEVTWidth = EVTWidth;
+ FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+ // Readjust position and vector position based on new load type
+ Idx = Idx * (oEVTWidth/EVTWidth);
+ VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp);
+ }
+
+ EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp,
+ DAG.getIntPtrConstant(Idx++));
+ StChain.push_back(DAG.getStore(Chain, EOp, BasePtr, SV,
+ SVOffset + Offset, isVolatile,
+ MinAlign(Alignment, Offset)));
+ StWidth -= EVTWidth;
+ }
+}
+
+
+SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST,
+ SDValue Chain,
+ SDValue BasePtr) {
+ // TODO: It might be cleaner if we can use SplitVector and have more legal
+ // vector types that can be stored into memory (e.g., v4xi8 can
+ // be stored as a word). This will not work when a vector register
+ // to memory mapping is strange (e.g., vector elements are not
+ // stored in some sequential order).
+
+ MVT StVT = ST->getMemoryVT();
+ SDValue ValOp = ST->getValue();
+
+ // Check if we have widen this node with another value
+ std::map<SDValue, SDValue>::iterator I = WidenNodes.find(ValOp);
+ if (I != WidenNodes.end())
+ ValOp = I->second;
+
+ MVT VVT = ValOp.getValueType();
+
+ // It must be true that we the widen vector type is bigger than where
+ // we need to store.
+ assert(StVT.isVector() && VVT.isVector());
+ assert(StVT.bitsLT(VVT));
+ assert(StVT.getVectorElementType() == VVT.getVectorElementType());
+
+ // Store value
+ SDValueVector StChain;
+ genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(),
+ ST->getSrcValueOffset(), ST->getAlignment(),
+ ST->isVolatile(), ValOp, StVT.getSizeInBits());
+ if (StChain.size() == 1)
+ return StChain[0];
+ else
+ return DAG.getNode(ISD::TokenFactor, MVT::Other,&StChain[0],StChain.size());
+}
+
// SelectionDAG::Legalize - This is the entry point for the file.
//
-void SelectionDAG::Legalize() {
+void SelectionDAG::Legalize(bool TypesNeedLegalizing) {
/// run - This is the main entry point to this class.
///
- SelectionDAGLegalize(*this).LegalizeDAG();
+ SelectionDAGLegalize(*this, TypesNeedLegalizing).LegalizeDAG();
}
projects / oota-llvm.git / blobdiff