#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Target/TargetFrameInfo.h"
#include "llvm/Target/TargetLowering.h"
#include "llvm/Target/TargetData.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/CallingConv.h"
#include "llvm/Constants.h"
#include "llvm/DerivedTypes.h"
-#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/Alignment.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SmallPtrSet.h"
static const bool ViewLegalizeDAGs = 0;
#endif
-namespace llvm {
-template<>
-struct DenseMapKeyInfo<SDOperand> {
- static inline SDOperand getEmptyKey() { return SDOperand((SDNode*)-1, -1U); }
- static inline SDOperand getTombstoneKey() { return SDOperand((SDNode*)-1, 0);}
- static unsigned getHashValue(const SDOperand &Val) {
- return DenseMapKeyInfo<void*>::getHashValue(Val.Val) + Val.ResNo;
- }
- static bool isPod() { return true; }
-};
-}
-
//===----------------------------------------------------------------------===//
/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and
/// hacks on it until the target machine can handle it. This involves
/// to avoid splitting the same node more than once.
std::map<SDOperand, std::pair<SDOperand, SDOperand> > SplitNodes;
- /// PackedNodes - For nodes that need to be packed from MVT::Vector types to
- /// concrete vector types, this contains the mapping of ones we have already
+ /// ScalarizedNodes - For nodes that need to be converted from vector types to
+ /// scalar types, this contains the mapping of ones we have already
/// processed to the result.
- std::map<SDOperand, SDOperand> PackedNodes;
+ std::map<SDOperand, SDOperand> ScalarizedNodes;
void AddLegalizedOperand(SDOperand From, SDOperand To) {
LegalizedNodes.insert(std::make_pair(From, To));
void LegalizeDAG();
private:
- /// HandleOp - Legalize, Promote, Expand or Pack the specified operand as
+ /// HandleOp - Legalize, Promote, or Expand the specified operand as
/// appropriate for its type.
void HandleOp(SDOperand Op);
/// result.
SDOperand LegalizeOp(SDOperand O);
+ /// UnrollVectorOp - We know that the given vector has a legal type, however
+ /// the operation it performs is not legal and is an operation that we have
+ /// no way of lowering. "Unroll" the vector, splitting out the scalars and
+ /// operating on each element individually.
+ SDOperand UnrollVectorOp(SDOperand O);
+
/// PromoteOp - Given an operation that produces a value in an invalid type,
/// promote it to compute the value into a larger type. The produced value
/// will have the correct bits for the low portion of the register, but no
/// types.
void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
- /// SplitVectorOp - Given an operand of MVT::Vector type, break it down into
- /// two smaller values of MVT::Vector type.
+ /// SplitVectorOp - Given an operand of vector type, break it down into
+ /// two smaller values.
void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
- /// PackVectorOp - Given an operand of MVT::Vector type, convert it into the
- /// equivalent operation that returns a packed value (e.g. MVT::V4F32). When
- /// this is called, we know that PackedVT is the right type for the result and
- /// we know that this type is legal for the target.
- SDOperand PackVectorOp(SDOperand O, MVT::ValueType PackedVT);
+ /// ScalarizeVectorOp - Given an operand of single-element vector type
+ /// (e.g. v1f32), convert it into the equivalent operation that returns a
+ /// scalar (e.g. f32) value.
+ SDOperand ScalarizeVectorOp(SDOperand O);
/// isShuffleLegal - Return true if a vector shuffle is legal with the
/// specified mask and type. Targets can specify exactly which masks they
void LegalizeSetCCOperands(SDOperand &LHS, SDOperand &RHS, SDOperand &CC);
- SDOperand CreateStackTemporary(MVT::ValueType VT);
-
SDOperand ExpandLibCall(const char *Name, SDNode *Node, bool isSigned,
SDOperand &Hi);
SDOperand ExpandIntToFP(bool isSigned, MVT::ValueType DestTy,
void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt,
SDOperand &Lo, SDOperand &Hi);
- SDOperand LowerVEXTRACT_VECTOR_ELT(SDOperand Op);
+ SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op);
SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op);
SDOperand getIntPtrConstant(uint64_t Val) {
return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0;
}
-/// getScalarizedOpcode - Return the scalar opcode that corresponds to the
-/// specified vector opcode.
-static unsigned getScalarizedOpcode(unsigned VecOp, MVT::ValueType VT) {
- switch (VecOp) {
- default: assert(0 && "Don't know how to scalarize this opcode!");
- case ISD::VADD: return MVT::isInteger(VT) ? ISD::ADD : ISD::FADD;
- case ISD::VSUB: return MVT::isInteger(VT) ? ISD::SUB : ISD::FSUB;
- case ISD::VMUL: return MVT::isInteger(VT) ? ISD::MUL : ISD::FMUL;
- case ISD::VSDIV: return MVT::isInteger(VT) ? ISD::SDIV: ISD::FDIV;
- case ISD::VUDIV: return MVT::isInteger(VT) ? ISD::UDIV: ISD::FDIV;
- case ISD::VAND: return MVT::isInteger(VT) ? ISD::AND : 0;
- case ISD::VOR: return MVT::isInteger(VT) ? ISD::OR : 0;
- case ISD::VXOR: return MVT::isInteger(VT) ? ISD::XOR : 0;
- }
-}
-
SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
: TLI(dag.getTargetLoweringInfo()), DAG(dag),
ValueTypeActions(TLI.getValueTypeActions()) {
"Too many value types for ValueTypeActions to hold!");
}
-/// ComputeTopDownOrdering - Add the specified node to the Order list if it has
-/// not been visited yet and if all of its operands have already been visited.
-static void ComputeTopDownOrdering(SDNode *N, SmallVector<SDNode*, 64> &Order,
- DenseMap<SDNode*, unsigned> &Visited) {
- if (++Visited[N] != N->getNumOperands())
- return; // Haven't visited all operands yet
-
- Order.push_back(N);
+/// ComputeTopDownOrdering - Compute a top-down ordering of the dag, where Order
+/// contains all of a nodes operands before it contains the node.
+static void ComputeTopDownOrdering(SelectionDAG &DAG,
+ SmallVector<SDNode*, 64> &Order) {
+
+ DenseMap<SDNode*, unsigned> Visited;
+ std::vector<SDNode*> Worklist;
+ Worklist.reserve(128);
- if (N->hasOneUse()) { // Tail recurse in common case.
- ComputeTopDownOrdering(*N->use_begin(), Order, Visited);
- return;
+ // Compute ordering from all of the leaves in the graphs, those (like the
+ // entry node) that have no operands.
+ for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
+ E = DAG.allnodes_end(); I != E; ++I) {
+ if (I->getNumOperands() == 0) {
+ Visited[I] = 0 - 1U;
+ Worklist.push_back(I);
+ }
}
- // Now that we have N in, add anything that uses it if all of their operands
- // are now done.
- for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E;++UI)
- ComputeTopDownOrdering(*UI, Order, Visited);
+ while (!Worklist.empty()) {
+ SDNode *N = Worklist.back();
+ Worklist.pop_back();
+
+ if (++Visited[N] != N->getNumOperands())
+ continue; // Haven't visited all operands yet
+
+ Order.push_back(N);
+
+ // Now that we have N in, add anything that uses it if all of their operands
+ // are now done.
+ for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
+ UI != E; ++UI)
+ Worklist.push_back(*UI);
+ }
+
+ assert(Order.size() == Visited.size() &&
+ Order.size() ==
+ (unsigned)std::distance(DAG.allnodes_begin(), DAG.allnodes_end()) &&
+ "Error: DAG is cyclic!");
}
// practice however, this causes us to run out of stack space on large basic
// blocks. To avoid this problem, compute an ordering of the nodes where each
// node is only legalized after all of its operands are legalized.
- DenseMap<SDNode*, unsigned> Visited;
SmallVector<SDNode*, 64> Order;
-
- // Compute ordering from all of the leaves in the graphs, those (like the
- // entry node) that have no operands.
- for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
- E = DAG.allnodes_end(); I != E; ++I) {
- if (I->getNumOperands() == 0) {
- Visited[I] = 0 - 1U;
- ComputeTopDownOrdering(I, Order, Visited);
- }
- }
-
- assert(Order.size() == Visited.size() &&
- Order.size() ==
- (unsigned)std::distance(DAG.allnodes_begin(), DAG.allnodes_end()) &&
- "Error: DAG is cyclic!");
- Visited.clear();
+ ComputeTopDownOrdering(DAG, Order);
for (unsigned i = 0, e = Order.size(); i != e; ++i)
HandleOp(SDOperand(Order[i], 0));
LegalizedNodes.clear();
PromotedNodes.clear();
SplitNodes.clear();
- PackedNodes.clear();
+ ScalarizedNodes.clear();
// Remove dead nodes now.
DAG.RemoveDeadNodes();
return false;
}
-/// HandleOp - Legalize, Promote, Expand or Pack the specified operand as
+/// HandleOp - Legalize, Promote, or Expand the specified operand as
/// appropriate for its type.
void SelectionDAGLegalize::HandleOp(SDOperand Op) {
- switch (getTypeAction(Op.getValueType())) {
+ MVT::ValueType VT = Op.getValueType();
+ switch (getTypeAction(VT)) {
default: assert(0 && "Bad type action!");
- case Legal: LegalizeOp(Op); break;
- case Promote: PromoteOp(Op); break;
+ case Legal: (void)LegalizeOp(Op); break;
+ case Promote: (void)PromoteOp(Op); break;
case Expand:
- if (Op.getValueType() != MVT::Vector) {
+ if (!MVT::isVector(VT)) {
+ // If this is an illegal scalar, expand it into its two component
+ // pieces.
SDOperand X, Y;
+ if (Op.getOpcode() == ISD::TargetConstant)
+ break; // Allow illegal target nodes.
ExpandOp(Op, X, Y);
+ } else if (MVT::getVectorNumElements(VT) == 1) {
+ // If this is an illegal single element vector, convert it to a
+ // scalar operation.
+ (void)ScalarizeVectorOp(Op);
} else {
- SDNode *N = Op.Val;
- unsigned NumOps = N->getNumOperands();
- unsigned NumElements =
- cast<ConstantSDNode>(N->getOperand(NumOps-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(N->getOperand(NumOps-1))->getVT();
- MVT::ValueType PackedVT = getVectorType(EVT, NumElements);
- if (PackedVT != MVT::Other && TLI.isTypeLegal(PackedVT)) {
- // In the common case, this is a legal vector type, convert it to the
- // packed operation and type now.
- PackVectorOp(Op, PackedVT);
- } else if (NumElements == 1) {
- // Otherwise, if this is a single element vector, convert it to a
- // scalar operation.
- PackVectorOp(Op, EVT);
- } else {
- // Otherwise, this is a multiple element vector that isn't supported.
- // Split it in half and legalize both parts.
- SDOperand X, Y;
- SplitVectorOp(Op, X, Y);
- }
+ // Otherwise, this is an illegal multiple element vector.
+ // Split it in half and legalize both parts.
+ SDOperand X, Y;
+ SplitVectorOp(Op, X, Y);
}
break;
}
// double.
MVT::ValueType VT = CFP->getValueType(0);
bool isDouble = VT == MVT::f64;
- ConstantFP *LLVMC = ConstantFP::get(isDouble ? Type::DoubleTy :
- Type::FloatTy, CFP->getValue());
+ ConstantFP *LLVMC = ConstantFP::get(MVT::getTypeForValueType(VT),
+ CFP->getValueAPF());
if (!UseCP) {
- double Val = LLVMC->getValue();
- return isDouble
- ? DAG.getConstant(DoubleToBits(Val), MVT::i64)
- : DAG.getConstant(FloatToBits(Val), MVT::i32);
+ if (VT!=MVT::f64 && VT!=MVT::f32)
+ assert(0 && "Invalid type expansion");
+ return DAG.getConstant(LLVMC->getValueAPF().convertToAPInt().getZExtValue(),
+ isDouble ? MVT::i64 : MVT::i32);
}
- if (isDouble && CFP->isExactlyValue((float)CFP->getValue()) &&
+ if (isDouble && CFP->isValueValidForType(MVT::f32, CFP->getValueAPF()) &&
// Only do this if the target has a native EXTLOAD instruction from f32.
+ // Do not try to be clever about long doubles (so far)
TLI.isLoadXLegal(ISD::EXTLOAD, MVT::f32)) {
LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC,Type::FloatTy));
VT = MVT::f32;
SelectionDAG &DAG, TargetLowering &TLI) {
MVT::ValueType VT = Node->getValueType(0);
MVT::ValueType SrcVT = Node->getOperand(1).getValueType();
+ assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) &&
+ "fcopysign expansion only supported for f32 and f64");
MVT::ValueType SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32;
// First get the sign bit of second operand.
return Result;
}
+/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores.
+static
+SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
+ TargetLowering &TLI) {
+ SDOperand Chain = ST->getChain();
+ SDOperand Ptr = ST->getBasePtr();
+ SDOperand Val = ST->getValue();
+ MVT::ValueType VT = Val.getValueType();
+ int Alignment = ST->getAlignment();
+ int SVOffset = ST->getSrcValueOffset();
+ if (MVT::isFloatingPoint(ST->getStoredVT())) {
+ // Expand to a bitconvert of the value to the integer type of the
+ // same size, then a (misaligned) int store.
+ MVT::ValueType intVT;
+ if (VT==MVT::f64)
+ intVT = MVT::i64;
+ else if (VT==MVT::f32)
+ intVT = MVT::i32;
+ else
+ assert(0 && "Unaligned load of unsupported floating point type");
+
+ SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val);
+ return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(),
+ SVOffset, ST->isVolatile(), Alignment);
+ }
+ assert(MVT::isInteger(ST->getStoredVT()) &&
+ "Unaligned store of unknown type.");
+ // Get the half-size VT
+ MVT::ValueType NewStoredVT = ST->getStoredVT() - 1;
+ int NumBits = MVT::getSizeInBits(NewStoredVT);
+ int IncrementSize = NumBits / 8;
+
+ // Divide the stored value in two parts.
+ SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy());
+ SDOperand Lo = Val;
+ SDOperand Hi = DAG.getNode(ISD::SRL, VT, Val, ShiftAmount);
+
+ // Store the two parts
+ SDOperand Store1, Store2;
+ Store1 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Lo:Hi, Ptr,
+ ST->getSrcValue(), SVOffset, NewStoredVT,
+ ST->isVolatile(), Alignment);
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+ DAG.getConstant(IncrementSize, TLI.getPointerTy()));
+ Alignment = MinAlign(Alignment, IncrementSize);
+ Store2 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Hi:Lo, Ptr,
+ ST->getSrcValue(), SVOffset + IncrementSize,
+ NewStoredVT, ST->isVolatile(), Alignment);
+
+ return DAG.getNode(ISD::TokenFactor, MVT::Other, Store1, Store2);
+}
+
+/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads.
+static
+SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
+ TargetLowering &TLI) {
+ int SVOffset = LD->getSrcValueOffset();
+ SDOperand Chain = LD->getChain();
+ SDOperand Ptr = LD->getBasePtr();
+ MVT::ValueType VT = LD->getValueType(0);
+ MVT::ValueType LoadedVT = LD->getLoadedVT();
+ if (MVT::isFloatingPoint(VT)) {
+ // Expand to a (misaligned) integer load of the same size,
+ // then bitconvert to floating point.
+ MVT::ValueType intVT;
+ if (LoadedVT==MVT::f64)
+ intVT = MVT::i64;
+ else if (LoadedVT==MVT::f32)
+ intVT = MVT::i32;
+ else
+ assert(0 && "Unaligned load of unsupported floating point type");
+
+ SDOperand newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(),
+ SVOffset, LD->isVolatile(),
+ LD->getAlignment());
+ SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad);
+ if (LoadedVT != VT)
+ Result = DAG.getNode(ISD::FP_EXTEND, VT, Result);
+
+ SDOperand Ops[] = { Result, Chain };
+ return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+ Ops, 2);
+ }
+ assert(MVT::isInteger(LoadedVT) && "Unaligned load of unsupported type.");
+ MVT::ValueType NewLoadedVT = LoadedVT - 1;
+ int NumBits = MVT::getSizeInBits(NewLoadedVT);
+ int Alignment = LD->getAlignment();
+ int IncrementSize = NumBits / 8;
+ ISD::LoadExtType HiExtType = LD->getExtensionType();
+
+ // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
+ if (HiExtType == ISD::NON_EXTLOAD)
+ HiExtType = ISD::ZEXTLOAD;
+
+ // Load the value in two parts
+ SDOperand Lo, Hi;
+ if (TLI.isLittleEndian()) {
+ Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(),
+ SVOffset, NewLoadedVT, LD->isVolatile(), Alignment);
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+ DAG.getConstant(IncrementSize, TLI.getPointerTy()));
+ Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(),
+ SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(),
+ MinAlign(Alignment, IncrementSize));
+ } else {
+ Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(), SVOffset,
+ NewLoadedVT,LD->isVolatile(), Alignment);
+ Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
+ DAG.getConstant(IncrementSize, TLI.getPointerTy()));
+ Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(),
+ SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(),
+ MinAlign(Alignment, IncrementSize));
+ }
+
+ // aggregate the two parts
+ SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy());
+ SDOperand Result = DAG.getNode(ISD::SHL, VT, Hi, ShiftAmount);
+ Result = DAG.getNode(ISD::OR, VT, Result, Lo);
+
+ SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
+ Hi.getValue(1));
+
+ SDOperand Ops[] = { Result, TF };
+ return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), Ops, 2);
+}
+
+/// UnrollVectorOp - We know that the given vector has a legal type, however
+/// the operation it performs is not legal and is an operation that we have
+/// no way of lowering. "Unroll" the vector, splitting out the scalars and
+/// operating on each element individually.
+SDOperand SelectionDAGLegalize::UnrollVectorOp(SDOperand Op) {
+ MVT::ValueType VT = Op.getValueType();
+ assert(isTypeLegal(VT) &&
+ "Caller should expand or promote operands that are not legal!");
+ assert(Op.Val->getNumValues() == 1 &&
+ "Can't unroll a vector with multiple results!");
+ unsigned NE = MVT::getVectorNumElements(VT);
+ MVT::ValueType EltVT = MVT::getVectorElementType(VT);
+
+ SmallVector<SDOperand, 8> Scalars;
+ SmallVector<SDOperand, 4> Operands(Op.getNumOperands());
+ for (unsigned i = 0; i != NE; ++i) {
+ for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
+ SDOperand Operand = Op.getOperand(j);
+ MVT::ValueType OperandVT = Operand.getValueType();
+ if (MVT::isVector(OperandVT)) {
+ // A vector operand; extract a single element.
+ MVT::ValueType OperandEltVT = MVT::getVectorElementType(OperandVT);
+ Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
+ OperandEltVT,
+ Operand,
+ DAG.getConstant(i, MVT::i32));
+ } else {
+ // A scalar operand; just use it as is.
+ Operands[j] = Operand;
+ }
+ }
+ Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT,
+ &Operands[0], Operands.size()));
+ }
+
+ return DAG.getNode(ISD::BUILD_VECTOR, VT, &Scalars[0], Scalars.size());
+}
-/// LegalizeOp - We know that the specified value has a legal type.
-/// Recursively ensure that the operands have legal types, then return the
-/// result.
+/// LegalizeOp - We know that the specified value has a legal type, and
+/// that its operands are legal. Now ensure that the operation itself
+/// is legal, recursively ensuring that the operands' operations remain
+/// legal.
SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
+ if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
+ return Op;
+
assert(isTypeLegal(Op.getValueType()) &&
"Caller should expand or promote operands that are not legal!");
SDNode *Node = Op.Val;
case ISD::TargetConstantFP:
case ISD::TargetConstantPool:
case ISD::TargetGlobalAddress:
+ case ISD::TargetGlobalTLSAddress:
case ISD::TargetExternalSymbol:
case ISD::VALUETYPE:
case ISD::SRCVALUE:
case ISD::STRING:
case ISD::CONDCODE:
- case ISD::GLOBAL_OFFSET_TABLE:
// Primitives must all be legal.
- assert(TLI.isOperationLegal(Node->getValueType(0), Node->getValueType(0)) &&
+ assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
"This must be legal!");
break;
default:
}
// Otherwise this is an unhandled builtin node. splat.
#ifndef NDEBUG
- cerr << "NODE: "; Node->dump(); cerr << "\n";
+ cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
#endif
assert(0 && "Do not know how to legalize this operator!");
abort();
+ case ISD::GLOBAL_OFFSET_TABLE:
case ISD::GlobalAddress:
+ case ISD::GlobalTLSAddress:
case ISD::ExternalSymbol:
case ISD::ConstantPool:
case ISD::JumpTable: // Nothing to do.
else
Result = DAG.getConstant(0, TLI.getPointerTy());
break;
+ case ISD::FRAME_TO_ARGS_OFFSET: {
+ MVT::ValueType VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal:
+ Result = DAG.getConstant(0, VT);
+ break;
+ }
+ }
+ break;
+ case ISD::EXCEPTIONADDR: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ MVT::ValueType VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand: {
+ unsigned Reg = TLI.getExceptionAddressRegister();
+ Result = DAG.getCopyFromReg(Tmp1, Reg, VT).getValue(Op.ResNo);
+ }
+ break;
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal: {
+ SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp1 };
+ Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+ Ops, 2).getValue(Op.ResNo);
+ break;
+ }
+ }
+ }
+ break;
+ case ISD::EHSELECTION: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ MVT::ValueType VT = Node->getValueType(0);
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Expand: {
+ unsigned Reg = TLI.getExceptionSelectorRegister();
+ Result = DAG.getCopyFromReg(Tmp2, Reg, VT).getValue(Op.ResNo);
+ }
+ break;
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal: {
+ SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp2 };
+ Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+ Ops, 2).getValue(Op.ResNo);
+ break;
+ }
+ }
+ }
+ break;
+ case ISD::EH_RETURN: {
+ MVT::ValueType VT = Node->getValueType(0);
+ // The only "good" option for this node is to custom lower it.
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
+ default: assert(0 && "This action is not supported at all!");
+ case TargetLowering::Custom:
+ Result = TLI.LowerOperation(Op, DAG);
+ if (Result.Val) break;
+ // Fall Thru
+ case TargetLowering::Legal:
+ // Target does not know, how to lower this, lower to noop
+ Result = LegalizeOp(Node->getOperand(0));
+ break;
+ }
+ }
+ break;
case ISD::AssertSext:
case ISD::AssertZext:
Tmp1 = LegalizeOp(Node->getOperand(0));
if (MVT::isInteger(VT))
Result = DAG.getConstant(0, VT);
else if (MVT::isFloatingPoint(VT))
- Result = DAG.getConstantFP(0, VT);
+ Result = DAG.getConstantFP(APFloat(APInt(MVT::getSizeInBits(VT), 0)),
+ VT);
else
assert(0 && "Unknown value type!");
break;
Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the label id.
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
break;
+ case TargetLowering::Expand:
+ Result = LegalizeOp(Node->getOperand(0));
+ break;
}
break;
- case ISD::Constant:
+ case ISD::Constant: {
+ ConstantSDNode *CN = cast<ConstantSDNode>(Node);
+ unsigned opAction =
+ TLI.getOperationAction(ISD::Constant, CN->getValueType(0));
+
// We know we don't need to expand constants here, constants only have one
// value and we check that it is fine above.
- // FIXME: Maybe we should handle things like targets that don't support full
- // 32-bit immediates?
+ if (opAction == TargetLowering::Custom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val)
+ Result = Tmp1;
+ }
break;
+ }
case ISD::ConstantFP: {
// Spill FP immediates to the constant pool if the target cannot directly
// codegen them. Targets often have some immediate values that can be
// If this is a legal constant, turn it into a TargetConstantFP node.
if (isLegal) {
- Result = DAG.getTargetConstantFP(CFP->getValue(), CFP->getValueType(0));
+ Result = DAG.getTargetConstantFP(CFP->getValueAPF(),
+ CFP->getValueType(0));
break;
}
AddLegalizedOperand(SDOperand(Node, i), Tmp1);
}
return Tmp2;
-
+ case ISD::EXTRACT_SUBREG: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(1));
+ assert(idx && "Operand must be a constant");
+ Tmp2 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ }
+ break;
+ case ISD::INSERT_SUBREG: {
+ Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(2));
+ assert(idx && "Operand must be a constant");
+ Tmp3 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
+ }
+ break;
case ISD::BUILD_VECTOR:
switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) {
default: assert(0 && "This action is not supported yet!");
unsigned NumElts = MVT::getVectorNumElements(Tmp1.getValueType());
MVT::ValueType ShufMaskVT = MVT::getIntVectorWithNumElements(NumElts);
- MVT::ValueType ShufMaskEltVT = MVT::getVectorBaseType(ShufMaskVT);
+ MVT::ValueType ShufMaskEltVT = MVT::getVectorElementType(ShufMaskVT);
// We generate a shuffle of InVec and ScVec, so the shuffle mask should
// be 0,1,2,3,4,5... with the appropriate element replaced with elt 0 of
MVT::ValueType EltVT = Tmp2.getValueType();
MVT::ValueType IdxVT = Tmp3.getValueType();
MVT::ValueType PtrVT = TLI.getPointerTy();
- SDOperand StackPtr = CreateStackTemporary(VT);
+ SDOperand StackPtr = DAG.CreateStackTemporary(VT);
// Store the vector.
SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, NULL, 0);
// FALLTHROUGH
case TargetLowering::Expand: {
MVT::ValueType VT = Node->getValueType(0);
- MVT::ValueType EltVT = MVT::getVectorBaseType(VT);
+ MVT::ValueType EltVT = MVT::getVectorElementType(VT);
MVT::ValueType PtrVT = TLI.getPointerTy();
SDOperand Mask = Node->getOperand(2);
unsigned NumElems = Mask.getNumOperands();
break;
case ISD::EXTRACT_VECTOR_ELT:
- Tmp1 = LegalizeOp(Node->getOperand(0));
+ Tmp1 = Node->getOperand(0);
Tmp2 = LegalizeOp(Node->getOperand(1));
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
-
- switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT,
- Tmp1.getValueType())) {
- default: assert(0 && "This action is not supported yet!");
- case TargetLowering::Legal:
- break;
- case TargetLowering::Custom:
- Tmp3 = TLI.LowerOperation(Result, DAG);
- if (Tmp3.Val) {
- Result = Tmp3;
- break;
- }
- // FALLTHROUGH
- case TargetLowering::Expand:
- Result = ExpandEXTRACT_VECTOR_ELT(Result);
- break;
- }
+ Result = ExpandEXTRACT_VECTOR_ELT(Result);
break;
- case ISD::VEXTRACT_VECTOR_ELT:
- Result = LegalizeOp(LowerVEXTRACT_VECTOR_ELT(Op));
+ case ISD::EXTRACT_SUBVECTOR:
+ Tmp1 = Node->getOperand(0);
+ Tmp2 = LegalizeOp(Node->getOperand(1));
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ Result = ExpandEXTRACT_SUBVECTOR(Result);
break;
case ISD::CALLSEQ_START: {
AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
return Result.getValue(Op.ResNo);
case ISD::DYNAMIC_STACKALLOC: {
+ MVT::ValueType VT = Node->getValueType(0);
Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the size.
Tmp3 = LegalizeOp(Node->getOperand(2)); // Legalize the alignment.
Tmp1 = Result.getValue(0);
Tmp2 = Result.getValue(1);
- switch (TLI.getOperationAction(Node->getOpcode(),
- Node->getValueType(0))) {
+ switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
default: assert(0 && "This action is not supported yet!");
case TargetLowering::Expand: {
unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
" not tell us which reg is the stack pointer!");
SDOperand Chain = Tmp1.getOperand(0);
SDOperand Size = Tmp2.getOperand(1);
- SDOperand SP = DAG.getCopyFromReg(Chain, SPReg, Node->getValueType(0));
- Tmp1 = DAG.getNode(ISD::SUB, Node->getValueType(0), SP, Size); // Value
- Tmp2 = DAG.getCopyToReg(SP.getValue(1), SPReg, Tmp1); // Output chain
+ SDOperand SP = DAG.getCopyFromReg(Chain, SPReg, VT);
+ Chain = SP.getValue(1);
+ unsigned Align = cast<ConstantSDNode>(Tmp3)->getValue();
+ unsigned StackAlign =
+ TLI.getTargetMachine().getFrameInfo()->getStackAlignment();
+ if (Align > StackAlign)
+ SP = DAG.getNode(ISD::AND, VT, SP,
+ DAG.getConstant(-(uint64_t)Align, VT));
+ Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size); // Value
+ Tmp2 = DAG.getCopyToReg(Chain, SPReg, Tmp1); // Output chain
Tmp1 = LegalizeOp(Tmp1);
Tmp2 = LegalizeOp(Tmp2);
break;
// The top bits of the promoted condition are not necessarily zero, ensure
// that the value is properly zero extended.
- if (!TLI.MaskedValueIsZero(Tmp2,
+ if (!DAG.MaskedValueIsZero(Tmp2,
MVT::getIntVTBitMask(Tmp2.getValueType())^1))
Tmp2 = DAG.getZeroExtendInReg(Tmp2, MVT::i1);
break;
switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
default: assert(0 && "This action is not supported yet!");
- case TargetLowering::Legal: break;
+ case TargetLowering::Legal:
+ // If this is an unaligned load and the target doesn't support it,
+ // expand it.
+ if (!TLI.allowsUnalignedMemoryAccesses()) {
+ unsigned ABIAlignment = TLI.getTargetData()->
+ getABITypeAlignment(MVT::getTypeForValueType(LD->getLoadedVT()));
+ if (LD->getAlignment() < ABIAlignment){
+ Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.Val), DAG,
+ TLI);
+ Tmp3 = Result.getOperand(0);
+ Tmp4 = Result.getOperand(1);
+ Tmp3 = LegalizeOp(Tmp3);
+ Tmp4 = LegalizeOp(Tmp4);
+ }
+ }
+ break;
case TargetLowering::Custom:
Tmp1 = TLI.LowerOperation(Tmp3, DAG);
if (Tmp1.Val) {
MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
Tmp1 = DAG.getLoad(NVT, Tmp1, Tmp2, LD->getSrcValue(),
- LD->getSrcValueOffset());
+ LD->getSrcValueOffset(),
+ LD->isVolatile(), LD->getAlignment());
Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp1));
Tmp4 = LegalizeOp(Tmp1.getValue(1));
break;
"Can only promote extending LOAD from i1 -> i8!");
Result = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2,
LD->getSrcValue(), LD->getSrcValueOffset(),
- MVT::i8);
- Tmp1 = Result.getValue(0);
- Tmp2 = Result.getValue(1);
+ MVT::i8, LD->isVolatile(), LD->getAlignment());
+ Tmp1 = Result.getValue(0);
+ Tmp2 = Result.getValue(1);
break;
case TargetLowering::Custom:
isCustom = true;
Tmp1 = LegalizeOp(Tmp3);
Tmp2 = LegalizeOp(Tmp3.getValue(1));
}
+ } else {
+ // If this is an unaligned load and the target doesn't support it,
+ // expand it.
+ if (!TLI.allowsUnalignedMemoryAccesses()) {
+ unsigned ABIAlignment = TLI.getTargetData()->
+ getABITypeAlignment(MVT::getTypeForValueType(LD->getLoadedVT()));
+ if (LD->getAlignment() < ABIAlignment){
+ Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.Val), DAG,
+ TLI);
+ Tmp1 = Result.getOperand(0);
+ Tmp2 = Result.getOperand(1);
+ Tmp1 = LegalizeOp(Tmp1);
+ Tmp2 = LegalizeOp(Tmp2);
+ }
+ }
}
break;
case TargetLowering::Expand:
// f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) {
SDOperand Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(),
- LD->getSrcValueOffset());
+ LD->getSrcValueOffset(),
+ LD->isVolatile(), LD->getAlignment());
Result = DAG.getNode(ISD::FP_EXTEND, Node->getValueType(0), Load);
Tmp1 = LegalizeOp(Result); // Relegalize new nodes.
Tmp2 = LegalizeOp(Load.getValue(1));
// zero/sign extend inreg.
Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
Tmp1, Tmp2, LD->getSrcValue(),
- LD->getSrcValueOffset(), SrcVT);
+ LD->getSrcValueOffset(), SrcVT,
+ LD->isVolatile(), LD->getAlignment());
SDOperand ValRes;
if (ExtType == ISD::SEXTLOAD)
ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3);
break;
case Expand:
- if (Tmp2.getValueType() != MVT::Vector) {
+ if (!MVT::isVector(Tmp2.getValueType())) {
SDOperand Lo, Hi;
ExpandOp(Tmp2, Lo, Hi);
+
+ // Big endian systems want the hi reg first.
+ if (!TLI.isLittleEndian())
+ std::swap(Lo, Hi);
+
if (Hi.Val)
Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
else
Result = LegalizeOp(Result);
} else {
SDNode *InVal = Tmp2.Val;
- unsigned NumElems =
- cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+ int InIx = Tmp2.ResNo;
+ unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(InIx));
+ MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(InIx));
- // Figure out if there is a Packed type corresponding to this Vector
+ // Figure out if there is a simple type corresponding to this Vector
// type. If so, convert to the vector type.
MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
- if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
+ if (TLI.isTypeLegal(TVT)) {
// Turn this into a return of the vector type.
- Tmp2 = PackVectorOp(Tmp2, TVT);
+ Tmp2 = LegalizeOp(Tmp2);
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
} else if (NumElems == 1) {
// Turn this into a return of the scalar type.
- Tmp2 = PackVectorOp(Tmp2, EVT);
+ Tmp2 = ScalarizeVectorOp(Tmp2);
+ Tmp2 = LegalizeOp(Tmp2);
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
// FIXME: Returns of gcc generic vectors smaller than a legal type
break;
case Expand: {
SDOperand Lo, Hi;
- assert(Node->getOperand(i).getValueType() != MVT::Vector &&
+ assert(!MVT::isExtendedVT(Node->getOperand(i).getValueType()) &&
"FIXME: TODO: implement returning non-legal vector types!");
ExpandOp(Node->getOperand(i), Lo, Hi);
NewValues.push_back(Lo);
StoreSDNode *ST = cast<StoreSDNode>(Node);
Tmp1 = LegalizeOp(ST->getChain()); // Legalize the chain.
Tmp2 = LegalizeOp(ST->getBasePtr()); // Legalize the pointer.
+ int SVOffset = ST->getSrcValueOffset();
+ unsigned Alignment = ST->getAlignment();
+ bool isVolatile = ST->isVolatile();
if (!ST->isTruncatingStore()) {
// Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
// to phase ordering between legalized code and the dag combiner. This
// probably means that we need to integrate dag combiner and legalizer
// together.
+ // We generally can't do this one for long doubles.
if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
- if (CFP->getValueType(0) == MVT::f32) {
- Tmp3 = DAG.getConstant(FloatToBits(CFP->getValue()), MVT::i32);
- } else {
- assert(CFP->getValueType(0) == MVT::f64 && "Unknown FP type!");
- Tmp3 = DAG.getConstant(DoubleToBits(CFP->getValue()), MVT::i64);
+ if (CFP->getValueType(0) == MVT::f32 &&
+ getTypeAction(MVT::i32) == Legal) {
+ Tmp3 = DAG.getConstant((uint32_t)CFP->getValueAPF().
+ convertToAPInt().getZExtValue(),
+ MVT::i32);
+ Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ break;
+ } else if (CFP->getValueType(0) == MVT::f64) {
+ // If this target supports 64-bit registers, do a single 64-bit store.
+ if (getTypeAction(MVT::i64) == Legal) {
+ Tmp3 = DAG.getConstant(CFP->getValueAPF().convertToAPInt().
+ getZExtValue(), MVT::i64);
+ Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ break;
+ } else if (getTypeAction(MVT::i32) == Legal) {
+ // Otherwise, if the target supports 32-bit registers, use 2 32-bit
+ // stores. If the target supports neither 32- nor 64-bits, this
+ // xform is certainly not worth it.
+ uint64_t IntVal =CFP->getValueAPF().convertToAPInt().getZExtValue();
+ SDOperand Lo = DAG.getConstant(uint32_t(IntVal), MVT::i32);
+ SDOperand Hi = DAG.getConstant(uint32_t(IntVal >>32), MVT::i32);
+ if (!TLI.isLittleEndian()) std::swap(Lo, Hi);
+
+ Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
+ SVOffset, isVolatile, Alignment);
+ Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
+ getIntPtrConstant(4));
+ Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), SVOffset+4,
+ isVolatile, MinAlign(Alignment, 4U));
+
+ Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
+ break;
+ }
}
- Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset());
- break;
}
switch (getTypeAction(ST->getStoredVT())) {
MVT::ValueType VT = Tmp3.getValueType();
switch (TLI.getOperationAction(ISD::STORE, VT)) {
default: assert(0 && "This action is not supported yet!");
- case TargetLowering::Legal: break;
+ case TargetLowering::Legal:
+ // If this is an unaligned store and the target doesn't support it,
+ // expand it.
+ if (!TLI.allowsUnalignedMemoryAccesses()) {
+ unsigned ABIAlignment = TLI.getTargetData()->
+ getABITypeAlignment(MVT::getTypeForValueType(ST->getStoredVT()));
+ if (ST->getAlignment() < ABIAlignment)
+ Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.Val), DAG,
+ TLI);
+ }
+ break;
case TargetLowering::Custom:
Tmp1 = TLI.LowerOperation(Result, DAG);
if (Tmp1.Val) Result = Tmp1;
Tmp3 = DAG.getNode(ISD::BIT_CONVERT,
TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
Result = DAG.getStore(Tmp1, Tmp3, Tmp2,
- ST->getSrcValue(), ST->getSrcValueOffset());
+ ST->getSrcValue(), SVOffset, isVolatile,
+ Alignment);
break;
}
break;
// Truncate the value and store the result.
Tmp3 = PromoteOp(ST->getValue());
Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset(), ST->getStoredVT());
+ SVOffset, ST->getStoredVT(),
+ isVolatile, Alignment);
break;
case Expand:
// If this is a vector type, then we have to calculate the increment as
// the product of the element size in bytes, and the number of elements
// in the high half of the vector.
- if (ST->getValue().getValueType() == MVT::Vector) {
+ if (MVT::isVector(ST->getValue().getValueType())) {
SDNode *InVal = ST->getValue().Val;
- unsigned NumElems =
- cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+ int InIx = ST->getValue().ResNo;
+ unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(InIx));
+ MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(InIx));
- // Figure out if there is a Packed type corresponding to this Vector
+ // Figure out if there is a simple type corresponding to this Vector
// type. If so, convert to the vector type.
MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
- if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
+ if (TLI.isTypeLegal(TVT)) {
// Turn this into a normal store of the vector type.
- Tmp3 = PackVectorOp(Node->getOperand(1), TVT);
+ Tmp3 = LegalizeOp(Node->getOperand(1));
Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset());
+ SVOffset, isVolatile, Alignment);
Result = LegalizeOp(Result);
break;
} else if (NumElems == 1) {
// Turn this into a normal store of the scalar type.
- Tmp3 = PackVectorOp(Node->getOperand(1), EVT);
+ Tmp3 = ScalarizeVectorOp(Node->getOperand(1));
Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset());
+ SVOffset, isVolatile, Alignment);
// The scalarized value type may not be legal, e.g. it might require
// promotion or expansion. Relegalize the scalar store.
Result = LegalizeOp(Result);
}
Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset());
+ SVOffset, isVolatile, Alignment);
if (Hi.Val == NULL) {
// Must be int <-> float one-to-one expansion.
getIntPtrConstant(IncrementSize));
assert(isTypeLegal(Tmp2.getValueType()) &&
"Pointers must be legal!");
- // FIXME: This sets the srcvalue of both halves to be the same, which is
- // wrong.
+ SVOffset += IncrementSize;
+ Alignment = MinAlign(Alignment, IncrementSize);
Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset());
+ SVOffset, isVolatile, Alignment);
Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
break;
}
Tmp3 = DAG.getNode(ISD::AND, Tmp3.getValueType(), Tmp3,
DAG.getConstant(1, Tmp3.getValueType()));
Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
- ST->getSrcValueOffset(), MVT::i8);
+ SVOffset, MVT::i8,
+ isVolatile, Alignment);
} else if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() ||
Tmp2 != ST->getBasePtr()) {
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
MVT::ValueType StVT = cast<StoreSDNode>(Result.Val)->getStoredVT();
switch (TLI.getStoreXAction(StVT)) {
default: assert(0 && "This action is not supported yet!");
- case TargetLowering::Legal: break;
+ case TargetLowering::Legal:
+ // If this is an unaligned store and the target doesn't support it,
+ // expand it.
+ if (!TLI.allowsUnalignedMemoryAccesses()) {
+ unsigned ABIAlignment = TLI.getTargetData()->
+ getABITypeAlignment(MVT::getTypeForValueType(ST->getStoredVT()));
+ if (ST->getAlignment() < ABIAlignment)
+ Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.Val), DAG,
+ TLI);
+ }
+ break;
case TargetLowering::Custom:
Tmp1 = TLI.LowerOperation(Result, DAG);
if (Tmp1.Val) Result = Tmp1;
case Promote:
Tmp1 = PromoteOp(Node->getOperand(0)); // Promote the condition.
// Make sure the condition is either zero or one.
- if (!TLI.MaskedValueIsZero(Tmp1,
+ if (!DAG.MaskedValueIsZero(Tmp1,
MVT::getIntVTBitMask(Tmp1.getValueType())^1))
Tmp1 = DAG.getZeroExtendInReg(Tmp1, MVT::i1);
break;
break;
}
+ SDOperand Tmp6;
+ switch (getTypeAction(Node->getOperand(5).getValueType())) { // bool
+ case Expand: assert(0 && "Cannot expand this yet!");
+ case Legal:
+ Tmp6 = LegalizeOp(Node->getOperand(5));
+ break;
+ case Promote:
+ Tmp6 = PromoteOp(Node->getOperand(5));
+ break;
+ }
+
switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
default: assert(0 && "This action not implemented for this operation!");
case TargetLowering::Custom:
isCustom = true;
// FALLTHROUGH
- case TargetLowering::Legal:
- Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5);
+ case TargetLowering::Legal: {
+ SDOperand Ops[] = { Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6 };
+ Result = DAG.UpdateNodeOperands(Result, Ops, 6);
if (isCustom) {
Tmp1 = TLI.LowerOperation(Result, DAG);
if (Tmp1.Val) Result = Tmp1;
}
break;
+ }
case TargetLowering::Expand: {
// Otherwise, the target does not support this operation. Lower the
// operation to an explicit libcall as appropriate.
const char *FnName = 0;
if (Node->getOpcode() == ISD::MEMSET) {
- Entry.Node = Tmp2; Entry.isSigned = false; Entry.Ty = IntPtrTy;
- Entry.isInReg = false; Entry.isSRet = false;
+ Entry.Node = Tmp2; Entry.Ty = IntPtrTy;
Args.push_back(Entry);
// Extend the (previously legalized) ubyte argument to be an int value
// for the call.
Tmp3 = DAG.getNode(ISD::TRUNCATE, MVT::i32, Tmp3);
else
Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Tmp3);
- Entry.Node = Tmp3; Entry.Ty = Type::Int32Ty; Entry.isSigned = true;
- Entry.isInReg = false; Entry.isSRet = false;
+ Entry.Node = Tmp3; Entry.Ty = Type::Int32Ty; Entry.isSExt = true;
Args.push_back(Entry);
- Entry.Node = Tmp4; Entry.Ty = IntPtrTy; Entry.isSigned = false;
+ Entry.Node = Tmp4; Entry.Ty = IntPtrTy; Entry.isSExt = false;
Args.push_back(Entry);
FnName = "memset";
} else if (Node->getOpcode() == ISD::MEMCPY ||
Node->getOpcode() == ISD::MEMMOVE) {
Entry.Ty = IntPtrTy;
- Entry.isSigned = false; Entry.isInReg = false; Entry.isSRet = false;
Entry.Node = Tmp2; Args.push_back(Entry);
Entry.Node = Tmp3; Args.push_back(Entry);
Entry.Node = Tmp4; Args.push_back(Entry);
case ISD::FSUB:
case ISD::FMUL:
case ISD::FDIV:
+ case ISD::FPOW:
Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
switch (getTypeAction(Node->getOperand(1).getValueType())) {
case Expand: assert(0 && "Not possible");
if (Tmp1.Val) Result = Tmp1;
break;
case TargetLowering::Expand: {
- if (Node->getValueType(0) == MVT::i32) {
- switch (Node->getOpcode()) {
- default: assert(0 && "Do not know how to expand this integer BinOp!");
- case ISD::UDIV:
- case ISD::SDIV:
- RTLIB::Libcall LC = Node->getOpcode() == ISD::UDIV
+ MVT::ValueType VT = Op.getValueType();
+
+ // See if multiply or divide can be lowered using two-result operations.
+ SDVTList VTs = DAG.getVTList(VT, VT);
+ if (Node->getOpcode() == ISD::MUL) {
+ // We just need the low half of the multiply; try both the signed
+ // and unsigned forms. If the target supports both SMUL_LOHI and
+ // UMUL_LOHI, form a preference by checking which forms of plain
+ // MULH it supports.
+ bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, VT);
+ bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, VT);
+ bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, VT);
+ bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, VT);
+ unsigned OpToUse = 0;
+ if (HasSMUL_LOHI && !HasMULHS) {
+ OpToUse = ISD::SMUL_LOHI;
+ } else if (HasUMUL_LOHI && !HasMULHU) {
+ OpToUse = ISD::UMUL_LOHI;
+ } else if (HasSMUL_LOHI) {
+ OpToUse = ISD::SMUL_LOHI;
+ } else if (HasUMUL_LOHI) {
+ OpToUse = ISD::UMUL_LOHI;
+ }
+ if (OpToUse) {
+ Result = SDOperand(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).Val, 0);
+ break;
+ }
+ }
+ if (Node->getOpcode() == ISD::MULHS &&
+ TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) {
+ Result = SDOperand(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1);
+ break;
+ }
+ if (Node->getOpcode() == ISD::MULHU &&
+ TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) {
+ Result = SDOperand(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1);
+ break;
+ }
+ if (Node->getOpcode() == ISD::SDIV &&
+ TLI.isOperationLegal(ISD::SDIVREM, VT)) {
+ Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 0);
+ break;
+ }
+ if (Node->getOpcode() == ISD::UDIV &&
+ TLI.isOperationLegal(ISD::UDIVREM, VT)) {
+ Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 0);
+ break;
+ }
+
+ // Check to see if we have a libcall for this operator.
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
+ bool isSigned = false;
+ switch (Node->getOpcode()) {
+ case ISD::UDIV:
+ case ISD::SDIV:
+ if (VT == MVT::i32) {
+ LC = Node->getOpcode() == ISD::UDIV
? RTLIB::UDIV_I32 : RTLIB::SDIV_I32;
- SDOperand Dummy;
- bool isSigned = Node->getOpcode() == ISD::SDIV;
- Result = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Dummy);
- };
+ isSigned = Node->getOpcode() == ISD::SDIV;
+ }
+ break;
+ case ISD::FPOW:
+ LC = VT == MVT::f32 ? RTLIB::POW_F32 :
+ VT == MVT::f64 ? RTLIB::POW_F64 :
+ VT == MVT::f80 ? RTLIB::POW_F80 :
+ VT == MVT::ppcf128 ? RTLIB::POW_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL;
+ break;
+ default: break;
+ }
+ if (LC != RTLIB::UNKNOWN_LIBCALL) {
+ SDOperand Dummy;
+ Result = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Dummy);
break;
}
assert(MVT::isVector(Node->getValueType(0)) &&
"Cannot expand this binary operator!");
// Expand the operation into a bunch of nasty scalar code.
- SmallVector<SDOperand, 8> Ops;
- MVT::ValueType EltVT = MVT::getVectorBaseType(Node->getValueType(0));
- MVT::ValueType PtrVT = TLI.getPointerTy();
- for (unsigned i = 0, e = MVT::getVectorNumElements(Node->getValueType(0));
- i != e; ++i) {
- SDOperand Idx = DAG.getConstant(i, PtrVT);
- SDOperand LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1, Idx);
- SDOperand RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp2, Idx);
- Ops.push_back(DAG.getNode(Node->getOpcode(), EltVT, LHS, RHS));
- }
- Result = DAG.getNode(ISD::BUILD_VECTOR, Node->getValueType(0),
- &Ops[0], Ops.size());
+ Result = LegalizeOp(UnrollVectorOp(Op));
break;
}
case TargetLowering::Promote: {
}
break;
+ case ISD::SMUL_LOHI:
+ case ISD::UMUL_LOHI:
+ case ISD::SDIVREM:
+ case ISD::UDIVREM:
+ // These nodes will only be produced by target-specific lowering, so
+ // they shouldn't be here if they aren't legal.
+ assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
+ "This must be legal!");
+
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
+ Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS
+ Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
+ break;
+
case ISD::FCOPYSIGN: // FCOPYSIGN does not require LHS/RHS to match type!
Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
switch (getTypeAction(Node->getOperand(1).getValueType())) {
if (Tmp1.Val) Result = Tmp1;
}
break;
- case TargetLowering::Expand:
+ case TargetLowering::Expand: {
unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV;
bool isSigned = DivOpc == ISD::SDIV;
- if (MVT::isInteger(Node->getValueType(0))) {
- if (TLI.getOperationAction(DivOpc, Node->getValueType(0)) ==
+ MVT::ValueType VT = Node->getValueType(0);
+
+ // See if remainder can be lowered using two-result operations.
+ SDVTList VTs = DAG.getVTList(VT, VT);
+ if (Node->getOpcode() == ISD::SREM &&
+ TLI.isOperationLegal(ISD::SDIVREM, VT)) {
+ Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 1);
+ break;
+ }
+ if (Node->getOpcode() == ISD::UREM &&
+ TLI.isOperationLegal(ISD::UDIVREM, VT)) {
+ Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 1);
+ break;
+ }
+
+ if (MVT::isInteger(VT)) {
+ if (TLI.getOperationAction(DivOpc, VT) ==
TargetLowering::Legal) {
// X % Y -> X-X/Y*Y
- MVT::ValueType VT = Node->getValueType(0);
Result = DAG.getNode(DivOpc, VT, Tmp1, Tmp2);
Result = DAG.getNode(ISD::MUL, VT, Result, Tmp2);
Result = DAG.getNode(ISD::SUB, VT, Tmp1, Result);
} else {
- assert(Node->getValueType(0) == MVT::i32 &&
+ assert(VT == MVT::i32 &&
"Cannot expand this binary operator!");
RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM
? RTLIB::UREM_I32 : RTLIB::SREM_I32;
}
} else {
// Floating point mod -> fmod libcall.
- RTLIB::Libcall LC = Node->getValueType(0) == MVT::f32
+ RTLIB::Libcall LC = VT == MVT::f32
? RTLIB::REM_F32 : RTLIB::REM_F64;
SDOperand Dummy;
Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
}
break;
}
+ }
break;
case ISD::VAARG: {
Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
case ISD::ROTR:
Tmp1 = LegalizeOp(Node->getOperand(0)); // LHS
Tmp2 = LegalizeOp(Node->getOperand(1)); // RHS
-
- assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
- "Cannot handle this yet!");
Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
- break;
-
- case ISD::BSWAP:
- Tmp1 = LegalizeOp(Node->getOperand(0)); // Op
switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
- case TargetLowering::Custom:
- assert(0 && "Cannot custom legalize this yet!");
+ default:
+ assert(0 && "ROTL/ROTR legalize operation not supported");
+ break;
case TargetLowering::Legal:
- Result = DAG.UpdateNodeOperands(Result, Tmp1);
break;
- case TargetLowering::Promote: {
- MVT::ValueType OVT = Tmp1.getValueType();
- MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
- unsigned DiffBits = getSizeInBits(NVT) - getSizeInBits(OVT);
-
+ case TargetLowering::Custom:
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) Result = Tmp1;
+ break;
+ case TargetLowering::Promote:
+ assert(0 && "Do not know how to promote ROTL/ROTR");
+ break;
+ case TargetLowering::Expand:
+ assert(0 && "Do not know how to expand ROTL/ROTR");
+ break;
+ }
+ break;
+
+ case ISD::BSWAP:
+ Tmp1 = LegalizeOp(Node->getOperand(0)); // Op
+ switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
+ case TargetLowering::Custom:
+ assert(0 && "Cannot custom legalize this yet!");
+ case TargetLowering::Legal:
+ Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ break;
+ case TargetLowering::Promote: {
+ MVT::ValueType OVT = Tmp1.getValueType();
+ MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
+ unsigned DiffBits = MVT::getSizeInBits(NVT) - MVT::getSizeInBits(OVT);
+
Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
case ISD::CTLZ:
Tmp1 = LegalizeOp(Node->getOperand(0)); // Op
switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
- case TargetLowering::Custom: assert(0 && "Cannot custom handle this yet!");
+ case TargetLowering::Custom:
case TargetLowering::Legal:
Result = DAG.UpdateNodeOperands(Result, Tmp1);
+ if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) ==
+ TargetLowering::Custom) {
+ Tmp1 = TLI.LowerOperation(Result, DAG);
+ if (Tmp1.Val) {
+ Result = Tmp1;
+ }
+ }
break;
case TargetLowering::Promote: {
MVT::ValueType OVT = Tmp1.getValueType();
case ISD::CTTZ:
//if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1,
- DAG.getConstant(getSizeInBits(NVT), NVT),
+ DAG.getConstant(MVT::getSizeInBits(NVT), NVT),
ISD::SETEQ);
Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
- DAG.getConstant(getSizeInBits(OVT),NVT), Tmp1);
+ DAG.getConstant(MVT::getSizeInBits(OVT),NVT), Tmp1);
break;
case ISD::CTLZ:
// Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
- DAG.getConstant(getSizeInBits(NVT) -
- getSizeInBits(OVT), NVT));
+ DAG.getConstant(MVT::getSizeInBits(NVT) -
+ MVT::getSizeInBits(OVT), NVT));
break;
}
break;
case ISD::FSIN:
case ISD::FCOS: {
MVT::ValueType VT = Node->getValueType(0);
+
+ // Expand unsupported unary vector operators by unrolling them.
+ if (MVT::isVector(VT)) {
+ Result = LegalizeOp(UnrollVectorOp(Op));
+ break;
+ }
+
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
switch(Node->getOpcode()) {
case ISD::FSQRT:
- LC = VT == MVT::f32 ? RTLIB::SQRT_F32 : RTLIB::SQRT_F64;
+ LC = VT == MVT::f32 ? RTLIB::SQRT_F32 :
+ VT == MVT::f64 ? RTLIB::SQRT_F64 :
+ VT == MVT::f80 ? RTLIB::SQRT_F80 :
+ VT == MVT::ppcf128 ? RTLIB::SQRT_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL;
break;
case ISD::FSIN:
LC = VT == MVT::f32 ? RTLIB::SIN_F32 : RTLIB::SIN_F64;
}
break;
case ISD::FPOWI: {
- // We always lower FPOWI into a libcall. No target support it yet.
- RTLIB::Libcall LC = Node->getValueType(0) == MVT::f32
- ? RTLIB::POWI_F32 : RTLIB::POWI_F64;
+ MVT::ValueType VT = Node->getValueType(0);
+
+ // Expand unsupported unary vector operators by unrolling them.
+ if (MVT::isVector(VT)) {
+ Result = LegalizeOp(UnrollVectorOp(Op));
+ break;
+ }
+
+ // We always lower FPOWI into a libcall. No target support for it yet.
+ RTLIB::Libcall LC =
+ VT == MVT::f32 ? RTLIB::POWI_F32 :
+ VT == MVT::f64 ? RTLIB::POWI_F64 :
+ VT == MVT::f80 ? RTLIB::POWI_F80 :
+ VT == MVT::ppcf128 ? RTLIB::POWI_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL;
SDOperand Dummy;
Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
false/*sign irrelevant*/, Dummy);
case ISD::BIT_CONVERT:
if (!isTypeLegal(Node->getOperand(0).getValueType())) {
Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
+ } else if (MVT::isVector(Op.getOperand(0).getValueType())) {
+ // The input has to be a vector type, we have to either scalarize it, pack
+ // it, or convert it based on whether the input vector type is legal.
+ SDNode *InVal = Node->getOperand(0).Val;
+ int InIx = Node->getOperand(0).ResNo;
+ unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(InIx));
+ MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(InIx));
+
+ // Figure out if there is a simple type corresponding to this Vector
+ // type. If so, convert to the vector type.
+ MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
+ if (TLI.isTypeLegal(TVT)) {
+ // Turn this into a bit convert of the vector input.
+ Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
+ LegalizeOp(Node->getOperand(0)));
+ break;
+ } else if (NumElems == 1) {
+ // Turn this into a bit convert of the scalar input.
+ Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
+ ScalarizeVectorOp(Node->getOperand(0)));
+ break;
+ } else {
+ // FIXME: UNIMP! Store then reload
+ assert(0 && "Cast from unsupported vector type not implemented yet!");
+ }
} else {
switch (TLI.getOperationAction(ISD::BIT_CONVERT,
Node->getOperand(0).getValueType())) {
}
}
break;
- case ISD::VBIT_CONVERT: {
- assert(Op.getOperand(0).getValueType() == MVT::Vector &&
- "Can only have VBIT_CONVERT where input or output is MVT::Vector!");
-
- // The input has to be a vector type, we have to either scalarize it, pack
- // it, or convert it based on whether the input vector type is legal.
- SDNode *InVal = Node->getOperand(0).Val;
- unsigned NumElems =
- cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-
- // Figure out if there is a Packed type corresponding to this Vector
- // type. If so, convert to the vector type.
- MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
- if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
- // Turn this into a bit convert of the packed input.
- Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
- PackVectorOp(Node->getOperand(0), TVT));
- break;
- } else if (NumElems == 1) {
- // Turn this into a bit convert of the scalar input.
- Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
- PackVectorOp(Node->getOperand(0), EVT));
- break;
- } else {
- // FIXME: UNIMP! Store then reload
- assert(0 && "Cast from unsupported vector type not implemented yet!");
- }
- }
// Conversion operators. The source and destination have different types.
case ISD::SINT_TO_FP:
SDOperand True, False;
MVT::ValueType VT = Node->getOperand(0).getValueType();
MVT::ValueType NVT = Node->getValueType(0);
- unsigned ShiftAmt = MVT::getSizeInBits(Node->getValueType(0))-1;
- Tmp2 = DAG.getConstantFP((double)(1ULL << ShiftAmt), VT);
+ unsigned ShiftAmt = MVT::getSizeInBits(NVT)-1;
+ const uint64_t zero[] = {0, 0};
+ APFloat apf = APFloat(APInt(MVT::getSizeInBits(VT), 2, zero));
+ uint64_t x = 1ULL << ShiftAmt;
+ (void)apf.convertFromZeroExtendedInteger
+ (&x, MVT::getSizeInBits(NVT), false, APFloat::rmNearestTiesToEven);
+ Tmp2 = DAG.getConstantFP(apf, VT);
Tmp3 = DAG.getSetCC(TLI.getSetCCResultTy(),
Node->getOperand(0), Tmp2, ISD::SETLT);
True = DAG.getNode(ISD::FP_TO_SINT, NVT, Node->getOperand(0));
}
break;
case Expand: {
- // Convert f32 / f64 to i32 / i64.
MVT::ValueType VT = Op.getValueType();
+ MVT::ValueType OVT = Node->getOperand(0).getValueType();
+ // Convert ppcf128 to i32
+ if (OVT == MVT::ppcf128 && VT == MVT::i32) {
+ if (Node->getOpcode()==ISD::FP_TO_SINT)
+ Result = DAG.getNode(ISD::FP_TO_SINT, VT,
+ DAG.getNode(ISD::FP_ROUND, MVT::f64,
+ (DAG.getNode(ISD::FP_ROUND_INREG,
+ MVT::ppcf128, Node->getOperand(0),
+ DAG.getValueType(MVT::f64)))));
+ else {
+ const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
+ APFloat apf = APFloat(APInt(128, 2, TwoE31));
+ Tmp2 = DAG.getConstantFP(apf, OVT);
+ // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
+ // FIXME: generated code sucks.
+ Result = DAG.getNode(ISD::SELECT_CC, VT, Node->getOperand(0), Tmp2,
+ DAG.getNode(ISD::ADD, MVT::i32,
+ DAG.getNode(ISD::FP_TO_SINT, VT,
+ DAG.getNode(ISD::FSUB, OVT,
+ Node->getOperand(0), Tmp2)),
+ DAG.getConstant(0x80000000, MVT::i32)),
+ DAG.getNode(ISD::FP_TO_SINT, VT,
+ Node->getOperand(0)),
+ DAG.getCondCode(ISD::SETGE));
+ }
+ break;
+ }
+ // Convert f32 / f64 to i32 / i64.
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
switch (Node->getOpcode()) {
- case ISD::FP_TO_SINT:
- if (Node->getOperand(0).getValueType() == MVT::f32)
+ case ISD::FP_TO_SINT: {
+ if (OVT == MVT::f32)
LC = (VT == MVT::i32)
? RTLIB::FPTOSINT_F32_I32 : RTLIB::FPTOSINT_F32_I64;
- else
+ else if (OVT == MVT::f64)
LC = (VT == MVT::i32)
? RTLIB::FPTOSINT_F64_I32 : RTLIB::FPTOSINT_F64_I64;
+ else if (OVT == MVT::f80) {
+ assert(VT == MVT::i64);
+ LC = RTLIB::FPTOSINT_F80_I64;
+ }
+ else if (OVT == MVT::ppcf128) {
+ assert(VT == MVT::i64);
+ LC = RTLIB::FPTOSINT_PPCF128_I64;
+ }
break;
- case ISD::FP_TO_UINT:
- if (Node->getOperand(0).getValueType() == MVT::f32)
+ }
+ case ISD::FP_TO_UINT: {
+ if (OVT == MVT::f32)
LC = (VT == MVT::i32)
? RTLIB::FPTOUINT_F32_I32 : RTLIB::FPTOSINT_F32_I64;
- else
+ else if (OVT == MVT::f64)
LC = (VT == MVT::i32)
? RTLIB::FPTOUINT_F64_I32 : RTLIB::FPTOSINT_F64_I64;
+ else if (OVT == MVT::f80) {
+ LC = (VT == MVT::i32)
+ ? RTLIB::FPTOUINT_F80_I32 : RTLIB::FPTOUINT_F80_I64;
+ }
+ else if (OVT == MVT::ppcf128) {
+ assert(VT == MVT::i64);
+ LC = RTLIB::FPTOUINT_PPCF128_I64;
+ }
break;
+ }
default: assert(0 && "Unreachable!");
}
SDOperand Dummy;
}
break;
+ case ISD::FP_EXTEND:
+ case ISD::FP_ROUND: {
+ MVT::ValueType newVT = Op.getValueType();
+ MVT::ValueType oldVT = Op.getOperand(0).getValueType();
+ if (TLI.getConvertAction(oldVT, newVT) == TargetLowering::Expand) {
+ if (Node->getOpcode() == ISD::FP_ROUND && oldVT == MVT::ppcf128) {
+ SDOperand Lo, Hi;
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ if (newVT == MVT::f64)
+ Result = Hi;
+ else
+ Result = DAG.getNode(ISD::FP_ROUND, newVT, Hi);
+ break;
+ } else {
+ // The only other way we can lower this is to turn it into a STORE,
+ // LOAD pair, targetting a temporary location (a stack slot).
+
+ // NOTE: there is a choice here between constantly creating new stack
+ // slots and always reusing the same one. We currently always create
+ // new ones, as reuse may inhibit scheduling.
+ MVT::ValueType slotVT =
+ (Node->getOpcode() == ISD::FP_EXTEND) ? oldVT : newVT;
+ const Type *Ty = MVT::getTypeForValueType(slotVT);
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
+ unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty);
+ MachineFunction &MF = DAG.getMachineFunction();
+ int SSFI =
+ MF.getFrameInfo()->CreateStackObject(TySize, Align);
+ SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
+ if (Node->getOpcode() == ISD::FP_EXTEND) {
+ Result = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0),
+ StackSlot, NULL, 0);
+ Result = DAG.getExtLoad(ISD::EXTLOAD, newVT,
+ Result, StackSlot, NULL, 0, oldVT);
+ } else {
+ Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
+ StackSlot, NULL, 0, newVT);
+ Result = DAG.getLoad(newVT, Result, StackSlot, NULL, 0);
+ }
+ break;
+ }
+ }
+ }
+ // FALL THROUGH
case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::SIGN_EXTEND:
- case ISD::FP_EXTEND:
- case ISD::FP_ROUND:
switch (getTypeAction(Node->getOperand(0).getValueType())) {
case Expand: assert(0 && "Shouldn't need to expand other operators here!");
case Legal:
// slots and always reusing the same one. We currently always create
// new ones, as reuse may inhibit scheduling.
const Type *Ty = MVT::getTypeForValueType(ExtraVT);
- unsigned TySize = (unsigned)TLI.getTargetData()->getTypeSize(Ty);
+ uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
unsigned Align = TLI.getTargetData()->getPrefTypeAlignment(Ty);
MachineFunction &MF = DAG.getMachineFunction();
int SSFI =
- MF.getFrameInfo()->CreateStackObject((unsigned)TySize, Align);
+ MF.getFrameInfo()->CreateStackObject(TySize, Align);
SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
StackSlot, NULL, 0, ExtraVT);
}
break;
}
+ case ISD::TRAMPOLINE: {
+ SDOperand Ops[6];
+ for (unsigned i = 0; i != 6; ++i)
+ Ops[i] = LegalizeOp(Node->getOperand(i));
+ Result = DAG.UpdateNodeOperands(Result, Ops, 6);
+ // The only option for this node is to custom lower it.
+ Result = TLI.LowerOperation(Result, DAG);
+ assert(Result.Val && "Should always custom lower!");
+
+ // Since trampoline produces two values, make sure to remember that we
+ // legalized both of them.
+ Tmp1 = LegalizeOp(Result.getValue(1));
+ Result = LegalizeOp(Result);
+ AddLegalizedOperand(SDOperand(Node, 0), Result);
+ AddLegalizedOperand(SDOperand(Node, 1), Tmp1);
+ return Op.ResNo ? Tmp1 : Result;
+ }
}
assert(Result.getValueType() == Op.getValueType() &&
assert(0 && "CopyFromReg must be legal!");
default:
#ifndef NDEBUG
- cerr << "NODE: "; Node->dump(); cerr << "\n";
+ cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
#endif
assert(0 && "Do not know how to promote this operator!");
abort();
DAG.getValueType(VT));
break;
+ case ISD::FPOWI: {
+ // Promote f32 powi to f64 powi. Note that this could insert a libcall
+ // directly as well, which may be better.
+ Tmp1 = PromoteOp(Node->getOperand(0));
+ assert(Tmp1.getValueType() == NVT);
+ Result = DAG.getNode(ISD::FPOWI, NVT, Tmp1, Node->getOperand(1));
+ if (NoExcessFPPrecision)
+ Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
+ DAG.getValueType(VT));
+ break;
+ }
+
case ISD::AND:
case ISD::OR:
case ISD::XOR:
Result = DAG.getExtLoad(ExtType, NVT,
LD->getChain(), LD->getBasePtr(),
LD->getSrcValue(), LD->getSrcValueOffset(),
- LD->getLoadedVT());
+ LD->getLoadedVT(),
+ LD->isVolatile(),
+ LD->getAlignment());
// Remember that we legalized the chain.
AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
break;
Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
- DAG.getConstant(getSizeInBits(NVT) - getSizeInBits(VT),
+ DAG.getConstant(MVT::getSizeInBits(NVT) -
+ MVT::getSizeInBits(VT),
TLI.getShiftAmountTy()));
break;
case ISD::CTPOP:
case ISD::CTTZ:
// if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1,
- DAG.getConstant(getSizeInBits(NVT), NVT), ISD::SETEQ);
+ DAG.getConstant(MVT::getSizeInBits(NVT), NVT),
+ ISD::SETEQ);
Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
- DAG.getConstant(getSizeInBits(VT), NVT), Tmp1);
+ DAG.getConstant(MVT::getSizeInBits(VT), NVT), Tmp1);
break;
case ISD::CTLZ:
//Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
- DAG.getConstant(getSizeInBits(NVT) -
- getSizeInBits(VT), NVT));
+ DAG.getConstant(MVT::getSizeInBits(NVT) -
+ MVT::getSizeInBits(VT), NVT));
break;
}
break;
- case ISD::VEXTRACT_VECTOR_ELT:
- Result = PromoteOp(LowerVEXTRACT_VECTOR_ELT(Op));
+ case ISD::EXTRACT_SUBVECTOR:
+ Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op));
break;
case ISD::EXTRACT_VECTOR_ELT:
Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op));
return Result;
}
-/// LowerVEXTRACT_VECTOR_ELT - Lower a VEXTRACT_VECTOR_ELT operation into a
-/// EXTRACT_VECTOR_ELT operation, to memory operations, or to scalar code based
-/// on the vector type. The return type of this matches the element type of the
-/// vector, which may not be legal for the target.
-SDOperand SelectionDAGLegalize::LowerVEXTRACT_VECTOR_ELT(SDOperand Op) {
+/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
+/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic,
+/// based on the vector type. The return type of this matches the element type
+/// of the vector, which may not be legal for the target.
+SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
// We know that operand #0 is the Vec vector. If the index is a constant
// or if the invec is a supported hardware type, we can use it. Otherwise,
// lower to a store then an indexed load.
SDOperand Vec = Op.getOperand(0);
- SDOperand Idx = LegalizeOp(Op.getOperand(1));
+ SDOperand Idx = Op.getOperand(1);
- SDNode *InVal = Vec.Val;
- unsigned NumElems = cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
+ MVT::ValueType TVT = Vec.getValueType();
+ unsigned NumElems = MVT::getVectorNumElements(TVT);
- // Figure out if there is a Packed type corresponding to this Vector
- // type. If so, convert to the vector type.
- MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
- if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
- // Turn this into a packed extract_vector_elt operation.
- Vec = PackVectorOp(Vec, TVT);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Op.getValueType(), Vec, Idx);
- } else if (NumElems == 1) {
+ switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) {
+ default: assert(0 && "This action is not supported yet!");
+ case TargetLowering::Custom: {
+ Vec = LegalizeOp(Vec);
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ SDOperand Tmp3 = TLI.LowerOperation(Op, DAG);
+ if (Tmp3.Val)
+ return Tmp3;
+ break;
+ }
+ case TargetLowering::Legal:
+ if (isTypeLegal(TVT)) {
+ Vec = LegalizeOp(Vec);
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ return Op;
+ }
+ break;
+ case TargetLowering::Expand:
+ break;
+ }
+
+ if (NumElems == 1) {
// This must be an access of the only element. Return it.
- return PackVectorOp(Vec, EVT);
- } else if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
+ Op = ScalarizeVectorOp(Vec);
+ } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) {
+ ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
SDOperand Lo, Hi;
SplitVectorOp(Vec, Lo, Hi);
if (CIdx->getValue() < NumElems/2) {
Vec = Lo;
} else {
Vec = Hi;
- Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType());
+ Idx = DAG.getConstant(CIdx->getValue() - NumElems/2,
+ Idx.getValueType());
}
-
+
// It's now an extract from the appropriate high or low part. Recurse.
Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
- return LowerVEXTRACT_VECTOR_ELT(Op);
+ Op = ExpandEXTRACT_VECTOR_ELT(Op);
} else {
- // Variable index case for extract element.
- // FIXME: IMPLEMENT STORE/LOAD lowering. Need alignment of stack slot!!
- assert(0 && "unimp!");
- return SDOperand();
+ // Store the value to a temporary stack slot, then LOAD the scalar
+ // element back out.
+ SDOperand StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
+ SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0);
+
+ // Add the offset to the index.
+ unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
+ Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
+ DAG.getConstant(EltSize, Idx.getValueType()));
+
+ if (MVT::getSizeInBits(Idx.getValueType()) >
+ MVT::getSizeInBits(TLI.getPointerTy()))
+ Idx = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), Idx);
+ else
+ Idx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), Idx);
+
+ StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
+
+ Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
}
+ return Op;
}
-/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
-/// memory traffic.
-SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
- SDOperand Vector = Op.getOperand(0);
- SDOperand Idx = Op.getOperand(1);
+/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now
+/// we assume the operation can be split if it is not already legal.
+SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) {
+ // We know that operand #0 is the Vec vector. For now we assume the index
+ // is a constant and that the extracted result is a supported hardware type.
+ SDOperand Vec = Op.getOperand(0);
+ SDOperand Idx = LegalizeOp(Op.getOperand(1));
- // If the target doesn't support this, store the value to a temporary
- // stack slot, then LOAD the scalar element back out.
- SDOperand StackPtr = CreateStackTemporary(Vector.getValueType());
- SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vector, StackPtr, NULL, 0);
+ unsigned NumElems = MVT::getVectorNumElements(Vec.getValueType());
- // Add the offset to the index.
- unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
- Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
- DAG.getConstant(EltSize, Idx.getValueType()));
- StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
+ if (NumElems == MVT::getVectorNumElements(Op.getValueType())) {
+ // This must be an access of the desired vector length. Return it.
+ return Vec;
+ }
+
+ ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
+ SDOperand Lo, Hi;
+ SplitVectorOp(Vec, Lo, Hi);
+ if (CIdx->getValue() < NumElems/2) {
+ Vec = Lo;
+ } else {
+ Vec = Hi;
+ Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType());
+ }
- return DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
+ // It's now an extract from the appropriate high or low part. Recurse.
+ Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
+ return ExpandEXTRACT_SUBVECTOR(Op);
}
-
/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC
/// with condition CC on the current target. This usually involves legalizing
/// or promoting the arguments. In the case where LHS and RHS must be expanded,
void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS,
SDOperand &RHS,
SDOperand &CC) {
- SDOperand Tmp1, Tmp2, Result;
+ SDOperand Tmp1, Tmp2, Tmp3, Result;
switch (getTypeAction(LHS.getValueType())) {
case Legal:
SDOperand LHSLo, LHSHi, RHSLo, RHSHi;
ExpandOp(LHS, LHSLo, LHSHi);
- ExpandOp(RHS, RHSLo, RHSHi);
- switch (cast<CondCodeSDNode>(CC)->get()) {
+ ExpandOp(RHS, RHSLo, RHSHi);
+ ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
+
+ if (VT==MVT::ppcf128) {
+ // FIXME: This generated code sucks. We want to generate
+ // FCMP crN, hi1, hi2
+ // BNE crN, L:
+ // FCMP crN, lo1, lo2
+ // The following can be improved, but not that much.
+ Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETEQ);
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, CCCode);
+ Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
+ Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETNE);
+ Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, CCCode);
+ Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
+ Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3);
+ Tmp2 = SDOperand();
+ break;
+ }
+
+ switch (CCCode) {
case ISD::SETEQ:
case ISD::SETNE:
if (RHSLo == RHSHi)
// FIXME: This generated code sucks.
ISD::CondCode LowCC;
- ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
switch (CCCode) {
default: assert(0 && "Unknown integer setcc!");
case ISD::SETLT:
Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
CCCode, false, DagCombineInfo);
if (!Tmp2.Val)
- Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHSHi, RHSHi, CC);
+ Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHSHi, RHSHi,CC);
ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val);
ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val);
SDOperand SelectionDAGLegalize::ExpandBIT_CONVERT(MVT::ValueType DestVT,
SDOperand SrcOp) {
// Create the stack frame object.
- SDOperand FIPtr = CreateStackTemporary(DestVT);
+ SDOperand FIPtr = DAG.CreateStackTemporary(DestVT);
// Emit a store to the stack slot.
SDOperand Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr, NULL, 0);
SDOperand SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
// Create a vector sized/aligned stack slot, store the value to element #0,
// then load the whole vector back out.
- SDOperand StackPtr = CreateStackTemporary(Node->getValueType(0));
+ SDOperand StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr,
NULL, 0);
return DAG.getLoad(Node->getValueType(0), Ch, StackPtr, NULL, 0);
/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't
-/// support the operation, but do support the resultant packed vector type.
+/// support the operation, but do support the resultant vector type.
SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
// If the only non-undef value is the low element, turn this into a
for (unsigned i = 0, e = NumElems; i != e; ++i) {
if (ConstantFPSDNode *V =
dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
- CV.push_back(ConstantFP::get(OpNTy, V->getValue()));
+ CV.push_back(ConstantFP::get(OpNTy, V->getValueAPF()));
} else if (ConstantSDNode *V =
dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
CV.push_back(ConstantInt::get(OpNTy, V->getValue()));
// Build the shuffle constant vector: <0, 0, 0, 0>
MVT::ValueType MaskVT =
MVT::getIntVectorWithNumElements(NumElems);
- SDOperand Zero = DAG.getConstant(0, MVT::getVectorBaseType(MaskVT));
+ SDOperand Zero = DAG.getConstant(0, MVT::getVectorElementType(MaskVT));
std::vector<SDOperand> ZeroVec(NumElems, Zero);
SDOperand SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
&ZeroVec[0], ZeroVec.size());
E = Values.end(); I != E; ++I) {
for (std::vector<unsigned>::iterator II = I->second.begin(),
EE = I->second.end(); II != EE; ++II)
- MaskVec[*II] = DAG.getConstant(i, MVT::getVectorBaseType(MaskVT));
+ MaskVec[*II] = DAG.getConstant(i, MVT::getVectorElementType(MaskVT));
i += NumElems;
}
SDOperand ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
// the result as a vector.
MVT::ValueType VT = Node->getValueType(0);
// Create the stack frame object.
- SDOperand FIPtr = CreateStackTemporary(VT);
+ SDOperand FIPtr = DAG.CreateStackTemporary(VT);
// Emit a store of each element to the stack slot.
SmallVector<SDOperand, 8> Stores;
return DAG.getLoad(VT, StoreChain, FIPtr, NULL, 0);
}
-/// CreateStackTemporary - Create a stack temporary, suitable for holding the
-/// specified value type.
-SDOperand SelectionDAGLegalize::CreateStackTemporary(MVT::ValueType VT) {
- MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
- unsigned ByteSize = MVT::getSizeInBits(VT)/8;
- const Type *Ty = MVT::getTypeForValueType(VT);
- unsigned StackAlign = (unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty);
- int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign);
- return DAG.getFrameIndex(FrameIdx, TLI.getPointerTy());
-}
-
void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp,
SDOperand Op, SDOperand Amt,
SDOperand &Lo, SDOperand &Hi) {
unsigned VTBits = MVT::getSizeInBits(Op.getValueType());
unsigned NVTBits = MVT::getSizeInBits(NVT);
- // Handle the case when Amt is an immediate. Other cases are currently broken
- // and are disabled.
+ // Handle the case when Amt is an immediate.
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.Val)) {
unsigned Cst = CN->getValue();
// Expand the incoming operand to be shifted, so that we have its parts
// Okay, the shift amount isn't constant. However, if we can tell that it is
// >= 32 or < 32, we can still simplify it, without knowing the actual value.
uint64_t Mask = NVTBits, KnownZero, KnownOne;
- TLI.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne);
+ DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne);
// If we know that the high bit of the shift amount is one, then we can do
// this as a couple of simple shifts.
MVT::ValueType ArgVT = Node->getOperand(i).getValueType();
const Type *ArgTy = MVT::getTypeForValueType(ArgVT);
Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
- Entry.isSigned = isSigned; Entry.isInReg = false; Entry.isSRet = false;
+ Entry.isSExt = isSigned;
Args.push_back(Entry);
}
SDOperand Callee = DAG.getExternalSymbol(Name, TLI.getPointerTy());
}
-/// ExpandIntToFP - Expand a [US]INT_TO_FP operation, assuming that the
-/// destination type is legal.
+/// ExpandIntToFP - Expand a [US]INT_TO_FP operation.
+///
SDOperand SelectionDAGLegalize::
ExpandIntToFP(bool isSigned, MVT::ValueType DestTy, SDOperand Source) {
- assert(isTypeLegal(DestTy) && "Destination type is not legal!");
assert(getTypeAction(Source.getValueType()) == Expand &&
"This is not an expansion!");
assert(Source.getValueType() == MVT::i64 && "Only handle expand from i64!");
SDOperand FudgeInReg;
if (DestTy == MVT::f32)
FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
- else {
- assert(DestTy == MVT::f64 && "Unexpected conversion");
- FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, MVT::f64, DAG.getEntryNode(),
+ else if (MVT::getSizeInBits(DestTy) > MVT::getSizeInBits(MVT::f32))
+ // FIXME: Avoid the extend by construction the right constantpool?
+ FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(),
CPIdx, NULL, 0, MVT::f32);
+ else
+ assert(0 && "Unexpected conversion");
+
+ MVT::ValueType SCVT = SignedConv.getValueType();
+ if (SCVT != DestTy) {
+ // Destination type needs to be expanded as well. The FADD now we are
+ // constructing will be expanded into a libcall.
+ if (MVT::getSizeInBits(SCVT) != MVT::getSizeInBits(DestTy)) {
+ assert(SCVT == MVT::i32 && DestTy == MVT::f64);
+ SignedConv = DAG.getNode(ISD::BUILD_PAIR, MVT::i64,
+ SignedConv, SignedConv.getValue(1));
+ }
+ SignedConv = DAG.getNode(ISD::BIT_CONVERT, DestTy, SignedConv);
}
return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg);
}
LC = RTLIB::SINTTOFP_I64_F64;
}
+ assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!");
Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source);
SDOperand UnusedHiPart;
return ExpandLibCall(TLI.getLibcallName(LC), Source.Val, isSigned,
if (DestVT == MVT::f64) {
// do nothing
Result = Sub;
- } else {
- // if f32 then cast to f32
- Result = DAG.getNode(ISD::FP_ROUND, MVT::f32, Sub);
+ } else if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(MVT::f64)) {
+ Result = DAG.getNode(ISD::FP_ROUND, DestVT, Sub);
+ } else if (MVT::getSizeInBits(DestVT) > MVT::getSizeInBits(MVT::f64)) {
+ Result = DAG.getNode(ISD::FP_EXTEND, DestVT, Sub);
}
return Result;
}
if (DestVT == MVT::f32)
FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
else {
- assert(DestVT == MVT::f64 && "Unexpected conversion");
- FudgeInReg = LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, MVT::f64,
+ FudgeInReg = LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, DestVT,
DAG.getEntryNode(), CPIdx,
NULL, 0, MVT::f32));
}
};
MVT::ValueType VT = Op.getValueType();
MVT::ValueType ShVT = TLI.getShiftAmountTy();
- unsigned len = getSizeInBits(VT);
+ unsigned len = MVT::getSizeInBits(VT);
for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
//x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8])
SDOperand Tmp2 = DAG.getConstant(mask[i], VT);
// but see also: http://www.hackersdelight.org/HDcode/nlz.cc
MVT::ValueType VT = Op.getValueType();
MVT::ValueType ShVT = TLI.getShiftAmountTy();
- unsigned len = getSizeInBits(VT);
+ unsigned len = MVT::getSizeInBits(VT);
for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3));
if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
TLI.isOperationLegal(ISD::CTLZ, VT))
return DAG.getNode(ISD::SUB, VT,
- DAG.getConstant(getSizeInBits(VT), VT),
+ DAG.getConstant(MVT::getSizeInBits(VT), VT),
DAG.getNode(ISD::CTLZ, VT, Tmp3));
return DAG.getNode(ISD::CTPOP, VT, Tmp3);
}
SDNode *Node = Op.Val;
assert(getTypeAction(VT) == Expand && "Not an expanded type!");
assert(((MVT::isInteger(NVT) && NVT < VT) || MVT::isFloatingPoint(VT) ||
- VT == MVT::Vector) &&
+ MVT::isVector(VT)) &&
"Cannot expand to FP value or to larger int value!");
// See if we already expanded it.
switch (Node->getOpcode()) {
case ISD::CopyFromReg:
assert(0 && "CopyFromReg must be legal!");
+ case ISD::FP_ROUND_INREG:
+ if (VT == MVT::ppcf128 &&
+ TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) ==
+ TargetLowering::Custom) {
+ SDOperand SrcLo, SrcHi, Src;
+ ExpandOp(Op.getOperand(0), SrcLo, SrcHi);
+ Src = DAG.getNode(ISD::BUILD_PAIR, VT, SrcLo, SrcHi);
+ SDOperand Result = TLI.LowerOperation(
+ DAG.getNode(ISD::FP_ROUND_INREG, VT, Src, Op.getOperand(1)), DAG);
+ assert(Result.Val->getOpcode() == ISD::BUILD_PAIR);
+ Lo = Result.Val->getOperand(0);
+ Hi = Result.Val->getOperand(1);
+ break;
+ }
+ // fall through
default:
#ifndef NDEBUG
- cerr << "NODE: "; Node->dump(); cerr << "\n";
+ cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
#endif
assert(0 && "Do not know how to expand this operator!");
abort();
}
case ISD::ConstantFP: {
ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
+ if (CFP->getValueType(0) == MVT::ppcf128) {
+ APInt api = CFP->getValueAPF().convertToAPInt();
+ Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])),
+ MVT::f64);
+ Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])),
+ MVT::f64);
+ break;
+ }
Lo = ExpandConstantFP(CFP, false, DAG, TLI);
if (getTypeAction(Lo.getValueType()) == Expand)
ExpandOp(Lo, Lo, Hi);
SDOperand Ch = LD->getChain(); // Legalize the chain.
SDOperand Ptr = LD->getBasePtr(); // Legalize the pointer.
ISD::LoadExtType ExtType = LD->getExtensionType();
+ int SVOffset = LD->getSrcValueOffset();
+ unsigned Alignment = LD->getAlignment();
+ bool isVolatile = LD->isVolatile();
if (ExtType == ISD::NON_EXTLOAD) {
- Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),LD->getSrcValueOffset());
+ Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
+ isVolatile, Alignment);
if (VT == MVT::f32 || VT == MVT::f64) {
// f32->i32 or f64->i64 one to one expansion.
// Remember that we legalized the chain.
unsigned IncrementSize = MVT::getSizeInBits(Lo.getValueType())/8;
Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
getIntPtrConstant(IncrementSize));
- // FIXME: This creates a bogus srcvalue!
- Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),LD->getSrcValueOffset());
+ SVOffset += IncrementSize;
+ Alignment = MinAlign(Alignment, IncrementSize);
+ Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
+ isVolatile, Alignment);
// Build a factor node to remember that this load is independent of the
// other one.
} else {
MVT::ValueType EVT = LD->getLoadedVT();
- if (VT == MVT::f64 && EVT == MVT::f32) {
+ if ((VT == MVT::f64 && EVT == MVT::f32) ||
+ (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) {
// f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(),
- LD->getSrcValueOffset());
+ SVOffset, isVolatile, Alignment);
// Remember that we legalized the chain.
AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1)));
ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi);
if (EVT == NVT)
Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),
- LD->getSrcValueOffset());
+ SVOffset, isVolatile, Alignment);
else
Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(),
- LD->getSrcValueOffset(), EVT);
+ SVOffset, EVT, isVolatile,
+ Alignment);
// Remember that we legalized the chain.
AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
}
}
- RTLIB::Libcall LC;
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (Node->getOperand(0).getValueType() == MVT::f32)
LC = RTLIB::FPTOSINT_F32_I64;
- else
+ else if (Node->getOperand(0).getValueType() == MVT::f64)
LC = RTLIB::FPTOSINT_F64_I64;
+ else if (Node->getOperand(0).getValueType() == MVT::f80)
+ LC = RTLIB::FPTOSINT_F80_I64;
+ else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
+ LC = RTLIB::FPTOSINT_PPCF128_I64;
Lo = ExpandLibCall(TLI.getLibcallName(LC), Node,
false/*sign irrelevant*/, Hi);
break;
}
}
- RTLIB::Libcall LC;
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (Node->getOperand(0).getValueType() == MVT::f32)
LC = RTLIB::FPTOUINT_F32_I64;
- else
+ else if (Node->getOperand(0).getValueType() == MVT::f64)
LC = RTLIB::FPTOUINT_F64_I64;
+ else if (Node->getOperand(0).getValueType() == MVT::f80)
+ LC = RTLIB::FPTOUINT_F80_I64;
+ else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
+ LC = RTLIB::FPTOUINT_PPCF128_I64;
Lo = ExpandLibCall(TLI.getLibcallName(LC), Node,
false/*sign irrelevant*/, Hi);
break;
}
break;
}
+
+ case ISD::ADDC:
+ case ISD::SUBC: {
+ // Expand the subcomponents.
+ SDOperand LHSL, LHSH, RHSL, RHSH;
+ ExpandOp(Node->getOperand(0), LHSL, LHSH);
+ ExpandOp(Node->getOperand(1), RHSL, RHSH);
+ SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
+ SDOperand LoOps[2] = { LHSL, RHSL };
+ SDOperand HiOps[3] = { LHSH, RHSH };
+
+ if (Node->getOpcode() == ISD::ADDC) {
+ Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
+ } else {
+ Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
+ }
+ // Remember that we legalized the flag.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
+ break;
+ }
+ case ISD::ADDE:
+ case ISD::SUBE: {
+ // Expand the subcomponents.
+ SDOperand LHSL, LHSH, RHSL, RHSH;
+ ExpandOp(Node->getOperand(0), LHSL, LHSH);
+ ExpandOp(Node->getOperand(1), RHSL, RHSH);
+ SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
+ SDOperand LoOps[3] = { LHSL, RHSL, Node->getOperand(2) };
+ SDOperand HiOps[3] = { LHSH, RHSH };
+
+ Lo = DAG.getNode(Node->getOpcode(), VTList, LoOps, 3);
+ HiOps[2] = Lo.getValue(1);
+ Hi = DAG.getNode(Node->getOpcode(), VTList, HiOps, 3);
+
+ // Remember that we legalized the flag.
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
+ break;
+ }
case ISD::MUL: {
// If the target wants to custom expand this, let them.
if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) {
bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
- if (HasMULHS || HasMULHU) {
+ bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT);
+ bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT);
+ if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
SDOperand LL, LH, RL, RH;
ExpandOp(Node->getOperand(0), LL, LH);
ExpandOp(Node->getOperand(1), RL, RH);
- unsigned SH = MVT::getSizeInBits(RH.getValueType())-1;
- // FIXME: Move this to the dag combiner.
- // MULHS implicitly sign extends its inputs. Check to see if ExpandOp
- // extended the sign bit of the low half through the upper half, and if so
- // emit a MULHS instead of the alternate sequence that is valid for any
- // i64 x i64 multiply.
- if (HasMULHS &&
- // is RH an extension of the sign bit of RL?
- RH.getOpcode() == ISD::SRA && RH.getOperand(0) == RL &&
- RH.getOperand(1).getOpcode() == ISD::Constant &&
- cast<ConstantSDNode>(RH.getOperand(1))->getValue() == SH &&
- // is LH an extension of the sign bit of LL?
- LH.getOpcode() == ISD::SRA && LH.getOperand(0) == LL &&
- LH.getOperand(1).getOpcode() == ISD::Constant &&
- cast<ConstantSDNode>(LH.getOperand(1))->getValue() == SH) {
- // Low part:
- Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
- // High part:
- Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
+ unsigned BitSize = MVT::getSizeInBits(RH.getValueType());
+ unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0));
+ unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1));
+ // FIXME: generalize this to handle other bit sizes
+ if (LHSSB == 32 && RHSSB == 32 &&
+ DAG.MaskedValueIsZero(Op.getOperand(0), 0xFFFFFFFF00000000ULL) &&
+ DAG.MaskedValueIsZero(Op.getOperand(1), 0xFFFFFFFF00000000ULL)) {
+ // The inputs are both zero-extended.
+ if (HasUMUL_LOHI) {
+ // We can emit a umul_lohi.
+ Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
+ Hi = SDOperand(Lo.Val, 1);
+ break;
+ }
+ if (HasMULHU) {
+ // We can emit a mulhu+mul.
+ Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
+ Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
+ break;
+ }
+ }
+ if (LHSSB > BitSize && RHSSB > BitSize) {
+ // The input values are both sign-extended.
+ if (HasSMUL_LOHI) {
+ // We can emit a smul_lohi.
+ Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
+ Hi = SDOperand(Lo.Val, 1);
+ break;
+ }
+ if (HasMULHS) {
+ // We can emit a mulhs+mul.
+ Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
+ Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
+ break;
+ }
+ }
+ if (HasUMUL_LOHI) {
+ // Lo,Hi = umul LHS, RHS.
+ SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI,
+ DAG.getVTList(NVT, NVT), LL, RL);
+ Lo = UMulLOHI;
+ Hi = UMulLOHI.getValue(1);
+ RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
+ LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
+ Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
+ Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
break;
- } else if (HasMULHU) {
- // Low part:
+ }
+ if (HasMULHU) {
Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
-
- // High part:
Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
}
}
+ // If nothing else, we can make a libcall.
Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::MUL_I64), Node,
false/*sign irrelevant*/, Hi);
break;
break;
case ISD::FADD:
- Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
- ? RTLIB::ADD_F32 : RTLIB::ADD_F64),
+ Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::ADD_F32 :
+ VT == MVT::f64 ? RTLIB::ADD_F64 :
+ VT == MVT::ppcf128 ?
+ RTLIB::ADD_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL),
Node, false, Hi);
break;
case ISD::FSUB:
- Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
- ? RTLIB::SUB_F32 : RTLIB::SUB_F64),
+ Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::SUB_F32 :
+ VT == MVT::f64 ? RTLIB::SUB_F64 :
+ VT == MVT::ppcf128 ?
+ RTLIB::SUB_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL),
Node, false, Hi);
break;
case ISD::FMUL:
- Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
- ? RTLIB::MUL_F32 : RTLIB::MUL_F64),
+ Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::MUL_F32 :
+ VT == MVT::f64 ? RTLIB::MUL_F64 :
+ VT == MVT::ppcf128 ?
+ RTLIB::MUL_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL),
Node, false, Hi);
break;
case ISD::FDIV:
- Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32)
- ? RTLIB::DIV_F32 : RTLIB::DIV_F64),
+ Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::DIV_F32 :
+ VT == MVT::f64 ? RTLIB::DIV_F64 :
+ VT == MVT::ppcf128 ?
+ RTLIB::DIV_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL),
Node, false, Hi);
break;
case ISD::FP_EXTEND:
+ if (VT == MVT::ppcf128) {
+ assert(Node->getOperand(0).getValueType()==MVT::f32 ||
+ Node->getOperand(0).getValueType()==MVT::f64);
+ const uint64_t zero = 0;
+ if (Node->getOperand(0).getValueType()==MVT::f32)
+ Hi = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Node->getOperand(0));
+ else
+ Hi = Node->getOperand(0);
+ Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
+ break;
+ }
Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::FPEXT_F32_F64), Node, true,Hi);
break;
case ISD::FP_ROUND:
Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::FPROUND_F64_F32),Node,true,Hi);
break;
+ case ISD::FPOWI:
+ Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32) ? RTLIB::POWI_F32 :
+ (VT == MVT::f64) ? RTLIB::POWI_F64 :
+ (VT == MVT::f80) ? RTLIB::POWI_F80 :
+ (VT == MVT::ppcf128) ?
+ RTLIB::POWI_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL),
+ Node, false, Hi);
+ break;
case ISD::FSQRT:
case ISD::FSIN:
case ISD::FCOS: {
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
switch(Node->getOpcode()) {
case ISD::FSQRT:
- LC = (VT == MVT::f32) ? RTLIB::SQRT_F32 : RTLIB::SQRT_F64;
+ LC = (VT == MVT::f32) ? RTLIB::SQRT_F32 :
+ (VT == MVT::f64) ? RTLIB::SQRT_F64 :
+ (VT == MVT::f80) ? RTLIB::SQRT_F80 :
+ (VT == MVT::ppcf128) ? RTLIB::SQRT_PPCF128 :
+ RTLIB::UNKNOWN_LIBCALL;
break;
case ISD::FSIN:
LC = (VT == MVT::f32) ? RTLIB::SIN_F32 : RTLIB::SIN_F64;
break;
}
case ISD::FABS: {
+ if (VT == MVT::ppcf128) {
+ SDOperand Tmp;
+ ExpandOp(Node->getOperand(0), Lo, Tmp);
+ Hi = DAG.getNode(ISD::FABS, NVT, Tmp);
+ // lo = hi==fabs(hi) ? lo : -lo;
+ Lo = DAG.getNode(ISD::SELECT_CC, NVT, Hi, Tmp,
+ Lo, DAG.getNode(ISD::FNEG, NVT, Lo),
+ DAG.getCondCode(ISD::SETEQ));
+ break;
+ }
SDOperand Mask = (VT == MVT::f64)
? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
: DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
break;
}
case ISD::FNEG: {
+ if (VT == MVT::ppcf128) {
+ ExpandOp(Node->getOperand(0), Lo, Hi);
+ Lo = DAG.getNode(ISD::FNEG, MVT::f64, Lo);
+ Hi = DAG.getNode(ISD::FNEG, MVT::f64, Hi);
+ break;
+ }
SDOperand Mask = (VT == MVT::f64)
? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT)
: DAG.getConstantFP(BitsToFloat(1U << 31), VT);
case ISD::UINT_TO_FP: {
bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
MVT::ValueType SrcVT = Node->getOperand(0).getValueType();
- RTLIB::Libcall LC;
+ if (VT == MVT::ppcf128 && SrcVT != MVT::i64) {
+ static uint64_t zero = 0;
+ if (isSigned) {
+ Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64,
+ Node->getOperand(0)));
+ Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
+ } else {
+ static uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
+ Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64,
+ Node->getOperand(0)));
+ Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
+ Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi);
+ // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32
+ ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0),
+ DAG.getConstant(0, MVT::i32),
+ DAG.getNode(ISD::FADD, MVT::ppcf128, Hi,
+ DAG.getConstantFP(
+ APFloat(APInt(128, 2, TwoE32)),
+ MVT::ppcf128)),
+ Hi,
+ DAG.getCondCode(ISD::SETLT)),
+ Lo, Hi);
+ }
+ break;
+ }
+ if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) {
+ // si64->ppcf128 done by libcall, below
+ static uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
+ ExpandOp(DAG.getNode(ISD::SINT_TO_FP, MVT::ppcf128, Node->getOperand(0)),
+ Lo, Hi);
+ Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi);
+ // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64
+ ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0),
+ DAG.getConstant(0, MVT::i64),
+ DAG.getNode(ISD::FADD, MVT::ppcf128, Hi,
+ DAG.getConstantFP(
+ APFloat(APInt(128, 2, TwoE64)),
+ MVT::ppcf128)),
+ Hi,
+ DAG.getCondCode(ISD::SETLT)),
+ Lo, Hi);
+ break;
+ }
+ RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (Node->getOperand(0).getValueType() == MVT::i64) {
if (VT == MVT::f32)
LC = isSigned ? RTLIB::SINTTOFP_I64_F32 : RTLIB::UINTTOFP_I64_F32;
- else
+ else if (VT == MVT::f64)
LC = isSigned ? RTLIB::SINTTOFP_I64_F64 : RTLIB::UINTTOFP_I64_F64;
+ else if (VT == MVT::f80) {
+ assert(isSigned);
+ LC = RTLIB::SINTTOFP_I64_F80;
+ }
+ else if (VT == MVT::ppcf128) {
+ assert(isSigned);
+ LC = RTLIB::SINTTOFP_I64_PPCF128;
+ }
} else {
if (VT == MVT::f32)
LC = isSigned ? RTLIB::SINTTOFP_I32_F32 : RTLIB::UINTTOFP_I32_F32;
: DAG.getZeroExtendInReg(Tmp, SrcVT);
Node = DAG.UpdateNodeOperands(Op, Tmp).Val;
}
- Lo = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Hi);
+
+ const char *LibCall = TLI.getLibcallName(LC);
+ if (LibCall)
+ Lo = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Hi);
+ else {
+ Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT,
+ Node->getOperand(0));
+ if (getTypeAction(Lo.getValueType()) == Expand)
+ ExpandOp(Lo, Lo, Hi);
+ }
break;
}
}
assert(isNew && "Value already expanded?!?");
}
-/// SplitVectorOp - Given an operand of MVT::Vector type, break it down into
-/// two smaller values of MVT::Vector type.
+/// SplitVectorOp - Given an operand of vector type, break it down into
+/// two smaller values, still of vector type.
void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
SDOperand &Hi) {
- assert(Op.getValueType() == MVT::Vector && "Cannot split non-vector type!");
+ assert(MVT::isVector(Op.getValueType()) && "Cannot split non-vector type!");
SDNode *Node = Op.Val;
- unsigned NumElements = cast<ConstantSDNode>(*(Node->op_end()-2))->getValue();
+ unsigned NumElements = MVT::getVectorNumElements(Op.getValueType());
assert(NumElements > 1 && "Cannot split a single element vector!");
unsigned NewNumElts = NumElements/2;
- SDOperand NewNumEltsNode = DAG.getConstant(NewNumElts, MVT::i32);
- SDOperand TypeNode = *(Node->op_end()-1);
+ MVT::ValueType NewEltVT = MVT::getVectorElementType(Op.getValueType());
+ MVT::ValueType NewVT = MVT::getVectorType(NewEltVT, NewNumElts);
// See if we already split it.
std::map<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
- Node->dump();
+ Node->dump(&DAG);
#endif
assert(0 && "Unhandled operation in SplitVectorOp!");
- case ISD::VBUILD_VECTOR: {
+ case ISD::BUILD_PAIR:
+ Lo = Node->getOperand(0);
+ Hi = Node->getOperand(1);
+ break;
+ case ISD::INSERT_VECTOR_ELT: {
+ SplitVectorOp(Node->getOperand(0), Lo, Hi);
+ unsigned Index = cast<ConstantSDNode>(Node->getOperand(2))->getValue();
+ SDOperand ScalarOp = Node->getOperand(1);
+ if (Index < NewNumElts)
+ Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT, Lo, ScalarOp,
+ DAG.getConstant(Index, TLI.getPointerTy()));
+ else
+ Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT, Hi, ScalarOp,
+ DAG.getConstant(Index - NewNumElts, TLI.getPointerTy()));
+ break;
+ }
+ case ISD::BUILD_VECTOR: {
SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
Node->op_begin()+NewNumElts);
- LoOps.push_back(NewNumEltsNode);
- LoOps.push_back(TypeNode);
- Lo = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &LoOps[0], LoOps.size());
+ Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &LoOps[0], LoOps.size());
SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumElts,
- Node->op_end()-2);
- HiOps.push_back(NewNumEltsNode);
- HiOps.push_back(TypeNode);
- Hi = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, &HiOps[0], HiOps.size());
+ Node->op_end());
+ Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &HiOps[0], HiOps.size());
+ break;
+ }
+ case ISD::CONCAT_VECTORS: {
+ unsigned NewNumSubvectors = Node->getNumOperands() / 2;
+ if (NewNumSubvectors == 1) {
+ Lo = Node->getOperand(0);
+ Hi = Node->getOperand(1);
+ } else {
+ SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
+ Node->op_begin()+NewNumSubvectors);
+ Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &LoOps[0], LoOps.size());
+
+ SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors,
+ Node->op_end());
+ Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &HiOps[0], HiOps.size());
+ }
break;
}
- case ISD::VADD:
- case ISD::VSUB:
- case ISD::VMUL:
- case ISD::VSDIV:
- case ISD::VUDIV:
- case ISD::VAND:
- case ISD::VOR:
- case ISD::VXOR: {
+ case ISD::SELECT: {
+ SDOperand Cond = Node->getOperand(0);
+
+ SDOperand LL, LH, RL, RH;
+ SplitVectorOp(Node->getOperand(1), LL, LH);
+ SplitVectorOp(Node->getOperand(2), RL, RH);
+
+ if (MVT::isVector(Cond.getValueType())) {
+ // Handle a vector merge.
+ SDOperand CL, CH;
+ SplitVectorOp(Cond, CL, CH);
+ Lo = DAG.getNode(Node->getOpcode(), NewVT, CL, LL, RL);
+ Hi = DAG.getNode(Node->getOpcode(), NewVT, CH, LH, RH);
+ } else {
+ // Handle a simple select with vector operands.
+ Lo = DAG.getNode(Node->getOpcode(), NewVT, Cond, LL, RL);
+ Hi = DAG.getNode(Node->getOpcode(), NewVT, Cond, LH, RH);
+ }
+ break;
+ }
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::SDIV:
+ case ISD::UDIV:
+ case ISD::FDIV:
+ case ISD::FPOW:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR: {
SDOperand LL, LH, RL, RH;
SplitVectorOp(Node->getOperand(0), LL, LH);
SplitVectorOp(Node->getOperand(1), RL, RH);
- Lo = DAG.getNode(Node->getOpcode(), MVT::Vector, LL, RL,
- NewNumEltsNode, TypeNode);
- Hi = DAG.getNode(Node->getOpcode(), MVT::Vector, LH, RH,
- NewNumEltsNode, TypeNode);
+ Lo = DAG.getNode(Node->getOpcode(), NewVT, LL, RL);
+ Hi = DAG.getNode(Node->getOpcode(), NewVT, LH, RH);
break;
}
- case ISD::VLOAD: {
- SDOperand Ch = Node->getOperand(0); // Legalize the chain.
- SDOperand Ptr = Node->getOperand(1); // Legalize the pointer.
- MVT::ValueType EVT = cast<VTSDNode>(TypeNode)->getVT();
-
- Lo = DAG.getVecLoad(NewNumElts, EVT, Ch, Ptr, Node->getOperand(2));
- unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(EVT)/8;
+ case ISD::FPOWI: {
+ SDOperand L, H;
+ SplitVectorOp(Node->getOperand(0), L, H);
+
+ Lo = DAG.getNode(Node->getOpcode(), NewVT, L, Node->getOperand(1));
+ Hi = DAG.getNode(Node->getOpcode(), NewVT, H, Node->getOperand(1));
+ break;
+ }
+ case ISD::CTTZ:
+ case ISD::CTLZ:
+ case ISD::CTPOP:
+ case ISD::FNEG:
+ case ISD::FABS:
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS: {
+ SDOperand L, H;
+ SplitVectorOp(Node->getOperand(0), L, H);
+
+ Lo = DAG.getNode(Node->getOpcode(), NewVT, L);
+ Hi = DAG.getNode(Node->getOpcode(), NewVT, H);
+ break;
+ }
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ SDOperand Ch = LD->getChain();
+ SDOperand Ptr = LD->getBasePtr();
+ const Value *SV = LD->getSrcValue();
+ int SVOffset = LD->getSrcValueOffset();
+ unsigned Alignment = LD->getAlignment();
+ bool isVolatile = LD->isVolatile();
+
+ Lo = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
+ unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(NewEltVT)/8;
Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
getIntPtrConstant(IncrementSize));
- // FIXME: This creates a bogus srcvalue!
- Hi = DAG.getVecLoad(NewNumElts, EVT, Ch, Ptr, Node->getOperand(2));
+ SVOffset += IncrementSize;
+ Alignment = MinAlign(Alignment, IncrementSize);
+ Hi = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
// Build a factor node to remember that this load is independent of the
// other one.
AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
break;
}
- case ISD::VBIT_CONVERT: {
+ case ISD::BIT_CONVERT: {
// We know the result is a vector. The input may be either a vector or a
// scalar value.
- if (Op.getOperand(0).getValueType() != MVT::Vector) {
- // Lower to a store/load. FIXME: this could be improved probably.
- SDOperand Ptr = CreateStackTemporary(Op.getOperand(0).getValueType());
+ SDOperand InOp = Node->getOperand(0);
+ if (!MVT::isVector(InOp.getValueType()) ||
+ MVT::getVectorNumElements(InOp.getValueType()) == 1) {
+ // The input is a scalar or single-element vector.
+ // Lower to a store/load so that it can be split.
+ // FIXME: this could be improved probably.
+ SDOperand Ptr = DAG.CreateStackTemporary(InOp.getValueType());
SDOperand St = DAG.getStore(DAG.getEntryNode(),
- Op.getOperand(0), Ptr, NULL, 0);
- MVT::ValueType EVT = cast<VTSDNode>(TypeNode)->getVT();
- St = DAG.getVecLoad(NumElements, EVT, St, Ptr, DAG.getSrcValue(0));
- SplitVectorOp(St, Lo, Hi);
- } else {
- // If the input is a vector type, we have to either scalarize it, pack it
- // or convert it based on whether the input vector type is legal.
- SDNode *InVal = Node->getOperand(0).Val;
- unsigned NumElems =
- cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-
- // If the input is from a single element vector, scalarize the vector,
- // then treat like a scalar.
- if (NumElems == 1) {
- SDOperand Scalar = PackVectorOp(Op.getOperand(0), EVT);
- Scalar = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Scalar,
- Op.getOperand(1), Op.getOperand(2));
- SplitVectorOp(Scalar, Lo, Hi);
- } else {
- // Split the input vector.
- SplitVectorOp(Op.getOperand(0), Lo, Hi);
-
- // Convert each of the pieces now.
- Lo = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Lo,
- NewNumEltsNode, TypeNode);
- Hi = DAG.getNode(ISD::VBIT_CONVERT, MVT::Vector, Hi,
- NewNumEltsNode, TypeNode);
- }
- break;
+ InOp, Ptr, NULL, 0);
+ InOp = DAG.getLoad(Op.getValueType(), St, Ptr, NULL, 0);
}
+ // Split the vector and convert each of the pieces now.
+ SplitVectorOp(InOp, Lo, Hi);
+ Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT, Lo);
+ Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT, Hi);
+ break;
}
}
// Remember in a map if the values will be reused later.
bool isNew =
SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
- assert(isNew && "Value already expanded?!?");
+ assert(isNew && "Value already split?!?");
}
-/// PackVectorOp - Given an operand of MVT::Vector type, convert it into the
-/// equivalent operation that returns a scalar (e.g. F32) or packed value
-/// (e.g. MVT::V4F32). When this is called, we know that PackedVT is the right
-/// type for the result.
-SDOperand SelectionDAGLegalize::PackVectorOp(SDOperand Op,
- MVT::ValueType NewVT) {
- assert(Op.getValueType() == MVT::Vector && "Bad PackVectorOp invocation!");
+/// ScalarizeVectorOp - Given an operand of single-element vector type
+/// (e.g. v1f32), convert it into the equivalent operation that returns a
+/// scalar (e.g. f32) value.
+SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) {
+ assert(MVT::isVector(Op.getValueType()) &&
+ "Bad ScalarizeVectorOp invocation!");
SDNode *Node = Op.Val;
+ MVT::ValueType NewVT = MVT::getVectorElementType(Op.getValueType());
+ assert(MVT::getVectorNumElements(Op.getValueType()) == 1);
- // See if we already packed it.
- std::map<SDOperand, SDOperand>::iterator I = PackedNodes.find(Op);
- if (I != PackedNodes.end()) return I->second;
+ // See if we already scalarized it.
+ std::map<SDOperand, SDOperand>::iterator I = ScalarizedNodes.find(Op);
+ if (I != ScalarizedNodes.end()) return I->second;
SDOperand Result;
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
- Node->dump(); cerr << "\n";
+ Node->dump(&DAG); cerr << "\n";
#endif
- assert(0 && "Unknown vector operation in PackVectorOp!");
- case ISD::VADD:
- case ISD::VSUB:
- case ISD::VMUL:
- case ISD::VSDIV:
- case ISD::VUDIV:
- case ISD::VAND:
- case ISD::VOR:
- case ISD::VXOR:
- Result = DAG.getNode(getScalarizedOpcode(Node->getOpcode(), NewVT),
+ assert(0 && "Unknown vector operation in ScalarizeVectorOp!");
+ case ISD::ADD:
+ case ISD::FADD:
+ case ISD::SUB:
+ case ISD::FSUB:
+ case ISD::MUL:
+ case ISD::FMUL:
+ case ISD::SDIV:
+ case ISD::UDIV:
+ case ISD::FDIV:
+ case ISD::SREM:
+ case ISD::UREM:
+ case ISD::FREM:
+ case ISD::FPOW:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ Result = DAG.getNode(Node->getOpcode(),
NewVT,
- PackVectorOp(Node->getOperand(0), NewVT),
- PackVectorOp(Node->getOperand(1), NewVT));
+ ScalarizeVectorOp(Node->getOperand(0)),
+ ScalarizeVectorOp(Node->getOperand(1)));
break;
- case ISD::VLOAD: {
- SDOperand Ch = LegalizeOp(Node->getOperand(0)); // Legalize the chain.
- SDOperand Ptr = LegalizeOp(Node->getOperand(1)); // Legalize the pointer.
-
- SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2));
- Result = DAG.getLoad(NewVT, Ch, Ptr, SV->getValue(), SV->getOffset());
+ case ISD::FNEG:
+ case ISD::FABS:
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS:
+ Result = DAG.getNode(Node->getOpcode(),
+ NewVT,
+ ScalarizeVectorOp(Node->getOperand(0)));
+ break;
+ case ISD::FPOWI:
+ Result = DAG.getNode(Node->getOpcode(),
+ NewVT,
+ ScalarizeVectorOp(Node->getOperand(0)),
+ Node->getOperand(1));
+ break;
+ case ISD::LOAD: {
+ LoadSDNode *LD = cast<LoadSDNode>(Node);
+ SDOperand Ch = LegalizeOp(LD->getChain()); // Legalize the chain.
+ SDOperand Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer.
+ const Value *SV = LD->getSrcValue();
+ int SVOffset = LD->getSrcValueOffset();
+ Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset,
+ LD->isVolatile(), LD->getAlignment());
+
// Remember that we legalized the chain.
AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
break;
}
- case ISD::VBUILD_VECTOR:
- if (Node->getOperand(0).getValueType() == NewVT) {
- // Returning a scalar?
- Result = Node->getOperand(0);
- } else {
- // Returning a BUILD_VECTOR?
-
- // If all elements of the build_vector are undefs, return an undef.
- bool AllUndef = true;
- for (unsigned i = 0, e = Node->getNumOperands()-2; i != e; ++i)
- if (Node->getOperand(i).getOpcode() != ISD::UNDEF) {
- AllUndef = false;
- break;
- }
- if (AllUndef) {
- Result = DAG.getNode(ISD::UNDEF, NewVT);
- } else {
- Result = DAG.getNode(ISD::BUILD_VECTOR, NewVT, Node->op_begin(),
- Node->getNumOperands()-2);
- }
- }
+ case ISD::BUILD_VECTOR:
+ Result = Node->getOperand(0);
break;
- case ISD::VINSERT_VECTOR_ELT:
- if (!MVT::isVector(NewVT)) {
- // Returning a scalar? Must be the inserted element.
- Result = Node->getOperand(1);
- } else {
- Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT,
- PackVectorOp(Node->getOperand(0), NewVT),
- Node->getOperand(1), Node->getOperand(2));
- }
+ case ISD::INSERT_VECTOR_ELT:
+ // Returning the inserted scalar element.
+ Result = Node->getOperand(1);
+ break;
+ case ISD::CONCAT_VECTORS:
+ assert(Node->getOperand(0).getValueType() == NewVT &&
+ "Concat of non-legal vectors not yet supported!");
+ Result = Node->getOperand(0);
+ break;
+ case ISD::VECTOR_SHUFFLE: {
+ // Figure out if the scalar is the LHS or RHS and return it.
+ SDOperand EltNum = Node->getOperand(2).getOperand(0);
+ if (cast<ConstantSDNode>(EltNum)->getValue())
+ Result = ScalarizeVectorOp(Node->getOperand(1));
+ else
+ Result = ScalarizeVectorOp(Node->getOperand(0));
break;
- case ISD::VVECTOR_SHUFFLE:
- if (!MVT::isVector(NewVT)) {
- // Returning a scalar? Figure out if it is the LHS or RHS and return it.
- SDOperand EltNum = Node->getOperand(2).getOperand(0);
- if (cast<ConstantSDNode>(EltNum)->getValue())
- Result = PackVectorOp(Node->getOperand(1), NewVT);
- else
- Result = PackVectorOp(Node->getOperand(0), NewVT);
- } else {
- // Otherwise, return a VECTOR_SHUFFLE node. First convert the index
- // vector from a VBUILD_VECTOR to a BUILD_VECTOR.
- std::vector<SDOperand> BuildVecIdx(Node->getOperand(2).Val->op_begin(),
- Node->getOperand(2).Val->op_end()-2);
- MVT::ValueType BVT = MVT::getIntVectorWithNumElements(BuildVecIdx.size());
- SDOperand BV = DAG.getNode(ISD::BUILD_VECTOR, BVT,
- Node->getOperand(2).Val->op_begin(),
- Node->getOperand(2).Val->getNumOperands()-2);
-
- Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NewVT,
- PackVectorOp(Node->getOperand(0), NewVT),
- PackVectorOp(Node->getOperand(1), NewVT), BV);
- }
+ }
+ case ISD::EXTRACT_SUBVECTOR:
+ Result = Node->getOperand(0);
+ assert(Result.getValueType() == NewVT);
break;
- case ISD::VBIT_CONVERT:
- if (Op.getOperand(0).getValueType() != MVT::Vector)
- Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0));
- else {
- // If the input is a vector type, we have to either scalarize it, pack it
- // or convert it based on whether the input vector type is legal.
- SDNode *InVal = Node->getOperand(0).Val;
- unsigned NumElems =
- cast<ConstantSDNode>(*(InVal->op_end()-2))->getValue();
- MVT::ValueType EVT = cast<VTSDNode>(*(InVal->op_end()-1))->getVT();
-
- // Figure out if there is a Packed type corresponding to this Vector
- // type. If so, convert to the vector type.
- MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
- if (TVT != MVT::Other && TLI.isTypeLegal(TVT)) {
- // Turn this into a bit convert of the packed input.
- Result = DAG.getNode(ISD::BIT_CONVERT, NewVT,
- PackVectorOp(Node->getOperand(0), TVT));
- break;
- } else if (NumElems == 1) {
- // Turn this into a bit convert of the scalar input.
- Result = DAG.getNode(ISD::BIT_CONVERT, NewVT,
- PackVectorOp(Node->getOperand(0), EVT));
- break;
- } else {
- // FIXME: UNIMP!
- assert(0 && "Cast from unsupported vector type not implemented yet!");
- }
- }
+ case ISD::BIT_CONVERT:
+ Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0));
break;
- case ISD::VSELECT:
+ case ISD::SELECT:
Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0),
- PackVectorOp(Op.getOperand(1), NewVT),
- PackVectorOp(Op.getOperand(2), NewVT));
+ ScalarizeVectorOp(Op.getOperand(1)),
+ ScalarizeVectorOp(Op.getOperand(2)));
break;
}
if (TLI.isTypeLegal(NewVT))
Result = LegalizeOp(Result);
- bool isNew = PackedNodes.insert(std::make_pair(Op, Result)).second;
- assert(isNew && "Value already packed?");
+ bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second;
+ assert(isNew && "Value already scalarized?");
return Result;
}
projects / oota-llvm.git / blobdiff