+SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) {
+ std::map<SDValue, SDValue>::iterator I = WidenNodes.find(Op);
+ if (I != WidenNodes.end()) return I->second;
+
+ MVT VT = Op.getValueType();
+ assert(VT.isVector() && "Cannot widen non-vector type!");
+
+ SDValue Result;
+ SDNode *Node = Op.getNode();
+ MVT EVT = VT.getVectorElementType();
+
+ unsigned NumElts = VT.getVectorNumElements();
+ unsigned NewNumElts = WidenVT.getVectorNumElements();
+ assert(NewNumElts > NumElts && "Cannot widen to smaller type!");
+ assert(NewNumElts < 17);
+
+ // When widen is called, it is assumed that it is more efficient to use a
+ // wide type. The default action is to widen to operation to a wider legal
+ // vector type and then do the operation if it is legal by calling LegalizeOp
+ // again. If there is no vector equivalent, we will unroll the operation, do
+ // it, and rebuild the vector. If most of the operations are vectorizible to
+ // the legal type, the resulting code will be more efficient. If this is not
+ // the case, the resulting code will preform badly as we end up generating
+ // code to pack/unpack the results. It is the function that calls widen
+ // that is responsible for seeing this doesn't happen.
+ switch (Node->getOpcode()) {
+ default:
+#ifndef NDEBUG
+ Node->dump(&DAG);
+#endif
+ assert(0 && "Unexpected operation in WidenVectorOp!");
+ break;
+ case ISD::CopyFromReg:
+ assert(0 && "CopyFromReg doesn't need widening!");
+ case ISD::Constant:
+ case ISD::ConstantFP:
+ // To build a vector of these elements, clients should call BuildVector
+ // and with each element instead of creating a node with a vector type
+ assert(0 && "Unexpected operation in WidenVectorOp!");
+ case ISD::VAARG:
+ // Variable Arguments with vector types doesn't make any sense to me
+ assert(0 && "Unexpected operation in WidenVectorOp!");
+ break;
+ case ISD::UNDEF:
+ Result = DAG.getNode(ISD::UNDEF, WidenVT);
+ break;
+ case ISD::BUILD_VECTOR: {
+ // Build a vector with undefined for the new nodes
+ SDValueVector NewOps(Node->op_begin(), Node->op_end());
+ for (unsigned i = NumElts; i < NewNumElts; ++i) {
+ NewOps.push_back(DAG.getNode(ISD::UNDEF,EVT));
+ }
+ Result = DAG.getNode(ISD::BUILD_VECTOR, WidenVT, &NewOps[0], NewOps.size());
+ break;
+ }
+ case ISD::INSERT_VECTOR_ELT: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, WidenVT, Tmp1,
+ Node->getOperand(1), Node->getOperand(2));
+ break;
+ }
+ case ISD::VECTOR_SHUFFLE: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT);
+ // VECTOR_SHUFFLE 3rd operand must be a constant build vector that is
+ // used as permutation array. We build the vector here instead of widening
+ // because we don't want to legalize and have it turned to something else.
+ SDValue PermOp = Node->getOperand(2);
+ SDValueVector NewOps;
+ MVT PVT = PermOp.getValueType().getVectorElementType();
+ for (unsigned i = 0; i < NumElts; ++i) {
+ if (PermOp.getOperand(i).getOpcode() == ISD::UNDEF) {
+ NewOps.push_back(PermOp.getOperand(i));
+ } else {
+ unsigned Idx =
+ cast<ConstantSDNode>(PermOp.getOperand(i))->getZExtValue();
+ if (Idx < NumElts) {
+ NewOps.push_back(PermOp.getOperand(i));
+ }
+ else {
+ NewOps.push_back(DAG.getConstant(Idx + NewNumElts - NumElts,
+ PermOp.getOperand(i).getValueType()));
+ }
+ }
+ }
+ for (unsigned i = NumElts; i < NewNumElts; ++i) {
+ NewOps.push_back(DAG.getNode(ISD::UNDEF,PVT));
+ }
+
+ SDValue Tmp3 = DAG.getNode(ISD::BUILD_VECTOR,
+ MVT::getVectorVT(PVT, NewOps.size()),
+ &NewOps[0], NewOps.size());
+
+ Result = DAG.getNode(ISD::VECTOR_SHUFFLE, WidenVT, Tmp1, Tmp2, Tmp3);
+ break;
+ }
+ case ISD::LOAD: {
+ // If the load widen returns true, we can use a single load for the
+ // vector. Otherwise, it is returning a token factor for multiple
+ // loads.
+ SDValue TFOp;
+ if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT))
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1)));
+ else
+ AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0)));
+ break;
+ }
+
+ case ISD::BIT_CONVERT: {
+ SDValue Tmp1 = Node->getOperand(0);
+ // Converts between two different types so we need to determine
+ // the correct widen type for the input operand.
+ MVT TVT = Tmp1.getValueType();
+ assert(TVT.isVector() && "can not widen non vector type");
+ MVT TEVT = TVT.getVectorElementType();
+ assert(WidenVT.getSizeInBits() % EVT.getSizeInBits() == 0 &&
+ "can not widen bit bit convert that are not multiple of element type");
+ MVT TWidenVT = MVT::getVectorVT(TEVT,
+ WidenVT.getSizeInBits()/EVT.getSizeInBits());
+ Tmp1 = WidenVectorOp(Tmp1, TWidenVT);
+ assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits());
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
+
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+
+ case ISD::SINT_TO_FP:
+ case ISD::UINT_TO_FP:
+ case ISD::FP_TO_SINT:
+ case ISD::FP_TO_UINT: {
+ SDValue Tmp1 = Node->getOperand(0);
+ // Converts between two different types so we need to determine
+ // the correct widen type for the input operand.
+ MVT TVT = Tmp1.getValueType();
+ assert(TVT.isVector() && "can not widen non vector type");
+ MVT TEVT = TVT.getVectorElementType();
+ MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts);
+ Tmp1 = WidenVectorOp(Tmp1, TWidenVT);
+ assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
+
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+
+ case ISD::FP_EXTEND:
+ assert(0 && "Case not implemented. Dynamically dead with 2 FP types!");
+ case ISD::TRUNCATE:
+ case ISD::SIGN_EXTEND:
+ case ISD::ZERO_EXTEND:
+ case ISD::ANY_EXTEND:
+ case ISD::FP_ROUND:
+ case ISD::SIGN_EXTEND_INREG:
+ case ISD::FABS:
+ case ISD::FNEG:
+ case ISD::FSQRT:
+ case ISD::FSIN:
+ case ISD::FCOS:
+ case ISD::CTPOP:
+ case ISD::CTTZ:
+ case ISD::CTLZ: {
+ // Unary op widening
+ SDValue Tmp1;
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+
+ Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1);
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+ case ISD::CONVERT_RNDSAT: {
+ SDValue RndOp = Node->getOperand(3);
+ SDValue SatOp = Node->getOperand(4);
+
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+
+ SDValue SrcOp = Node->getOperand(0);
+
+ // Converts between two different types so we need to determine
+ // the correct widen type for the input operand.
+ MVT SVT = SrcOp.getValueType();
+ assert(SVT.isVector() && "can not widen non vector type");
+ MVT SEVT = SVT.getVectorElementType();
+ MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts);
+
+ SrcOp = WidenVectorOp(SrcOp, SWidenVT);
+ assert(SrcOp.getValueType() == WidenVT);
+ SDValue DTyOp = DAG.getValueType(WidenVT);
+ SDValue STyOp = DAG.getValueType(SrcOp.getValueType());
+ ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(Node)->getCvtCode();
+
+ Result = DAG.getConvertRndSat(WidenVT, SrcOp, DTyOp, STyOp,
+ RndOp, SatOp, CvtCode);
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+ case ISD::FPOW:
+ case ISD::FPOWI:
+ case ISD::ADD:
+ case ISD::SUB:
+ case ISD::MUL:
+ case ISD::MULHS:
+ case ISD::MULHU:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ case ISD::FADD:
+ case ISD::FSUB:
+ case ISD::FMUL:
+ case ISD::SDIV:
+ case ISD::SREM:
+ case ISD::FDIV:
+ case ISD::FREM:
+ case ISD::FCOPYSIGN:
+ case ISD::UDIV:
+ case ISD::UREM:
+ case ISD::BSWAP: {
+ // Binary op widening
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2);
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code by first
+ // Widening to the right type and then unroll the beast.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+
+ case ISD::SHL:
+ case ISD::SRA:
+ case ISD::SRL: {
+ // Binary op with one non vector operand
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Node->getOperand(1));
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+ case ISD::EXTRACT_VECTOR_ELT: {
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), EVT, Tmp1, Node->getOperand(1));
+ break;
+ }
+ case ISD::CONCAT_VECTORS: {
+ // We concurrently support only widen on a multiple of the incoming vector.
+ // We could widen on a multiple of the incoming operand if necessary.
+ unsigned NumConcat = NewNumElts / NumElts;
+ assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector");
+ std::vector<SDValue> UnOps(NumElts, DAG.getNode(ISD::UNDEF,
+ VT.getVectorElementType()));
+ SDValue UndefVal = DAG.getNode(ISD::BUILD_VECTOR, VT,
+ &UnOps[0], UnOps.size());
+ SmallVector<SDValue, 8> MOps;
+ MOps.push_back(Op);
+ for (unsigned i = 1; i != NumConcat; ++i) {
+ MOps.push_back(UndefVal);
+ }
+ Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT,
+ &MOps[0], MOps.size()));
+ break;
+ }
+ case ISD::EXTRACT_SUBVECTOR: {
+ SDValue Tmp1 = Node->getOperand(0);
+ SDValue Idx = Node->getOperand(1);
+ ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx);
+ if (CIdx && CIdx->getZExtValue() == 0) {
+ // Since we are access the start of the vector, the incoming
+ // vector type might be the proper.
+ MVT Tmp1VT = Tmp1.getValueType();
+ if (Tmp1VT == WidenVT)
+ return Tmp1;
+ else {
+ unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements();
+ if (Tmp1VTNumElts < NewNumElts)
+ Result = WidenVectorOp(Tmp1, WidenVT);
+ else
+ Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, WidenVT, Tmp1, Idx);
+ }
+ } else if (NewNumElts % NumElts == 0) {
+ // Widen the extracted subvector.
+ unsigned NumConcat = NewNumElts / NumElts;
+ SDValue UndefVal = DAG.getNode(ISD::UNDEF, VT);
+ SmallVector<SDValue, 8> MOps;
+ MOps.push_back(Op);
+ for (unsigned i = 1; i != NumConcat; ++i) {
+ MOps.push_back(UndefVal);
+ }
+ Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, WidenVT,
+ &MOps[0], MOps.size()));
+ } else {
+ assert(0 && "can not widen extract subvector");
+ // This could be implemented using insert and build vector but I would
+ // like to see when this happens.
+ }
+ break;
+ }
+
+ case ISD::SELECT: {
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+
+ // Determine new condition widen type and widen
+ SDValue Cond1 = Node->getOperand(0);
+ MVT CondVT = Cond1.getValueType();
+ assert(CondVT.isVector() && "can not widen non vector type");
+ MVT CondEVT = CondVT.getVectorElementType();
+ MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts);
+ Cond1 = WidenVectorOp(Cond1, CondWidenVT);
+ assert(Cond1.getValueType() == CondWidenVT && "Condition not widen");
+
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Tmp1, Tmp2);
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code by first
+ // Widening to the right type and then unroll the beast.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+
+ case ISD::SELECT_CC: {
+ TargetLowering::LegalizeAction action =
+ TLI.getOperationAction(Node->getOpcode(), WidenVT);
+
+ // Determine new condition widen type and widen
+ SDValue Cond1 = Node->getOperand(0);
+ SDValue Cond2 = Node->getOperand(1);
+ MVT CondVT = Cond1.getValueType();
+ assert(CondVT.isVector() && "can not widen non vector type");
+ assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs");
+ MVT CondEVT = CondVT.getVectorElementType();
+ MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts);
+ Cond1 = WidenVectorOp(Cond1, CondWidenVT);
+ Cond2 = WidenVectorOp(Cond2, CondWidenVT);
+ assert(Cond1.getValueType() == CondWidenVT &&
+ Cond2.getValueType() == CondWidenVT && "condition not widen");
+
+ SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT);
+ assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT &&
+ "operands not widen");
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Cond1, Cond2, Tmp1,
+ Tmp2, Node->getOperand(4));
+ switch (action) {
+ default: assert(0 && "action not supported");
+ case TargetLowering::Legal:
+ break;
+ case TargetLowering::Promote:
+ // We defer the promotion to when we legalize the op
+ break;
+ case TargetLowering::Expand:
+ // Expand the operation into a bunch of nasty scalar code by first
+ // Widening to the right type and then unroll the beast.
+ Result = LegalizeOp(UnrollVectorOp(Result));
+ break;
+ }
+ break;
+ }
+ case ISD::VSETCC: {
+ // Determine widen for the operand
+ SDValue Tmp1 = Node->getOperand(0);
+ MVT TmpVT = Tmp1.getValueType();
+ assert(TmpVT.isVector() && "can not widen non vector type");
+ MVT TmpEVT = TmpVT.getVectorElementType();
+ MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts);
+ Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT);
+ SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT);
+ Result = DAG.getNode(Node->getOpcode(), WidenVT, Tmp1, Tmp2,
+ Node->getOperand(2));
+ break;
+ }
+ case ISD::ATOMIC_CMP_SWAP_8:
+ case ISD::ATOMIC_CMP_SWAP_16:
+ case ISD::ATOMIC_CMP_SWAP_32:
+ case ISD::ATOMIC_CMP_SWAP_64:
+ case ISD::ATOMIC_LOAD_ADD_8:
+ case ISD::ATOMIC_LOAD_SUB_8:
+ case ISD::ATOMIC_LOAD_AND_8:
+ case ISD::ATOMIC_LOAD_OR_8:
+ case ISD::ATOMIC_LOAD_XOR_8:
+ case ISD::ATOMIC_LOAD_NAND_8:
+ case ISD::ATOMIC_LOAD_MIN_8:
+ case ISD::ATOMIC_LOAD_MAX_8:
+ case ISD::ATOMIC_LOAD_UMIN_8:
+ case ISD::ATOMIC_LOAD_UMAX_8:
+ case ISD::ATOMIC_SWAP_8:
+ case ISD::ATOMIC_LOAD_ADD_16:
+ case ISD::ATOMIC_LOAD_SUB_16:
+ case ISD::ATOMIC_LOAD_AND_16:
+ case ISD::ATOMIC_LOAD_OR_16:
+ case ISD::ATOMIC_LOAD_XOR_16:
+ case ISD::ATOMIC_LOAD_NAND_16:
+ case ISD::ATOMIC_LOAD_MIN_16:
+ case ISD::ATOMIC_LOAD_MAX_16:
+ case ISD::ATOMIC_LOAD_UMIN_16:
+ case ISD::ATOMIC_LOAD_UMAX_16:
+ case ISD::ATOMIC_SWAP_16:
+ case ISD::ATOMIC_LOAD_ADD_32:
+ case ISD::ATOMIC_LOAD_SUB_32:
+ case ISD::ATOMIC_LOAD_AND_32:
+ case ISD::ATOMIC_LOAD_OR_32:
+ case ISD::ATOMIC_LOAD_XOR_32:
+ case ISD::ATOMIC_LOAD_NAND_32:
+ case ISD::ATOMIC_LOAD_MIN_32:
+ case ISD::ATOMIC_LOAD_MAX_32:
+ case ISD::ATOMIC_LOAD_UMIN_32:
+ case ISD::ATOMIC_LOAD_UMAX_32:
+ case ISD::ATOMIC_SWAP_32:
+ case ISD::ATOMIC_LOAD_ADD_64:
+ case ISD::ATOMIC_LOAD_SUB_64:
+ case ISD::ATOMIC_LOAD_AND_64:
+ case ISD::ATOMIC_LOAD_OR_64:
+ case ISD::ATOMIC_LOAD_XOR_64:
+ case ISD::ATOMIC_LOAD_NAND_64:
+ case ISD::ATOMIC_LOAD_MIN_64:
+ case ISD::ATOMIC_LOAD_MAX_64:
+ case ISD::ATOMIC_LOAD_UMIN_64:
+ case ISD::ATOMIC_LOAD_UMAX_64:
+ case ISD::ATOMIC_SWAP_64: {
+ // For now, we assume that using vectors for these operations don't make
+ // much sense so we just split it. We return an empty result
+ SDValue X, Y;
+ SplitVectorOp(Op, X, Y);
+ return Result;
+ break;
+ }
+
+ } // end switch (Node->getOpcode())
+
+ assert(Result.getNode() && "Didn't set a result!");
+ if (Result != Op)
+ Result = LegalizeOp(Result);
+
+ AddWidenedOperand(Op, Result);
+ return Result;
+}
+
+// Utility function to find a legal vector type and its associated element
+// type from a preferred width and whose vector type must be the same size
+// as the VVT.
+// TLI: Target lowering used to determine legal types
+// Width: Preferred width of element type
+// VVT: Vector value type whose size we must match.
+// Returns VecEVT and EVT - the vector type and its associated element type
+static void FindWidenVecType(TargetLowering &TLI, unsigned Width, MVT VVT,
+ MVT& EVT, MVT& VecEVT) {
+ // We start with the preferred width, make it a power of 2 and see if
+ // we can find a vector type of that width. If not, we reduce it by
+ // another power of 2. If we have widen the type, a vector of bytes should
+ // always be legal.
+ assert(TLI.isTypeLegal(VVT));
+ unsigned EWidth = Width + 1;
+ do {
+ assert(EWidth > 0);
+ EWidth = (1 << Log2_32(EWidth-1));
+ EVT = MVT::getIntegerVT(EWidth);
+ unsigned NumEVT = VVT.getSizeInBits()/EWidth;
+ VecEVT = MVT::getVectorVT(EVT, NumEVT);
+ } while (!TLI.isTypeLegal(VecEVT) ||
+ VVT.getSizeInBits() != VecEVT.getSizeInBits());
+}
+
+SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain,
+ SDValue Chain,
+ SDValue BasePtr,
+ const Value *SV,
+ int SVOffset,
+ unsigned Alignment,
+ bool isVolatile,
+ unsigned LdWidth,
+ MVT ResType) {
+ // We assume that we have good rules to handle loading power of two loads so
+ // we break down the operations to power of 2 loads. The strategy is to
+ // load the largest power of 2 that we can easily transform to a legal vector
+ // and then insert into that vector, and the cast the result into the legal
+ // vector that we want. This avoids unnecessary stack converts.
+ // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and
+ // the load is nonvolatile, we an use a wider load for the value.
+ // Find a vector length we can load a large chunk
+ MVT EVT, VecEVT;
+ unsigned EVTWidth;
+ FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+
+ SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV, SVOffset,
+ isVolatile, Alignment);
+ SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecEVT, LdOp);
+ LdChain.push_back(LdOp.getValue(1));
+
+ // Check if we can load the element with one instruction
+ if (LdWidth == EVTWidth) {
+ return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp);
+ }
+
+ // The vector element order is endianness dependent.
+ unsigned Idx = 1;
+ LdWidth -= EVTWidth;
+ unsigned Offset = 0;
+
+ while (LdWidth > 0) {
+ unsigned Increment = EVTWidth / 8;
+ Offset += Increment;
+ BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr,
+ DAG.getIntPtrConstant(Increment));
+
+ if (LdWidth < EVTWidth) {
+ // Our current type we are using is too large, use a smaller size by
+ // using a smaller power of 2
+ unsigned oEVTWidth = EVTWidth;
+ FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+ // Readjust position and vector position based on new load type
+ Idx = Idx * (oEVTWidth/EVTWidth);
+ VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp);
+ }
+
+ SDValue LdOp = DAG.getLoad(EVT, Chain, BasePtr, SV,
+ SVOffset+Offset, isVolatile,
+ MinAlign(Alignment, Offset));
+ LdChain.push_back(LdOp.getValue(1));
+ VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, VecEVT, VecOp, LdOp,
+ DAG.getIntPtrConstant(Idx++));
+
+ LdWidth -= EVTWidth;
+ }
+
+ return DAG.getNode(ISD::BIT_CONVERT, ResType, VecOp);
+}
+
+bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result,
+ SDValue& TFOp,
+ SDValue Op,
+ MVT NVT) {
+ // TODO: Add support for ConcatVec and the ability to load many vector
+ // types (e.g., v4i8). This will not work when a vector register
+ // to memory mapping is strange (e.g., vector elements are not
+ // stored in some sequential order).
+
+ // It must be true that the widen vector type is bigger than where
+ // we need to load from.
+ LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
+ MVT LdVT = LD->getMemoryVT();
+ assert(LdVT.isVector() && NVT.isVector());
+ assert(LdVT.getVectorElementType() == NVT.getVectorElementType());
+
+ // Load information
+ SDValue Chain = LD->getChain();
+ SDValue BasePtr = LD->getBasePtr();
+ int SVOffset = LD->getSrcValueOffset();
+ unsigned Alignment = LD->getAlignment();
+ bool isVolatile = LD->isVolatile();
+ const Value *SV = LD->getSrcValue();
+ unsigned int LdWidth = LdVT.getSizeInBits();
+
+ // Load value as a large register
+ SDValueVector LdChain;
+ Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset,
+ Alignment, isVolatile, LdWidth, NVT);
+
+ if (LdChain.size() == 1) {
+ TFOp = LdChain[0];
+ return true;
+ }
+ else {
+ TFOp=DAG.getNode(ISD::TokenFactor, MVT::Other, &LdChain[0], LdChain.size());
+ return false;
+ }
+}
+
+
+void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain,
+ SDValue Chain,
+ SDValue BasePtr,
+ const Value *SV,
+ int SVOffset,
+ unsigned Alignment,
+ bool isVolatile,
+ SDValue ValOp,
+ unsigned StWidth) {
+ // Breaks the stores into a series of power of 2 width stores. For any
+ // width, we convert the vector to the vector of element size that we
+ // want to store. This avoids requiring a stack convert.
+
+ // Find a width of the element type we can store with
+ MVT VVT = ValOp.getValueType();
+ MVT EVT, VecEVT;
+ unsigned EVTWidth;
+ FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+
+ SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, ValOp);
+ SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp,
+ DAG.getIntPtrConstant(0));
+ SDValue StOp = DAG.getStore(Chain, EOp, BasePtr, SV, SVOffset,
+ isVolatile, Alignment);
+ StChain.push_back(StOp);
+
+ // Check if we are done
+ if (StWidth == EVTWidth) {
+ return;
+ }
+
+ unsigned Idx = 1;
+ StWidth -= EVTWidth;
+ unsigned Offset = 0;
+
+ while (StWidth > 0) {
+ unsigned Increment = EVTWidth / 8;
+ Offset += Increment;
+ BasePtr = DAG.getNode(ISD::ADD, BasePtr.getValueType(), BasePtr,
+ DAG.getIntPtrConstant(Increment));
+
+ if (StWidth < EVTWidth) {
+ // Our current type we are using is too large, use a smaller size by
+ // using a smaller power of 2
+ unsigned oEVTWidth = EVTWidth;
+ FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT);
+ EVTWidth = EVT.getSizeInBits();
+ // Readjust position and vector position based on new load type
+ Idx = Idx * (oEVTWidth/EVTWidth);
+ VecOp = DAG.getNode(ISD::BIT_CONVERT, VecEVT, VecOp);
+ }
+
+ EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EVT, VecOp,
+ DAG.getIntPtrConstant(Idx++));
+ StChain.push_back(DAG.getStore(Chain, EOp, BasePtr, SV,
+ SVOffset + Offset, isVolatile,
+ MinAlign(Alignment, Offset)));
+ StWidth -= EVTWidth;
+ }
+}
+
+
+SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST,
+ SDValue Chain,
+ SDValue BasePtr) {
+ // TODO: It might be cleaner if we can use SplitVector and have more legal
+ // vector types that can be stored into memory (e.g., v4xi8 can
+ // be stored as a word). This will not work when a vector register
+ // to memory mapping is strange (e.g., vector elements are not
+ // stored in some sequential order).
+
+ MVT StVT = ST->getMemoryVT();
+ SDValue ValOp = ST->getValue();
+
+ // Check if we have widen this node with another value
+ std::map<SDValue, SDValue>::iterator I = WidenNodes.find(ValOp);
+ if (I != WidenNodes.end())
+ ValOp = I->second;
+
+ MVT VVT = ValOp.getValueType();
+
+ // It must be true that we the widen vector type is bigger than where
+ // we need to store.
+ assert(StVT.isVector() && VVT.isVector());
+ assert(StVT.getSizeInBits() < VVT.getSizeInBits());
+ assert(StVT.getVectorElementType() == VVT.getVectorElementType());
+
+ // Store value
+ SDValueVector StChain;
+ genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(),
+ ST->getSrcValueOffset(), ST->getAlignment(),
+ ST->isVolatile(), ValOp, StVT.getSizeInBits());
+ if (StChain.size() == 1)
+ return StChain[0];
+ else
+ return DAG.getNode(ISD::TokenFactor, MVT::Other,&StChain[0],StChain.size());
+}
+
+