1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file performs vector type splitting and scalarization for LegalizeTypes.
11 // Scalarization is the act of changing a computation in an illegal one-element
12 // vector type to be a computation in its scalar element type. For example,
13 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
14 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
15 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
17 // Splitting is the act of changing a computation in an invalid vector type to
18 // be a computation in multiple vectors of a smaller type. For example,
19 // implementing <128 x f32> operations in terms of two <64 x f32> operations.
21 //===----------------------------------------------------------------------===//
23 #include "LegalizeTypes.h"
24 #include "llvm/CodeGen/PseudoSourceValue.h"
25 #include "llvm/Target/TargetData.h"
28 //===----------------------------------------------------------------------===//
29 // Result Vector Scalarization: <1 x ty> -> ty.
30 //===----------------------------------------------------------------------===//
32 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
33 DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
35 SDValue R = SDValue();
37 switch (N->getOpcode()) {
40 cerr << "ScalarizeVectorResult #" << ResNo << ": ";
41 N->dump(&DAG); cerr << "\n";
43 assert(0 && "Do not know how to scalarize the result of this operator!");
46 case ISD::BIT_CONVERT: R = ScalarizeVecRes_BIT_CONVERT(N); break;
47 case ISD::BUILD_VECTOR: R = N->getOperand(0); break;
48 case ISD::CONVERT_RNDSAT: R = ScalarizeVecRes_CONVERT_RNDSAT(N); break;
49 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
50 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
51 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
52 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
53 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
54 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
55 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
56 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
57 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
58 case ISD::VSETCC: R = ScalarizeVecRes_VSETCC(N); break;
77 case ISD::UINT_TO_FP: R = ScalarizeVecRes_UnaryOp(N); break;
94 case ISD::XOR: R = ScalarizeVecRes_BinOp(N); break;
98 case ISD::SRL: R = ScalarizeVecRes_ShiftOp(N); break;
101 // If R is null, the sub-method took care of registering the result.
103 SetScalarizedVector(SDValue(N, ResNo), R);
106 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
107 SDValue LHS = GetScalarizedVector(N->getOperand(0));
108 SDValue RHS = GetScalarizedVector(N->getOperand(1));
109 return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
110 LHS.getValueType(), LHS, RHS);
113 SDValue DAGTypeLegalizer::ScalarizeVecRes_ShiftOp(SDNode *N) {
114 SDValue LHS = GetScalarizedVector(N->getOperand(0));
115 SDValue ShiftAmt = GetScalarizedVector(N->getOperand(1));
116 return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
117 LHS.getValueType(), LHS, ShiftAmt);
120 SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) {
121 MVT NewVT = N->getValueType(0).getVectorElementType();
122 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
123 NewVT, N->getOperand(0));
126 SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) {
127 MVT NewVT = N->getValueType(0).getVectorElementType();
128 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
129 return DAG.getConvertRndSat(NewVT, N->getDebugLoc(),
130 Op0, DAG.getValueType(NewVT),
131 DAG.getValueType(Op0.getValueType()),
134 cast<CvtRndSatSDNode>(N)->getCvtCode());
137 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
138 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
139 N->getValueType(0).getVectorElementType(),
140 N->getOperand(0), N->getOperand(1));
143 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
144 SDValue Op = GetScalarizedVector(N->getOperand(0));
145 return DAG.getNode(ISD::FPOWI, N->getDebugLoc(),
146 Op.getValueType(), Op, N->getOperand(1));
149 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
150 // The value to insert may have a wider type than the vector element type,
151 // so be sure to truncate it to the element type if necessary.
152 SDValue Op = N->getOperand(1);
153 MVT EltVT = N->getValueType(0).getVectorElementType();
154 if (Op.getValueType() != EltVT)
155 // FIXME: Can this happen for floating point types?
156 Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op);
160 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
161 assert(N->isUnindexed() && "Indexed vector load?");
163 SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(),
164 N->getExtensionType(),
165 N->getValueType(0).getVectorElementType(),
166 N->getChain(), N->getBasePtr(),
167 DAG.getUNDEF(N->getBasePtr().getValueType()),
168 N->getSrcValue(), N->getSrcValueOffset(),
169 N->getMemoryVT().getVectorElementType(),
170 N->isVolatile(), N->getAlignment());
172 // Legalized the chain result - switch anything that used the old chain to
174 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
178 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
179 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
180 MVT DestVT = N->getValueType(0).getVectorElementType();
181 SDValue Op = GetScalarizedVector(N->getOperand(0));
182 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op);
185 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
186 // If the operand is wider than the vector element type then it is implicitly
187 // truncated. Make that explicit here.
188 MVT EltVT = N->getValueType(0).getVectorElementType();
189 SDValue InOp = N->getOperand(0);
190 if (InOp.getValueType() != EltVT)
191 return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp);
195 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
196 SDValue LHS = GetScalarizedVector(N->getOperand(1));
197 return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
198 LHS.getValueType(), N->getOperand(0), LHS,
199 GetScalarizedVector(N->getOperand(2)));
202 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
203 SDValue LHS = GetScalarizedVector(N->getOperand(2));
204 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(),
205 N->getOperand(0), N->getOperand(1),
206 LHS, GetScalarizedVector(N->getOperand(3)),
210 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
211 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
214 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
215 // Figure out if the scalar is the LHS or RHS and return it.
216 SDValue Arg = N->getOperand(2).getOperand(0);
217 if (Arg.getOpcode() == ISD::UNDEF)
218 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
219 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
220 return GetScalarizedVector(N->getOperand(Op));
223 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
224 SDValue LHS = GetScalarizedVector(N->getOperand(0));
225 SDValue RHS = GetScalarizedVector(N->getOperand(1));
226 MVT NVT = N->getValueType(0).getVectorElementType();
227 MVT SVT = TLI.getSetCCResultType(LHS.getValueType());
228 DebugLoc dl = N->getDebugLoc();
230 // Turn it into a scalar SETCC.
231 SDValue Res = DAG.getNode(ISD::SETCC, dl, SVT, LHS, RHS, N->getOperand(2));
233 // VSETCC always returns a sign-extended value, while SETCC may not. The
234 // SETCC result type may not match the vector element type. Correct these.
235 if (NVT.bitsLE(SVT)) {
236 // The SETCC result type is bigger than the vector element type.
237 // Ensure the SETCC result is sign-extended.
238 if (TLI.getBooleanContents() !=
239 TargetLowering::ZeroOrNegativeOneBooleanContent)
240 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, SVT, Res,
241 DAG.getValueType(MVT::i1));
242 // Truncate to the final type.
243 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
245 // The SETCC result type is smaller than the vector element type.
246 // If the SetCC result is not sign-extended, chop it down to MVT::i1.
247 if (TLI.getBooleanContents() !=
248 TargetLowering::ZeroOrNegativeOneBooleanContent)
249 Res = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Res);
250 // Sign extend to the final type.
251 return DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Res);
256 //===----------------------------------------------------------------------===//
257 // Operand Vector Scalarization <1 x ty> -> ty.
258 //===----------------------------------------------------------------------===//
260 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
261 DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
263 SDValue Res = SDValue();
265 if (Res.getNode() == 0) {
266 switch (N->getOpcode()) {
269 cerr << "ScalarizeVectorOperand Op #" << OpNo << ": ";
270 N->dump(&DAG); cerr << "\n";
272 assert(0 && "Do not know how to scalarize this operator's operand!");
275 case ISD::BIT_CONVERT:
276 Res = ScalarizeVecOp_BIT_CONVERT(N); break;
278 case ISD::CONCAT_VECTORS:
279 Res = ScalarizeVecOp_CONCAT_VECTORS(N); break;
281 case ISD::EXTRACT_VECTOR_ELT:
282 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N); break;
285 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo); break;
289 // If the result is null, the sub-method took care of registering results etc.
290 if (!Res.getNode()) return false;
292 // If the result is N, the sub-method updated N in place. Tell the legalizer
294 if (Res.getNode() == N)
297 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
298 "Invalid operand expansion");
300 ReplaceValueWith(SDValue(N, 0), Res);
304 /// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs
305 /// to be scalarized, it must be <1 x ty>. Convert the element instead.
306 SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) {
307 SDValue Elt = GetScalarizedVector(N->getOperand(0));
308 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
309 N->getValueType(0), Elt);
312 /// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
313 /// use a BUILD_VECTOR instead.
314 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
315 SmallVector<SDValue, 8> Ops(N->getNumOperands());
316 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
317 Ops[i] = GetScalarizedVector(N->getOperand(i));
318 return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0),
319 &Ops[0], Ops.size());
322 /// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
323 /// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
325 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
326 return GetScalarizedVector(N->getOperand(0));
329 /// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
330 /// scalarized, it must be <1 x ty>. Just store the element.
331 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
332 assert(N->isUnindexed() && "Indexed store of one-element vector?");
333 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
334 DebugLoc dl = N->getDebugLoc();
336 if (N->isTruncatingStore())
337 return DAG.getTruncStore(N->getChain(), dl,
338 GetScalarizedVector(N->getOperand(1)),
340 N->getSrcValue(), N->getSrcValueOffset(),
341 N->getMemoryVT().getVectorElementType(),
342 N->isVolatile(), N->getAlignment());
344 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
345 N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(),
346 N->isVolatile(), N->getAlignment());
350 //===----------------------------------------------------------------------===//
351 // Result Vector Splitting
352 //===----------------------------------------------------------------------===//
354 /// SplitVectorResult - This method is called when the specified result of the
355 /// specified node is found to need vector splitting. At this point, the node
356 /// may also have invalid operands or may have other results that need
357 /// legalization, we just know that (at least) one result needs vector
359 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
360 DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n");
363 switch (N->getOpcode()) {
366 cerr << "SplitVectorResult #" << ResNo << ": ";
367 N->dump(&DAG); cerr << "\n";
369 assert(0 && "Do not know how to split the result of this operator!");
372 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break;
373 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
374 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
375 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
377 case ISD::BIT_CONVERT: SplitVecRes_BIT_CONVERT(N, Lo, Hi); break;
378 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
379 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
380 case ISD::CONVERT_RNDSAT: SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break;
381 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
382 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
383 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
384 case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
385 case ISD::LOAD: SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);break;
386 case ISD::VECTOR_SHUFFLE:
387 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi); break;
388 case ISD::VSETCC: SplitVecRes_VSETCC(N, Lo, Hi); break;
402 case ISD::FNEARBYINT:
403 case ISD::FP_TO_SINT:
404 case ISD::FP_TO_UINT:
405 case ISD::SINT_TO_FP:
407 case ISD::UINT_TO_FP: SplitVecRes_UnaryOp(N, Lo, Hi); break;
427 case ISD::FREM: SplitVecRes_BinOp(N, Lo, Hi); break;
430 // If Lo/Hi is null, the sub-method took care of registering results etc.
432 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
435 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
437 SDValue LHSLo, LHSHi;
438 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
439 SDValue RHSLo, RHSHi;
440 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
441 DebugLoc dl = N->getDebugLoc();
443 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo);
444 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi);
447 void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
449 // We know the result is a vector. The input may be either a vector or a
452 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
453 DebugLoc dl = N->getDebugLoc();
455 SDValue InOp = N->getOperand(0);
456 MVT InVT = InOp.getValueType();
458 // Handle some special cases efficiently.
459 switch (getTypeAction(InVT)) {
461 assert(false && "Unknown type action!");
465 case ScalarizeVector:
469 // A scalar to vector conversion, where the scalar needs expansion.
470 // If the vector is being split in two then we can just convert the
473 GetExpandedOp(InOp, Lo, Hi);
474 if (TLI.isBigEndian())
476 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
477 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
482 // If the input is a vector that needs to be split, convert each split
483 // piece of the input now.
484 GetSplitVector(InOp, Lo, Hi);
485 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
486 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
490 // In the general case, convert the input to an integer and split it by hand.
491 MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits());
492 MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits());
493 if (TLI.isBigEndian())
494 std::swap(LoIntVT, HiIntVT);
496 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
498 if (TLI.isBigEndian())
500 Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
501 Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
504 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
507 DebugLoc dl = N->getDebugLoc();
508 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
509 unsigned LoNumElts = LoVT.getVectorNumElements();
510 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
511 Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size());
513 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
514 Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size());
517 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
519 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
520 DebugLoc dl = N->getDebugLoc();
521 unsigned NumSubvectors = N->getNumOperands() / 2;
522 if (NumSubvectors == 1) {
523 Lo = N->getOperand(0);
524 Hi = N->getOperand(1);
529 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
531 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
532 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size());
534 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
535 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size());
538 void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo,
541 DebugLoc dl = N->getDebugLoc();
542 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
544 SDValue DTyOpLo = DAG.getValueType(LoVT);
545 SDValue DTyOpHi = DAG.getValueType(HiVT);
547 SDValue RndOp = N->getOperand(3);
548 SDValue SatOp = N->getOperand(4);
549 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
553 MVT InVT = N->getOperand(0).getValueType();
554 switch (getTypeAction(InVT)) {
555 default: assert(0 && "Unexpected type action!");
557 assert(LoVT == HiVT && "Legal non-power-of-two vector type?");
558 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
559 LoVT.getVectorNumElements());
560 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
561 DAG.getIntPtrConstant(0));
562 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
563 DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
567 GetSplitVector(N->getOperand(0), VLo, VHi);
570 // If the result needs to be split and the input needs to be widened,
571 // the two types must have different lengths. Use the widened result
572 // and extract from it to do the split.
573 assert(LoVT == HiVT && "Legal non-power-of-two vector type?");
574 SDValue InOp = GetWidenedVector(N->getOperand(0));
575 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
576 LoVT.getVectorNumElements());
577 VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
578 DAG.getIntPtrConstant(0));
579 VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
580 DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
585 SDValue STyOpLo = DAG.getValueType(VLo.getValueType());
586 SDValue STyOpHi = DAG.getValueType(VHi.getValueType());
588 Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp,
590 Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp,
594 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
596 SDValue Vec = N->getOperand(0);
597 SDValue Idx = N->getOperand(1);
598 MVT IdxVT = Idx.getValueType();
599 DebugLoc dl = N->getDebugLoc();
602 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
603 // The indices are not guaranteed to be a multiple of the new vector
604 // size unless the original vector type was split in two.
605 assert(LoVT == HiVT && "Non power-of-two vectors not supported!");
607 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
608 Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx,
609 DAG.getConstant(LoVT.getVectorNumElements(), IdxVT));
610 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx);
613 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
615 DebugLoc dl = N->getDebugLoc();
616 GetSplitVector(N->getOperand(0), Lo, Hi);
617 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
618 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
621 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
623 SDValue Vec = N->getOperand(0);
624 SDValue Elt = N->getOperand(1);
625 SDValue Idx = N->getOperand(2);
626 DebugLoc dl = N->getDebugLoc();
627 GetSplitVector(Vec, Lo, Hi);
629 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
630 unsigned IdxVal = CIdx->getZExtValue();
631 unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
632 if (IdxVal < LoNumElts)
633 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
634 Lo.getValueType(), Lo, Elt, Idx);
636 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
637 DAG.getIntPtrConstant(IdxVal - LoNumElts));
641 // Spill the vector to the stack.
642 MVT VecVT = Vec.getValueType();
643 MVT EltVT = VecVT.getVectorElementType();
644 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
645 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0);
647 // Store the new element. This may be larger than the vector element type,
648 // so use a truncating store.
649 SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
651 TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForMVT());
652 Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT);
654 // Load the Lo part from the stack slot.
655 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0);
657 // Increment the pointer to the other part.
658 unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
659 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
660 DAG.getIntPtrConstant(IncrementSize));
662 // Load the Hi part from the stack slot.
663 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false,
664 MinAlign(Alignment, IncrementSize));
667 void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
670 DebugLoc dl = N->getDebugLoc();
671 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
672 Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
673 Hi = DAG.getUNDEF(HiVT);
676 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
678 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
680 DebugLoc dl = LD->getDebugLoc();
681 GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT);
683 ISD::LoadExtType ExtType = LD->getExtensionType();
684 SDValue Ch = LD->getChain();
685 SDValue Ptr = LD->getBasePtr();
686 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
687 const Value *SV = LD->getSrcValue();
688 int SVOffset = LD->getSrcValueOffset();
689 MVT MemoryVT = LD->getMemoryVT();
690 unsigned Alignment = LD->getAlignment();
691 bool isVolatile = LD->isVolatile();
693 MVT LoMemVT, HiMemVT;
694 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
696 Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset,
697 SV, SVOffset, LoMemVT, isVolatile, Alignment);
699 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
700 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
701 DAG.getIntPtrConstant(IncrementSize));
702 SVOffset += IncrementSize;
703 Alignment = MinAlign(Alignment, IncrementSize);
704 Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset,
705 SV, SVOffset, HiMemVT, isVolatile, Alignment);
707 // Build a factor node to remember that this load is independent of the
709 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
712 // Legalized the chain result - switch anything that used the old chain to
714 ReplaceValueWith(SDValue(LD, 1), Ch);
717 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
719 // Get the dest types - they may not match the input types, e.g. int_to_fp.
721 DebugLoc dl = N->getDebugLoc();
722 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
725 MVT InVT = N->getOperand(0).getValueType();
726 switch (getTypeAction(InVT)) {
727 default: assert(0 && "Unexpected type action!");
729 assert(LoVT == HiVT && "Legal non-power-of-two vector type?");
730 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
731 LoVT.getVectorNumElements());
732 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
733 DAG.getIntPtrConstant(0));
734 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
735 DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
739 GetSplitVector(N->getOperand(0), Lo, Hi);
742 // If the result needs to be split and the input needs to be widened,
743 // the two types must have different lengths. Use the widened result
744 // and extract from it to do the split.
745 assert(LoVT == HiVT && "Legal non-power-of-two vector type?");
746 SDValue InOp = GetWidenedVector(N->getOperand(0));
747 MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
748 LoVT.getVectorNumElements());
749 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
750 DAG.getIntPtrConstant(0));
751 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
752 DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
757 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
758 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
761 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
762 SDValue &Lo, SDValue &Hi) {
763 // The low and high parts of the original input give four input vectors.
765 DebugLoc dl = N->getDebugLoc();
766 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
767 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
768 MVT NewVT = Inputs[0].getValueType();
769 unsigned NewElts = NewVT.getVectorNumElements();
770 assert(NewVT == Inputs[1].getValueType() &&
771 "Non power-of-two vectors not supported!");
773 // If Lo or Hi uses elements from at most two of the four input vectors, then
774 // express it as a vector shuffle of those two inputs. Otherwise extract the
775 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
776 SmallVector<int, 16> Ops;
777 for (unsigned High = 0; High < 2; ++High) {
778 SDValue &Output = High ? Hi : Lo;
780 // Build a shuffle mask for the output, discovering on the fly which
781 // input vectors to use as shuffle operands (recorded in InputUsed).
782 // If building a suitable shuffle vector proves too hard, then bail
783 // out with useBuildVector set.
784 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
785 unsigned FirstMaskIdx = High * NewElts;
786 bool useBuildVector = false;
787 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
788 // The mask element. This indexes into the input.
789 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
791 // The input vector this mask element indexes into.
792 unsigned Input = (unsigned)Idx / NewElts;
794 if (Input >= array_lengthof(Inputs)) {
795 // The mask element does not index into any input vector.
800 // Turn the index into an offset from the start of the input vector.
801 Idx -= Input * NewElts;
803 // Find or create a shuffle vector operand to hold this input.
805 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
806 if (InputUsed[OpNo] == Input) {
807 // This input vector is already an operand.
809 } else if (InputUsed[OpNo] == -1U) {
810 // Create a new operand for this input vector.
811 InputUsed[OpNo] = Input;
816 if (OpNo >= array_lengthof(InputUsed)) {
817 // More than two input vectors used! Give up on trying to create a
818 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
819 useBuildVector = true;
823 // Add the mask index for the new shuffle vector.
824 Ops.push_back(Idx + OpNo * NewElts);
827 if (useBuildVector) {
828 MVT EltVT = NewVT.getVectorElementType();
829 SmallVector<SDValue, 16> SVOps;
831 // Extract the input elements by hand.
832 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
833 // The mask element. This indexes into the input.
834 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
836 // The input vector this mask element indexes into.
837 unsigned Input = (unsigned)Idx / NewElts;
839 if (Input >= array_lengthof(Inputs)) {
840 // The mask element is "undef" or indexes off the end of the input.
841 SVOps.push_back(DAG.getUNDEF(EltVT));
845 // Turn the index into an offset from the start of the input vector.
846 Idx -= Input * NewElts;
848 // Extract the vector element by hand.
849 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
850 Inputs[Input], DAG.getIntPtrConstant(Idx)));
853 // Construct the Lo/Hi output using a BUILD_VECTOR.
854 Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size());
855 } else if (InputUsed[0] == -1U) {
856 // No input vectors were used! The result is undefined.
857 Output = DAG.getUNDEF(NewVT);
859 SDValue Op0 = Inputs[InputUsed[0]];
860 // If only one input was used, use an undefined vector for the other.
861 SDValue Op1 = InputUsed[1] == -1U ?
862 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
863 // At least one input vector was used. Create a new shuffle vector.
864 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]);
871 void DAGTypeLegalizer::SplitVecRes_VSETCC(SDNode *N, SDValue &Lo,
874 DebugLoc dl = N->getDebugLoc();
875 GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
877 SDValue LL, LH, RL, RH;
878 GetSplitVector(N->getOperand(0), LL, LH);
879 GetSplitVector(N->getOperand(1), RL, RH);
881 Lo = DAG.getNode(ISD::VSETCC, dl, LoVT, LL, RL, N->getOperand(2));
882 Hi = DAG.getNode(ISD::VSETCC, dl, HiVT, LH, RH, N->getOperand(2));
886 //===----------------------------------------------------------------------===//
887 // Operand Vector Splitting
888 //===----------------------------------------------------------------------===//
890 /// SplitVectorOperand - This method is called when the specified operand of the
891 /// specified node is found to need vector splitting. At this point, all of the
892 /// result types of the node are known to be legal, but other operands of the
893 /// node may need legalization as well as the specified one.
894 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
895 DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n");
896 SDValue Res = SDValue();
898 if (Res.getNode() == 0) {
899 switch (N->getOpcode()) {
902 cerr << "SplitVectorOperand Op #" << OpNo << ": ";
903 N->dump(&DAG); cerr << "\n";
905 assert(0 && "Do not know how to split this operator's operand!");
908 case ISD::BIT_CONVERT: Res = SplitVecOp_BIT_CONVERT(N); break;
909 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
910 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
911 case ISD::STORE: Res = SplitVecOp_STORE(cast<StoreSDNode>(N),
917 case ISD::FP_TO_SINT:
918 case ISD::FP_TO_UINT:
919 case ISD::SINT_TO_FP:
921 case ISD::UINT_TO_FP: Res = SplitVecOp_UnaryOp(N); break;
925 // If the result is null, the sub-method took care of registering results etc.
926 if (!Res.getNode()) return false;
928 // If the result is N, the sub-method updated N in place. Tell the legalizer
930 if (Res.getNode() == N)
933 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
934 "Invalid operand expansion");
936 ReplaceValueWith(SDValue(N, 0), Res);
940 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
941 // The result has a legal vector type, but the input needs splitting.
942 MVT ResVT = N->getValueType(0);
944 DebugLoc dl = N->getDebugLoc();
945 GetSplitVector(N->getOperand(0), Lo, Hi);
946 assert(Lo.getValueType() == Hi.getValueType() &&
947 "Returns legal non-power-of-two vector type?");
948 MVT InVT = Lo.getValueType();
950 MVT OutVT = MVT::getVectorVT(ResVT.getVectorElementType(),
951 InVT.getVectorNumElements());
953 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
954 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
956 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
959 SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) {
960 // For example, i64 = BIT_CONVERT v4i16 on alpha. Typically the vector will
961 // end up being split all the way down to individual components. Convert the
962 // split pieces into integers and reassemble.
964 GetSplitVector(N->getOperand(0), Lo, Hi);
965 Lo = BitConvertToInteger(Lo);
966 Hi = BitConvertToInteger(Hi);
968 if (TLI.isBigEndian())
971 return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0),
972 JoinIntegers(Lo, Hi));
975 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
976 // We know that the extracted result type is legal. For now, assume the index
978 MVT SubVT = N->getValueType(0);
979 SDValue Idx = N->getOperand(1);
980 DebugLoc dl = N->getDebugLoc();
982 GetSplitVector(N->getOperand(0), Lo, Hi);
984 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
985 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
987 if (IdxVal < LoElts) {
988 assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
989 "Extracted subvector crosses vector split!");
990 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
992 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
993 DAG.getConstant(IdxVal - LoElts, Idx.getValueType()));
997 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
998 SDValue Vec = N->getOperand(0);
999 SDValue Idx = N->getOperand(1);
1000 MVT VecVT = Vec.getValueType();
1002 if (isa<ConstantSDNode>(Idx)) {
1003 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1004 assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1007 GetSplitVector(Vec, Lo, Hi);
1009 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1011 if (IdxVal < LoElts)
1012 return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx);
1014 return DAG.UpdateNodeOperands(SDValue(N, 0), Hi,
1015 DAG.getConstant(IdxVal - LoElts,
1016 Idx.getValueType()));
1019 // Store the vector to the stack.
1020 MVT EltVT = VecVT.getVectorElementType();
1021 DebugLoc dl = N->getDebugLoc();
1022 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1023 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1024 const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
1025 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0);
1027 // Load back the required element.
1028 StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
1029 return DAG.getLoad(EltVT, dl, Store, StackPtr, SV, 0);
1032 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1033 assert(N->isUnindexed() && "Indexed store of vector?");
1034 assert(OpNo == 1 && "Can only split the stored value");
1035 DebugLoc dl = N->getDebugLoc();
1037 bool isTruncating = N->isTruncatingStore();
1038 SDValue Ch = N->getChain();
1039 SDValue Ptr = N->getBasePtr();
1040 int SVOffset = N->getSrcValueOffset();
1041 MVT MemoryVT = N->getMemoryVT();
1042 unsigned Alignment = N->getAlignment();
1043 bool isVol = N->isVolatile();
1045 GetSplitVector(N->getOperand(1), Lo, Hi);
1047 MVT LoMemVT, HiMemVT;
1048 GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
1050 unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1053 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset,
1054 LoMemVT, isVol, Alignment);
1056 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset,
1059 // Increment the pointer to the other half.
1060 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1061 DAG.getIntPtrConstant(IncrementSize));
1064 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
1065 N->getSrcValue(), SVOffset+IncrementSize,
1067 isVol, MinAlign(Alignment, IncrementSize));
1069 Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
1070 isVol, MinAlign(Alignment, IncrementSize));
1072 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
1076 //===----------------------------------------------------------------------===//
1077 // Result Vector Widening
1078 //===----------------------------------------------------------------------===//
1080 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
1081 DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG);
1083 SDValue Res = SDValue();
1085 switch (N->getOpcode()) {
1088 cerr << "WidenVectorResult #" << ResNo << ": ";
1089 N->dump(&DAG); cerr << "\n";
1091 assert(0 && "Do not know how to widen the result of this operator!");
1094 case ISD::BIT_CONVERT: Res = WidenVecRes_BIT_CONVERT(N); break;
1095 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
1096 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
1097 case ISD::CONVERT_RNDSAT: Res = WidenVecRes_CONVERT_RNDSAT(N); break;
1098 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
1099 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
1100 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
1101 case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
1102 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
1103 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
1104 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
1105 case ISD::VECTOR_SHUFFLE:
1106 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N)); break;
1107 case ISD::VSETCC: Res = WidenVecRes_VSETCC(N); break;
1113 case ISD::FCOPYSIGN:
1129 case ISD::XOR: Res = WidenVecRes_Binary(N); break;
1133 case ISD::SRL: Res = WidenVecRes_Shift(N); break;
1135 case ISD::ANY_EXTEND:
1137 case ISD::FP_TO_SINT:
1138 case ISD::FP_TO_UINT:
1139 case ISD::SIGN_EXTEND:
1140 case ISD::SINT_TO_FP:
1142 case ISD::ZERO_EXTEND:
1143 case ISD::UINT_TO_FP: Res = WidenVecRes_Convert(N); break;
1152 case ISD::FSQRT: Res = WidenVecRes_Unary(N); break;
1155 // If Res is null, the sub-method took care of registering the result.
1157 SetWidenedVector(SDValue(N, ResNo), Res);
1160 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
1161 // Binary op widening.
1162 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1163 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1164 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1165 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2);
1168 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
1169 SDValue InOp = N->getOperand(0);
1170 DebugLoc dl = N->getDebugLoc();
1172 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1173 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1175 MVT InVT = InOp.getValueType();
1176 MVT InEltVT = InVT.getVectorElementType();
1177 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts);
1179 unsigned Opcode = N->getOpcode();
1180 unsigned InVTNumElts = InVT.getVectorNumElements();
1182 if (getTypeAction(InVT) == WidenVector) {
1183 InOp = GetWidenedVector(N->getOperand(0));
1184 InVT = InOp.getValueType();
1185 InVTNumElts = InVT.getVectorNumElements();
1186 if (InVTNumElts == WidenNumElts)
1187 return DAG.getNode(Opcode, dl, WidenVT, InOp);
1190 if (TLI.isTypeLegal(InWidenVT)) {
1191 // Because the result and the input are different vector types, widening
1192 // the result could create a legal type but widening the input might make
1193 // it an illegal type that might lead to repeatedly splitting the input
1194 // and then widening it. To avoid this, we widen the input only if
1195 // it results in a legal type.
1196 if (WidenNumElts % InVTNumElts == 0) {
1197 // Widen the input and call convert on the widened input vector.
1198 unsigned NumConcat = WidenNumElts/InVTNumElts;
1199 SmallVector<SDValue, 16> Ops(NumConcat);
1201 SDValue UndefVal = DAG.getUNDEF(InVT);
1202 for (unsigned i = 1; i != NumConcat; ++i)
1204 return DAG.getNode(Opcode, dl, WidenVT,
1205 DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT,
1206 &Ops[0], NumConcat));
1209 if (InVTNumElts % WidenNumElts == 0) {
1210 // Extract the input and convert the shorten input vector.
1211 return DAG.getNode(Opcode, dl, WidenVT,
1212 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT,
1213 InOp, DAG.getIntPtrConstant(0)));
1217 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1218 SmallVector<SDValue, 16> Ops(WidenNumElts);
1219 MVT EltVT = WidenVT.getVectorElementType();
1220 unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
1222 for (i=0; i < MinElts; ++i)
1223 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
1224 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
1225 DAG.getIntPtrConstant(i)));
1227 SDValue UndefVal = DAG.getUNDEF(EltVT);
1228 for (; i < WidenNumElts; ++i)
1231 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1234 SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
1235 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1236 SDValue InOp = GetWidenedVector(N->getOperand(0));
1237 SDValue ShOp = N->getOperand(1);
1239 MVT ShVT = ShOp.getValueType();
1240 if (getTypeAction(ShVT) == WidenVector) {
1241 ShOp = GetWidenedVector(ShOp);
1242 ShVT = ShOp.getValueType();
1244 MVT ShWidenVT = MVT::getVectorVT(ShVT.getVectorElementType(),
1245 WidenVT.getVectorNumElements());
1246 if (ShVT != ShWidenVT)
1247 ShOp = ModifyToType(ShOp, ShWidenVT);
1249 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp);
1252 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
1253 // Unary op widening.
1254 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1255 SDValue InOp = GetWidenedVector(N->getOperand(0));
1256 return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp);
1259 SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) {
1260 SDValue InOp = N->getOperand(0);
1261 MVT InVT = InOp.getValueType();
1262 MVT VT = N->getValueType(0);
1263 MVT WidenVT = TLI.getTypeToTransformTo(VT);
1264 DebugLoc dl = N->getDebugLoc();
1266 switch (getTypeAction(InVT)) {
1268 assert(false && "Unknown type action!");
1272 case PromoteInteger:
1273 // If the InOp is promoted to the same size, convert it. Otherwise,
1274 // fall out of the switch and widen the promoted input.
1275 InOp = GetPromotedInteger(InOp);
1276 InVT = InOp.getValueType();
1277 if (WidenVT.bitsEq(InVT))
1278 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp);
1283 case ScalarizeVector:
1287 // If the InOp is widened to the same size, convert it. Otherwise, fall
1288 // out of the switch and widen the widened input.
1289 InOp = GetWidenedVector(InOp);
1290 InVT = InOp.getValueType();
1291 if (WidenVT.bitsEq(InVT))
1292 // The input widens to the same size. Convert to the widen value.
1293 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp);
1297 unsigned WidenSize = WidenVT.getSizeInBits();
1298 unsigned InSize = InVT.getSizeInBits();
1299 if (WidenSize % InSize == 0) {
1300 // Determine new input vector type. The new input vector type will use
1301 // the same element type (if its a vector) or use the input type as a
1302 // vector. It is the same size as the type to widen to.
1304 unsigned NewNumElts = WidenSize / InSize;
1305 if (InVT.isVector()) {
1306 MVT InEltVT = InVT.getVectorElementType();
1307 NewInVT= MVT::getVectorVT(InEltVT, WidenSize / InEltVT.getSizeInBits());
1309 NewInVT = MVT::getVectorVT(InVT, NewNumElts);
1312 if (TLI.isTypeLegal(NewInVT)) {
1313 // Because the result and the input are different vector types, widening
1314 // the result could create a legal type but widening the input might make
1315 // it an illegal type that might lead to repeatedly splitting the input
1316 // and then widening it. To avoid this, we widen the input only if
1317 // it results in a legal type.
1318 SmallVector<SDValue, 16> Ops(NewNumElts);
1319 SDValue UndefVal = DAG.getUNDEF(InVT);
1321 for (unsigned i = 1; i < NewNumElts; ++i)
1325 if (InVT.isVector())
1326 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl,
1327 NewInVT, &Ops[0], NewNumElts);
1329 NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
1330 NewInVT, &Ops[0], NewNumElts);
1331 return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec);
1335 // This should occur rarely. Lower the bit-convert to a store/load
1336 // from the stack. Create the stack frame object. Make sure it is aligned
1337 // for both the source and destination types.
1338 SDValue FIPtr = DAG.CreateStackTemporary(InVT, WidenVT);
1339 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1340 const Value *SV = PseudoSourceValue::getFixedStack(FI);
1342 // Emit a store to the stack slot.
1343 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0);
1345 // Result is a load from the stack slot.
1346 return DAG.getLoad(WidenVT, dl, Store, FIPtr, SV, 0);
1349 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
1350 DebugLoc dl = N->getDebugLoc();
1351 // Build a vector with undefined for the new nodes.
1352 MVT VT = N->getValueType(0);
1353 MVT EltVT = VT.getVectorElementType();
1354 unsigned NumElts = VT.getVectorNumElements();
1356 MVT WidenVT = TLI.getTypeToTransformTo(VT);
1357 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1359 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
1360 NewOps.reserve(WidenNumElts);
1361 for (unsigned i = NumElts; i < WidenNumElts; ++i)
1362 NewOps.push_back(DAG.getUNDEF(EltVT));
1364 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size());
1367 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
1368 MVT InVT = N->getOperand(0).getValueType();
1369 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1370 DebugLoc dl = N->getDebugLoc();
1371 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1372 unsigned NumOperands = N->getNumOperands();
1374 bool InputWidened = false; // Indicates we need to widen the input.
1375 if (getTypeAction(InVT) != WidenVector) {
1376 if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
1377 // Add undef vectors to widen to correct length.
1378 unsigned NumConcat = WidenVT.getVectorNumElements() /
1379 InVT.getVectorNumElements();
1380 SDValue UndefVal = DAG.getUNDEF(InVT);
1381 SmallVector<SDValue, 16> Ops(NumConcat);
1382 for (unsigned i=0; i < NumOperands; ++i)
1383 Ops[i] = N->getOperand(i);
1384 for (unsigned i = NumOperands; i != NumConcat; ++i)
1386 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat);
1389 InputWidened = true;
1390 if (WidenVT == TLI.getTypeToTransformTo(InVT)) {
1391 // The inputs and the result are widen to the same value.
1393 for (i=1; i < NumOperands; ++i)
1394 if (N->getOperand(i).getOpcode() != ISD::UNDEF)
1397 if (i > NumOperands)
1398 // Everything but the first operand is an UNDEF so just return the
1399 // widened first operand.
1400 return GetWidenedVector(N->getOperand(0));
1402 if (NumOperands == 2) {
1403 // Replace concat of two operands with a shuffle.
1404 SmallVector<int, 16> MaskOps(WidenNumElts);
1405 for (unsigned i=0; i < WidenNumElts/2; ++i) {
1407 MaskOps[i+WidenNumElts/2] = i+WidenNumElts;
1409 return DAG.getVectorShuffle(WidenVT, dl,
1410 GetWidenedVector(N->getOperand(0)),
1411 GetWidenedVector(N->getOperand(1)),
1417 // Fall back to use extracts and build vector.
1418 MVT EltVT = WidenVT.getVectorElementType();
1419 unsigned NumInElts = InVT.getVectorNumElements();
1420 SmallVector<SDValue, 16> Ops(WidenNumElts);
1422 for (unsigned i=0; i < NumOperands; ++i) {
1423 SDValue InOp = N->getOperand(i);
1425 InOp = GetWidenedVector(InOp);
1426 for (unsigned j=0; j < NumInElts; ++j)
1427 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1428 DAG.getIntPtrConstant(j));
1430 SDValue UndefVal = DAG.getUNDEF(EltVT);
1431 for (; Idx < WidenNumElts; ++Idx)
1432 Ops[Idx] = UndefVal;
1433 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1436 SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
1437 DebugLoc dl = N->getDebugLoc();
1438 SDValue InOp = N->getOperand(0);
1439 SDValue RndOp = N->getOperand(3);
1440 SDValue SatOp = N->getOperand(4);
1442 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1443 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1445 MVT InVT = InOp.getValueType();
1446 MVT InEltVT = InVT.getVectorElementType();
1447 MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts);
1449 SDValue DTyOp = DAG.getValueType(WidenVT);
1450 SDValue STyOp = DAG.getValueType(InWidenVT);
1451 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
1453 unsigned InVTNumElts = InVT.getVectorNumElements();
1454 if (getTypeAction(InVT) == WidenVector) {
1455 InOp = GetWidenedVector(InOp);
1456 InVT = InOp.getValueType();
1457 InVTNumElts = InVT.getVectorNumElements();
1458 if (InVTNumElts == WidenNumElts)
1459 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
1463 if (TLI.isTypeLegal(InWidenVT)) {
1464 // Because the result and the input are different vector types, widening
1465 // the result could create a legal type but widening the input might make
1466 // it an illegal type that might lead to repeatedly splitting the input
1467 // and then widening it. To avoid this, we widen the input only if
1468 // it results in a legal type.
1469 if (WidenNumElts % InVTNumElts == 0) {
1470 // Widen the input and call convert on the widened input vector.
1471 unsigned NumConcat = WidenNumElts/InVTNumElts;
1472 SmallVector<SDValue, 16> Ops(NumConcat);
1474 SDValue UndefVal = DAG.getUNDEF(InVT);
1475 for (unsigned i = 1; i != NumConcat; ++i) {
1478 InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat);
1479 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
1483 if (InVTNumElts % WidenNumElts == 0) {
1484 // Extract the input and convert the shorten input vector.
1485 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
1486 DAG.getIntPtrConstant(0));
1487 return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
1492 // Otherwise unroll into some nasty scalar code and rebuild the vector.
1493 SmallVector<SDValue, 16> Ops(WidenNumElts);
1494 MVT EltVT = WidenVT.getVectorElementType();
1495 DTyOp = DAG.getValueType(EltVT);
1496 STyOp = DAG.getValueType(InEltVT);
1498 unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
1500 for (i=0; i < MinElts; ++i) {
1501 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
1502 DAG.getIntPtrConstant(i));
1503 Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp,
1507 SDValue UndefVal = DAG.getUNDEF(EltVT);
1508 for (; i < WidenNumElts; ++i)
1511 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1514 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
1515 MVT VT = N->getValueType(0);
1516 MVT WidenVT = TLI.getTypeToTransformTo(VT);
1517 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1518 SDValue InOp = N->getOperand(0);
1519 SDValue Idx = N->getOperand(1);
1520 DebugLoc dl = N->getDebugLoc();
1522 if (getTypeAction(InOp.getValueType()) == WidenVector)
1523 InOp = GetWidenedVector(InOp);
1525 MVT InVT = InOp.getValueType();
1527 ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx);
1529 unsigned IdxVal = CIdx->getZExtValue();
1530 // Check if we can just return the input vector after widening.
1531 if (IdxVal == 0 && InVT == WidenVT)
1534 // Check if we can extract from the vector.
1535 unsigned InNumElts = InVT.getVectorNumElements();
1536 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
1537 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
1540 // We could try widening the input to the right length but for now, extract
1541 // the original elements, fill the rest with undefs and build a vector.
1542 SmallVector<SDValue, 16> Ops(WidenNumElts);
1543 MVT EltVT = VT.getVectorElementType();
1544 MVT IdxVT = Idx.getValueType();
1545 unsigned NumElts = VT.getVectorNumElements();
1548 unsigned IdxVal = CIdx->getZExtValue();
1549 for (i=0; i < NumElts; ++i)
1550 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1551 DAG.getConstant(IdxVal+i, IdxVT));
1553 Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx);
1554 for (i=1; i < NumElts; ++i) {
1555 SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
1556 DAG.getConstant(i, IdxVT));
1557 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx);
1561 SDValue UndefVal = DAG.getUNDEF(EltVT);
1562 for (; i < WidenNumElts; ++i)
1564 return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1567 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
1568 SDValue InOp = GetWidenedVector(N->getOperand(0));
1569 return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(),
1570 InOp.getValueType(), InOp,
1571 N->getOperand(1), N->getOperand(2));
1574 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
1575 LoadSDNode *LD = cast<LoadSDNode>(N);
1576 MVT WidenVT = TLI.getTypeToTransformTo(LD->getValueType(0));
1577 MVT LdVT = LD->getMemoryVT();
1578 DebugLoc dl = N->getDebugLoc();
1579 assert(LdVT.isVector() && WidenVT.isVector());
1582 SDValue Chain = LD->getChain();
1583 SDValue BasePtr = LD->getBasePtr();
1584 int SVOffset = LD->getSrcValueOffset();
1585 unsigned Align = LD->getAlignment();
1586 bool isVolatile = LD->isVolatile();
1587 const Value *SV = LD->getSrcValue();
1588 ISD::LoadExtType ExtType = LD->getExtensionType();
1591 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
1592 if (ExtType != ISD::NON_EXTLOAD) {
1593 // For extension loads, we can not play the tricks of chopping legal
1594 // vector types and bit cast it to the right type. Instead, we unroll
1595 // the load and build a vector.
1596 MVT EltVT = WidenVT.getVectorElementType();
1597 MVT LdEltVT = LdVT.getVectorElementType();
1598 unsigned NumElts = LdVT.getVectorNumElements();
1600 // Load each element and widen
1601 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1602 SmallVector<SDValue, 16> Ops(WidenNumElts);
1603 unsigned Increment = LdEltVT.getSizeInBits() / 8;
1604 Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset,
1605 LdEltVT, isVolatile, Align);
1606 LdChain.push_back(Ops[0].getValue(1));
1607 unsigned i = 0, Offset = Increment;
1608 for (i=1; i < NumElts; ++i, Offset += Increment) {
1609 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
1610 BasePtr, DAG.getIntPtrConstant(Offset));
1611 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV,
1612 SVOffset + Offset, LdEltVT, isVolatile, Align);
1613 LdChain.push_back(Ops[i].getValue(1));
1616 // Fill the rest with undefs
1617 SDValue UndefVal = DAG.getUNDEF(EltVT);
1618 for (; i != WidenNumElts; ++i)
1621 Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size());
1623 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
1624 unsigned int LdWidth = LdVT.getSizeInBits();
1625 Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset,
1626 Align, isVolatile, LdWidth, WidenVT, dl);
1629 // If we generate a single load, we can use that for the chain. Otherwise,
1630 // build a factor node to remember the multiple loads are independent and
1633 if (LdChain.size() == 1)
1634 NewChain = LdChain[0];
1636 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0],
1639 // Modified the chain - switch anything that used the old chain to use
1641 ReplaceValueWith(SDValue(N, 1), Chain);
1646 SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
1647 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1648 return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(),
1649 WidenVT, N->getOperand(0));
1652 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
1653 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1654 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1656 SDValue Cond1 = N->getOperand(0);
1657 MVT CondVT = Cond1.getValueType();
1658 if (CondVT.isVector()) {
1659 MVT CondEltVT = CondVT.getVectorElementType();
1660 MVT CondWidenVT = MVT::getVectorVT(CondEltVT, WidenNumElts);
1661 if (getTypeAction(CondVT) == WidenVector)
1662 Cond1 = GetWidenedVector(Cond1);
1664 if (Cond1.getValueType() != CondWidenVT)
1665 Cond1 = ModifyToType(Cond1, CondWidenVT);
1668 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
1669 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
1670 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
1671 return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
1672 WidenVT, Cond1, InOp1, InOp2);
1675 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
1676 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
1677 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
1678 return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(),
1679 InOp1.getValueType(), N->getOperand(0),
1680 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
1683 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
1684 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1685 return DAG.getUNDEF(WidenVT);
1688 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
1689 MVT VT = N->getValueType(0);
1690 DebugLoc dl = N->getDebugLoc();
1692 MVT WidenVT = TLI.getTypeToTransformTo(VT);
1693 unsigned NumElts = VT.getVectorNumElements();
1694 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1696 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1697 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1699 // Adjust mask based on new input vector length.
1700 SmallVector<int, 16> NewMask;
1701 for (unsigned i = 0; i != NumElts; ++i) {
1702 int Idx = N->getMaskElt(i);
1703 if (Idx < (int)NumElts)
1704 NewMask.push_back(Idx);
1706 NewMask.push_back(Idx - NumElts + WidenNumElts);
1708 for (unsigned i = NumElts; i != WidenNumElts; ++i)
1709 NewMask.push_back(-1);
1710 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]);
1713 SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
1714 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1715 unsigned WidenNumElts = WidenVT.getVectorNumElements();
1717 SDValue InOp1 = N->getOperand(0);
1718 MVT InVT = InOp1.getValueType();
1719 assert(InVT.isVector() && "can not widen non vector type");
1720 MVT WidenInVT = MVT::getVectorVT(InVT.getVectorElementType(), WidenNumElts);
1721 InOp1 = GetWidenedVector(InOp1);
1722 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1724 // Assume that the input and output will be widen appropriately. If not,
1725 // we will have to unroll it at some point.
1726 assert(InOp1.getValueType() == WidenInVT &&
1727 InOp2.getValueType() == WidenInVT &&
1728 "Input not widened to expected type!");
1729 return DAG.getNode(ISD::VSETCC, N->getDebugLoc(),
1730 WidenVT, InOp1, InOp2, N->getOperand(2));
1734 //===----------------------------------------------------------------------===//
1735 // Widen Vector Operand
1736 //===----------------------------------------------------------------------===//
1737 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) {
1738 DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG);
1740 SDValue Res = SDValue();
1742 switch (N->getOpcode()) {
1745 cerr << "WidenVectorOperand op #" << ResNo << ": ";
1746 N->dump(&DAG); cerr << "\n";
1748 assert(0 && "Do not know how to widen this operator's operand!");
1751 case ISD::BIT_CONVERT: Res = WidenVecOp_BIT_CONVERT(N); break;
1752 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
1753 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
1754 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
1757 case ISD::FP_TO_SINT:
1758 case ISD::FP_TO_UINT:
1759 case ISD::SINT_TO_FP:
1761 case ISD::UINT_TO_FP: Res = WidenVecOp_Convert(N); break;
1764 // If Res is null, the sub-method took care of registering the result.
1765 if (!Res.getNode()) return false;
1767 // If the result is N, the sub-method updated N in place. Tell the legalizer
1769 if (Res.getNode() == N)
1773 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1774 "Invalid operand expansion");
1776 ReplaceValueWith(SDValue(N, 0), Res);
1780 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
1781 // Since the result is legal and the input is illegal, it is unlikely
1782 // that we can fix the input to a legal type so unroll the convert
1783 // into some scalar code and create a nasty build vector.
1784 MVT VT = N->getValueType(0);
1785 MVT EltVT = VT.getVectorElementType();
1786 DebugLoc dl = N->getDebugLoc();
1787 unsigned NumElts = VT.getVectorNumElements();
1788 SDValue InOp = N->getOperand(0);
1789 if (getTypeAction(InOp.getValueType()) == WidenVector)
1790 InOp = GetWidenedVector(InOp);
1791 MVT InVT = InOp.getValueType();
1792 MVT InEltVT = InVT.getVectorElementType();
1794 unsigned Opcode = N->getOpcode();
1795 SmallVector<SDValue, 16> Ops(NumElts);
1796 for (unsigned i=0; i < NumElts; ++i)
1797 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
1798 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
1799 DAG.getIntPtrConstant(i)));
1801 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
1804 SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) {
1805 MVT VT = N->getValueType(0);
1806 SDValue InOp = GetWidenedVector(N->getOperand(0));
1807 MVT InWidenVT = InOp.getValueType();
1808 DebugLoc dl = N->getDebugLoc();
1810 // Check if we can convert between two legal vector types and extract.
1811 unsigned InWidenSize = InWidenVT.getSizeInBits();
1812 unsigned Size = VT.getSizeInBits();
1813 if (InWidenSize % Size == 0 && !VT.isVector()) {
1814 unsigned NewNumElts = InWidenSize / Size;
1815 MVT NewVT = MVT::getVectorVT(VT, NewNumElts);
1816 if (TLI.isTypeLegal(NewVT)) {
1817 SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp);
1818 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
1819 DAG.getIntPtrConstant(0));
1823 // Lower the bit-convert to a store/load from the stack. Create the stack
1824 // frame object. Make sure it is aligned for both the source and destination
1826 SDValue FIPtr = DAG.CreateStackTemporary(InWidenVT, VT);
1827 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1828 const Value *SV = PseudoSourceValue::getFixedStack(FI);
1830 // Emit a store to the stack slot.
1831 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, FIPtr, SV, 0);
1833 // Result is a load from the stack slot.
1834 return DAG.getLoad(VT, dl, Store, FIPtr, SV, 0);
1837 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
1838 // If the input vector is not legal, it is likely that we will not find a
1839 // legal vector of the same size. Replace the concatenate vector with a
1840 // nasty build vector.
1841 MVT VT = N->getValueType(0);
1842 MVT EltVT = VT.getVectorElementType();
1843 DebugLoc dl = N->getDebugLoc();
1844 unsigned NumElts = VT.getVectorNumElements();
1845 SmallVector<SDValue, 16> Ops(NumElts);
1847 MVT InVT = N->getOperand(0).getValueType();
1848 unsigned NumInElts = InVT.getVectorNumElements();
1851 unsigned NumOperands = N->getNumOperands();
1852 for (unsigned i=0; i < NumOperands; ++i) {
1853 SDValue InOp = N->getOperand(i);
1854 if (getTypeAction(InOp.getValueType()) == WidenVector)
1855 InOp = GetWidenedVector(InOp);
1856 for (unsigned j=0; j < NumInElts; ++j)
1857 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1858 DAG.getIntPtrConstant(j));
1860 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
1863 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1864 SDValue InOp = GetWidenedVector(N->getOperand(0));
1865 MVT EltVT = InOp.getValueType().getVectorElementType();
1866 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
1867 EltVT, InOp, N->getOperand(1));
1870 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
1871 // We have to widen the value but we want only to store the original
1873 StoreSDNode *ST = cast<StoreSDNode>(N);
1874 SDValue Chain = ST->getChain();
1875 SDValue BasePtr = ST->getBasePtr();
1876 const Value *SV = ST->getSrcValue();
1877 int SVOffset = ST->getSrcValueOffset();
1878 unsigned Align = ST->getAlignment();
1879 bool isVolatile = ST->isVolatile();
1880 SDValue ValOp = GetWidenedVector(ST->getValue());
1881 DebugLoc dl = N->getDebugLoc();
1883 MVT StVT = ST->getMemoryVT();
1884 MVT ValVT = ValOp.getValueType();
1885 // It must be true that we the widen vector type is bigger than where
1886 // we need to store.
1887 assert(StVT.isVector() && ValOp.getValueType().isVector());
1888 assert(StVT.bitsLT(ValOp.getValueType()));
1890 SmallVector<SDValue, 16> StChain;
1891 if (ST->isTruncatingStore()) {
1892 // For truncating stores, we can not play the tricks of chopping legal
1893 // vector types and bit cast it to the right type. Instead, we unroll
1895 MVT StEltVT = StVT.getVectorElementType();
1896 MVT ValEltVT = ValVT.getVectorElementType();
1897 unsigned Increment = ValEltVT.getSizeInBits() / 8;
1898 unsigned NumElts = StVT.getVectorNumElements();
1899 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
1900 DAG.getIntPtrConstant(0));
1901 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV,
1903 isVolatile, Align));
1904 unsigned Offset = Increment;
1905 for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
1906 SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
1907 BasePtr, DAG.getIntPtrConstant(Offset));
1908 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
1909 DAG.getIntPtrConstant(0));
1910 StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV,
1911 SVOffset + Offset, StEltVT,
1912 isVolatile, MinAlign(Align, Offset)));
1916 assert(StVT.getVectorElementType() == ValVT.getVectorElementType());
1918 GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset,
1919 Align, isVolatile, ValOp, StVT.getSizeInBits(), dl);
1921 if (StChain.size() == 1)
1924 return DAG.getNode(ISD::TokenFactor, dl,
1925 MVT::Other,&StChain[0],StChain.size());
1928 //===----------------------------------------------------------------------===//
1929 // Vector Widening Utilities
1930 //===----------------------------------------------------------------------===//
1933 // Utility function to find a vector type and its associated element
1934 // type from a preferred width and whose vector type must be the same size
1936 // TLI: Target lowering used to determine legal types.
1937 // Width: Preferred width to store.
1938 // VecVT: Vector value type whose size we must match.
1939 // Returns NewVecVT and NewEltVT - the vector type and its associated
1941 static void FindAssocWidenVecType(const TargetLowering &TLI, unsigned Width,
1943 MVT& NewEltVT, MVT& NewVecVT) {
1944 unsigned EltWidth = Width + 1;
1945 if (TLI.isTypeLegal(VecVT)) {
1946 // We start with the preferred with, making it a power of 2 and find a
1947 // legal vector type of that width. If not, we reduce it by another of 2.
1948 // For incoming type is legal, this process will end as a vector of the
1949 // smallest loadable type should always be legal.
1951 assert(EltWidth > 0);
1952 EltWidth = 1 << Log2_32(EltWidth - 1);
1953 NewEltVT = MVT::getIntegerVT(EltWidth);
1954 unsigned NumElts = VecVT.getSizeInBits() / EltWidth;
1955 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts);
1956 } while (!TLI.isTypeLegal(NewVecVT) ||
1957 VecVT.getSizeInBits() != NewVecVT.getSizeInBits());
1959 // The incoming vector type is illegal and is the result of widening
1960 // a vector to a power of 2. In this case, we will use the preferred
1961 // with as long as it is a multiple of the incoming vector length.
1962 // The legalization process will eventually make this into a legal type
1963 // and remove the illegal bit converts (which would turn to stack converts
1964 // if they are allow to exist).
1966 assert(EltWidth > 0);
1967 EltWidth = 1 << Log2_32(EltWidth - 1);
1968 NewEltVT = MVT::getIntegerVT(EltWidth);
1969 unsigned NumElts = VecVT.getSizeInBits() / EltWidth;
1970 NewVecVT = MVT::getVectorVT(NewEltVT, NumElts);
1971 } while (!TLI.isTypeLegal(NewEltVT) ||
1972 VecVT.getSizeInBits() != NewVecVT.getSizeInBits());
1976 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain,
1986 // The strategy assumes that we can efficiently load powers of two widths.
1987 // The routines chops the vector into the largest power of 2 load and
1988 // can be inserted into a legal vector and then cast the result into the
1989 // vector type we want. This avoids unnecessary stack converts.
1991 // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and
1992 // the load is nonvolatile, we an use a wider load for the value.
1994 // Find the vector type that can load from.
1995 MVT NewEltVT, NewVecVT;
1996 unsigned NewEltVTWidth;
1997 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT);
1998 NewEltVTWidth = NewEltVT.getSizeInBits();
2000 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset,
2001 isVolatile, Alignment);
2002 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
2003 LdChain.push_back(LdOp.getValue(1));
2005 // Check if we can load the element with one instruction
2006 if (LdWidth == NewEltVTWidth) {
2007 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp);
2011 LdWidth -= NewEltVTWidth;
2012 unsigned Offset = 0;
2014 while (LdWidth > 0) {
2015 unsigned Increment = NewEltVTWidth / 8;
2016 Offset += Increment;
2017 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2018 DAG.getIntPtrConstant(Increment));
2020 if (LdWidth < NewEltVTWidth) {
2021 // Our current type we are using is too large, use a smaller size by
2022 // using a smaller power of 2
2023 unsigned oNewEltVTWidth = NewEltVTWidth;
2024 FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT);
2025 NewEltVTWidth = NewEltVT.getSizeInBits();
2026 // Readjust position and vector position based on new load type
2027 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth);
2028 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp);
2031 SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV,
2032 SVOffset+Offset, isVolatile,
2033 MinAlign(Alignment, Offset));
2034 LdChain.push_back(LdOp.getValue(1));
2035 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp,
2036 DAG.getIntPtrConstant(Idx++));
2038 LdWidth -= NewEltVTWidth;
2041 return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp);
2044 void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain,
2054 // Breaks the stores into a series of power of 2 width stores. For any
2055 // width, we convert the vector to the vector of element size that we
2056 // want to store. This avoids requiring a stack convert.
2058 // Find a width of the element type we can store with
2059 MVT WidenVT = ValOp.getValueType();
2060 MVT NewEltVT, NewVecVT;
2062 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT);
2063 unsigned NewEltVTWidth = NewEltVT.getSizeInBits();
2065 SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp);
2066 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp,
2067 DAG.getIntPtrConstant(0));
2068 SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset,
2069 isVolatile, Alignment);
2070 StChain.push_back(StOp);
2072 // Check if we are done
2073 if (StWidth == NewEltVTWidth) {
2078 StWidth -= NewEltVTWidth;
2079 unsigned Offset = 0;
2081 while (StWidth > 0) {
2082 unsigned Increment = NewEltVTWidth / 8;
2083 Offset += Increment;
2084 BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2085 DAG.getIntPtrConstant(Increment));
2087 if (StWidth < NewEltVTWidth) {
2088 // Our current type we are using is too large, use a smaller size by
2089 // using a smaller power of 2
2090 unsigned oNewEltVTWidth = NewEltVTWidth;
2091 FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT);
2092 NewEltVTWidth = NewEltVT.getSizeInBits();
2093 // Readjust position and vector position based on new load type
2094 Idx = Idx * (oNewEltVTWidth/NewEltVTWidth);
2095 VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp);
2098 EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp,
2099 DAG.getIntPtrConstant(Idx++));
2100 StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV,
2101 SVOffset + Offset, isVolatile,
2102 MinAlign(Alignment, Offset)));
2103 StWidth -= NewEltVTWidth;
2107 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
2108 /// input vector must have the same element type as NVT.
2109 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, MVT NVT) {
2110 // Note that InOp might have been widened so it might already have
2111 // the right width or it might need be narrowed.
2112 MVT InVT = InOp.getValueType();
2113 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
2114 "input and widen element type must match");
2115 DebugLoc dl = InOp.getDebugLoc();
2117 // Check if InOp already has the right width.
2121 unsigned InNumElts = InVT.getVectorNumElements();
2122 unsigned WidenNumElts = NVT.getVectorNumElements();
2123 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
2124 unsigned NumConcat = WidenNumElts / InNumElts;
2125 SmallVector<SDValue, 16> Ops(NumConcat);
2126 SDValue UndefVal = DAG.getUNDEF(InVT);
2128 for (unsigned i = 1; i != NumConcat; ++i)
2131 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat);
2134 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
2135 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
2136 DAG.getIntPtrConstant(0));
2138 // Fall back to extract and build.
2139 SmallVector<SDValue, 16> Ops(WidenNumElts);
2140 MVT EltVT = NVT.getVectorElementType();
2141 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
2143 for (Idx = 0; Idx < MinNumElts; ++Idx)
2144 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2145 DAG.getIntPtrConstant(Idx));
2147 SDValue UndefVal = DAG.getUNDEF(EltVT);
2148 for ( ; Idx < WidenNumElts; ++Idx)
2149 Ops[Idx] = UndefVal;
2150 return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts);