1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the interfaces that X86 uses to lower LLVM code into a
13 //===----------------------------------------------------------------------===//
15 #include "X86ISelLowering.h"
16 #include "Utils/X86ShuffleDecode.h"
17 #include "X86CallingConv.h"
18 #include "X86FrameLowering.h"
19 #include "X86InstrBuilder.h"
20 #include "X86MachineFunctionInfo.h"
21 #include "X86TargetMachine.h"
22 #include "X86TargetObjectFile.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/ADT/StringExtras.h"
27 #include "llvm/ADT/StringSwitch.h"
28 #include "llvm/ADT/VariadicFunction.h"
29 #include "llvm/CodeGen/IntrinsicLowering.h"
30 #include "llvm/CodeGen/MachineFrameInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineJumpTableInfo.h"
34 #include "llvm/CodeGen/MachineModuleInfo.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/IR/CallSite.h"
37 #include "llvm/IR/CallingConv.h"
38 #include "llvm/IR/Constants.h"
39 #include "llvm/IR/DerivedTypes.h"
40 #include "llvm/IR/Function.h"
41 #include "llvm/IR/GlobalAlias.h"
42 #include "llvm/IR/GlobalVariable.h"
43 #include "llvm/IR/Instructions.h"
44 #include "llvm/IR/Intrinsics.h"
45 #include "llvm/MC/MCAsmInfo.h"
46 #include "llvm/MC/MCContext.h"
47 #include "llvm/MC/MCExpr.h"
48 #include "llvm/MC/MCSymbol.h"
49 #include "llvm/Support/CommandLine.h"
50 #include "llvm/Support/Debug.h"
51 #include "llvm/Support/ErrorHandling.h"
52 #include "llvm/Support/MathExtras.h"
53 #include "llvm/Target/TargetOptions.h"
54 #include "X86IntrinsicsInfo.h"
60 #define DEBUG_TYPE "x86-isel"
62 STATISTIC(NumTailCalls, "Number of tail calls");
64 static cl::opt<bool> ExperimentalVectorWideningLegalization(
65 "x86-experimental-vector-widening-legalization", cl::init(false),
66 cl::desc("Enable an experimental vector type legalization through widening "
67 "rather than promotion."),
70 static cl::opt<bool> ExperimentalVectorShuffleLowering(
71 "x86-experimental-vector-shuffle-lowering", cl::init(true),
72 cl::desc("Enable an experimental vector shuffle lowering code path."),
75 static cl::opt<bool> ExperimentalVectorShuffleLegality(
76 "x86-experimental-vector-shuffle-legality", cl::init(false),
77 cl::desc("Enable experimental shuffle legality based on the experimental "
78 "shuffle lowering. Should only be used with the experimental "
82 static cl::opt<int> ReciprocalEstimateRefinementSteps(
83 "x86-recip-refinement-steps", cl::init(1),
84 cl::desc("Specify the number of Newton-Raphson iterations applied to the "
85 "result of the hardware reciprocal estimate instruction."),
88 // Forward declarations.
89 static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
92 static SDValue ExtractSubVector(SDValue Vec, unsigned IdxVal,
93 SelectionDAG &DAG, SDLoc dl,
94 unsigned vectorWidth) {
95 assert((vectorWidth == 128 || vectorWidth == 256) &&
96 "Unsupported vector width");
97 EVT VT = Vec.getValueType();
98 EVT ElVT = VT.getVectorElementType();
99 unsigned Factor = VT.getSizeInBits()/vectorWidth;
100 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
101 VT.getVectorNumElements()/Factor);
103 // Extract from UNDEF is UNDEF.
104 if (Vec.getOpcode() == ISD::UNDEF)
105 return DAG.getUNDEF(ResultVT);
107 // Extract the relevant vectorWidth bits. Generate an EXTRACT_SUBVECTOR
108 unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
110 // This is the index of the first element of the vectorWidth-bit chunk
112 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / vectorWidth)
115 // If the input is a buildvector just emit a smaller one.
116 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
117 return DAG.getNode(ISD::BUILD_VECTOR, dl, ResultVT,
118 makeArrayRef(Vec->op_begin() + NormalizedIdxVal,
121 SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
122 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
125 /// Generate a DAG to grab 128-bits from a vector > 128 bits. This
126 /// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
127 /// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
128 /// instructions or a simple subregister reference. Idx is an index in the
129 /// 128 bits we want. It need not be aligned to a 128-bit boundary. That makes
130 /// lowering EXTRACT_VECTOR_ELT operations easier.
131 static SDValue Extract128BitVector(SDValue Vec, unsigned IdxVal,
132 SelectionDAG &DAG, SDLoc dl) {
133 assert((Vec.getValueType().is256BitVector() ||
134 Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
135 return ExtractSubVector(Vec, IdxVal, DAG, dl, 128);
138 /// Generate a DAG to grab 256-bits from a 512-bit vector.
139 static SDValue Extract256BitVector(SDValue Vec, unsigned IdxVal,
140 SelectionDAG &DAG, SDLoc dl) {
141 assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
142 return ExtractSubVector(Vec, IdxVal, DAG, dl, 256);
145 static SDValue InsertSubVector(SDValue Result, SDValue Vec,
146 unsigned IdxVal, SelectionDAG &DAG,
147 SDLoc dl, unsigned vectorWidth) {
148 assert((vectorWidth == 128 || vectorWidth == 256) &&
149 "Unsupported vector width");
150 // Inserting UNDEF is Result
151 if (Vec.getOpcode() == ISD::UNDEF)
153 EVT VT = Vec.getValueType();
154 EVT ElVT = VT.getVectorElementType();
155 EVT ResultVT = Result.getValueType();
157 // Insert the relevant vectorWidth bits.
158 unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
160 // This is the index of the first element of the vectorWidth-bit chunk
162 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/vectorWidth)
165 SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
166 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
169 /// Generate a DAG to put 128-bits into a vector > 128 bits. This
170 /// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
171 /// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
172 /// simple superregister reference. Idx is an index in the 128 bits
173 /// we want. It need not be aligned to a 128-bit boundary. That makes
174 /// lowering INSERT_VECTOR_ELT operations easier.
175 static SDValue Insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
176 SelectionDAG &DAG,SDLoc dl) {
177 assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
178 return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
181 static SDValue Insert256BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
182 SelectionDAG &DAG, SDLoc dl) {
183 assert(Vec.getValueType().is256BitVector() && "Unexpected vector size!");
184 return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 256);
187 /// Concat two 128-bit vectors into a 256 bit vector using VINSERTF128
188 /// instructions. This is used because creating CONCAT_VECTOR nodes of
189 /// BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower
190 /// large BUILD_VECTORS.
191 static SDValue Concat128BitVectors(SDValue V1, SDValue V2, EVT VT,
192 unsigned NumElems, SelectionDAG &DAG,
194 SDValue V = Insert128BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
195 return Insert128BitVector(V, V2, NumElems/2, DAG, dl);
198 static SDValue Concat256BitVectors(SDValue V1, SDValue V2, EVT VT,
199 unsigned NumElems, SelectionDAG &DAG,
201 SDValue V = Insert256BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
202 return Insert256BitVector(V, V2, NumElems/2, DAG, dl);
205 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
206 const X86Subtarget &STI)
207 : TargetLowering(TM), Subtarget(&STI) {
208 X86ScalarSSEf64 = Subtarget->hasSSE2();
209 X86ScalarSSEf32 = Subtarget->hasSSE1();
210 TD = getDataLayout();
212 // Set up the TargetLowering object.
213 static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 };
215 // X86 is weird. It always uses i8 for shift amounts and setcc results.
216 setBooleanContents(ZeroOrOneBooleanContent);
217 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
218 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
220 // For 64-bit, since we have so many registers, use the ILP scheduler.
221 // For 32-bit, use the register pressure specific scheduling.
222 // For Atom, always use ILP scheduling.
223 if (Subtarget->isAtom())
224 setSchedulingPreference(Sched::ILP);
225 else if (Subtarget->is64Bit())
226 setSchedulingPreference(Sched::ILP);
228 setSchedulingPreference(Sched::RegPressure);
229 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
230 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
232 // Bypass expensive divides on Atom when compiling with O2.
233 if (TM.getOptLevel() >= CodeGenOpt::Default) {
234 if (Subtarget->hasSlowDivide32())
235 addBypassSlowDiv(32, 8);
236 if (Subtarget->hasSlowDivide64() && Subtarget->is64Bit())
237 addBypassSlowDiv(64, 16);
240 if (Subtarget->isTargetKnownWindowsMSVC()) {
241 // Setup Windows compiler runtime calls.
242 setLibcallName(RTLIB::SDIV_I64, "_alldiv");
243 setLibcallName(RTLIB::UDIV_I64, "_aulldiv");
244 setLibcallName(RTLIB::SREM_I64, "_allrem");
245 setLibcallName(RTLIB::UREM_I64, "_aullrem");
246 setLibcallName(RTLIB::MUL_I64, "_allmul");
247 setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall);
248 setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall);
249 setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall);
250 setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall);
251 setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall);
253 // The _ftol2 runtime function has an unusual calling conv, which
254 // is modeled by a special pseudo-instruction.
255 setLibcallName(RTLIB::FPTOUINT_F64_I64, nullptr);
256 setLibcallName(RTLIB::FPTOUINT_F32_I64, nullptr);
257 setLibcallName(RTLIB::FPTOUINT_F64_I32, nullptr);
258 setLibcallName(RTLIB::FPTOUINT_F32_I32, nullptr);
261 if (Subtarget->isTargetDarwin()) {
262 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
263 setUseUnderscoreSetJmp(false);
264 setUseUnderscoreLongJmp(false);
265 } else if (Subtarget->isTargetWindowsGNU()) {
266 // MS runtime is weird: it exports _setjmp, but longjmp!
267 setUseUnderscoreSetJmp(true);
268 setUseUnderscoreLongJmp(false);
270 setUseUnderscoreSetJmp(true);
271 setUseUnderscoreLongJmp(true);
274 // Set up the register classes.
275 addRegisterClass(MVT::i8, &X86::GR8RegClass);
276 addRegisterClass(MVT::i16, &X86::GR16RegClass);
277 addRegisterClass(MVT::i32, &X86::GR32RegClass);
278 if (Subtarget->is64Bit())
279 addRegisterClass(MVT::i64, &X86::GR64RegClass);
281 for (MVT VT : MVT::integer_valuetypes())
282 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
284 // We don't accept any truncstore of integer registers.
285 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
286 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
287 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
288 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
289 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
290 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
292 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
294 // SETOEQ and SETUNE require checking two conditions.
295 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
296 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
297 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand);
298 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand);
299 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
300 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand);
302 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
304 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
305 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
306 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
308 if (Subtarget->is64Bit()) {
309 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
310 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
311 } else if (!TM.Options.UseSoftFloat) {
312 // We have an algorithm for SSE2->double, and we turn this into a
313 // 64-bit FILD followed by conditional FADD for other targets.
314 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
315 // We have an algorithm for SSE2, and we turn this into a 64-bit
316 // FILD for other targets.
317 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom);
320 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
322 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
323 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
325 if (!TM.Options.UseSoftFloat) {
326 // SSE has no i16 to fp conversion, only i32
327 if (X86ScalarSSEf32) {
328 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
329 // f32 and f64 cases are Legal, f80 case is not
330 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
332 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
333 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
336 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
337 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote);
340 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
341 // are Legal, f80 is custom lowered.
342 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
343 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
345 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
347 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
348 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
350 if (X86ScalarSSEf32) {
351 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
352 // f32 and f64 cases are Legal, f80 case is not
353 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
355 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
356 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
359 // Handle FP_TO_UINT by promoting the destination to a larger signed
361 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
362 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
363 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
365 if (Subtarget->is64Bit()) {
366 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand);
367 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
368 } else if (!TM.Options.UseSoftFloat) {
369 // Since AVX is a superset of SSE3, only check for SSE here.
370 if (Subtarget->hasSSE1() && !Subtarget->hasSSE3())
371 // Expand FP_TO_UINT into a select.
372 // FIXME: We would like to use a Custom expander here eventually to do
373 // the optimal thing for SSE vs. the default expansion in the legalizer.
374 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
376 // With SSE3 we can use fisttpll to convert to a signed i64; without
377 // SSE, we're stuck with a fistpll.
378 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom);
381 if (isTargetFTOL()) {
382 // Use the _ftol2 runtime function, which has a pseudo-instruction
383 // to handle its weird calling convention.
384 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom);
387 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
388 if (!X86ScalarSSEf64) {
389 setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
390 setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
391 if (Subtarget->is64Bit()) {
392 setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
393 // Without SSE, i64->f64 goes through memory.
394 setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
398 // Scalar integer divide and remainder are lowered to use operations that
399 // produce two results, to match the available instructions. This exposes
400 // the two-result form to trivial CSE, which is able to combine x/y and x%y
401 // into a single instruction.
403 // Scalar integer multiply-high is also lowered to use two-result
404 // operations, to match the available instructions. However, plain multiply
405 // (low) operations are left as Legal, as there are single-result
406 // instructions for this in x86. Using the two-result multiply instructions
407 // when both high and low results are needed must be arranged by dagcombine.
408 for (unsigned i = 0; i != array_lengthof(IntVTs); ++i) {
410 setOperationAction(ISD::MULHS, VT, Expand);
411 setOperationAction(ISD::MULHU, VT, Expand);
412 setOperationAction(ISD::SDIV, VT, Expand);
413 setOperationAction(ISD::UDIV, VT, Expand);
414 setOperationAction(ISD::SREM, VT, Expand);
415 setOperationAction(ISD::UREM, VT, Expand);
417 // Add/Sub overflow ops with MVT::Glues are lowered to EFLAGS dependences.
418 setOperationAction(ISD::ADDC, VT, Custom);
419 setOperationAction(ISD::ADDE, VT, Custom);
420 setOperationAction(ISD::SUBC, VT, Custom);
421 setOperationAction(ISD::SUBE, VT, Custom);
424 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
425 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
426 setOperationAction(ISD::BR_CC , MVT::f32, Expand);
427 setOperationAction(ISD::BR_CC , MVT::f64, Expand);
428 setOperationAction(ISD::BR_CC , MVT::f80, Expand);
429 setOperationAction(ISD::BR_CC , MVT::i8, Expand);
430 setOperationAction(ISD::BR_CC , MVT::i16, Expand);
431 setOperationAction(ISD::BR_CC , MVT::i32, Expand);
432 setOperationAction(ISD::BR_CC , MVT::i64, Expand);
433 setOperationAction(ISD::SELECT_CC , MVT::f32, Expand);
434 setOperationAction(ISD::SELECT_CC , MVT::f64, Expand);
435 setOperationAction(ISD::SELECT_CC , MVT::f80, Expand);
436 setOperationAction(ISD::SELECT_CC , MVT::i8, Expand);
437 setOperationAction(ISD::SELECT_CC , MVT::i16, Expand);
438 setOperationAction(ISD::SELECT_CC , MVT::i32, Expand);
439 setOperationAction(ISD::SELECT_CC , MVT::i64, Expand);
440 if (Subtarget->is64Bit())
441 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
442 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
443 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
444 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
445 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
446 setOperationAction(ISD::FREM , MVT::f32 , Expand);
447 setOperationAction(ISD::FREM , MVT::f64 , Expand);
448 setOperationAction(ISD::FREM , MVT::f80 , Expand);
449 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom);
451 // Promote the i8 variants and force them on up to i32 which has a shorter
453 setOperationAction(ISD::CTTZ , MVT::i8 , Promote);
454 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32);
455 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote);
456 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32);
457 if (Subtarget->hasBMI()) {
458 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand);
459 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand);
460 if (Subtarget->is64Bit())
461 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
463 setOperationAction(ISD::CTTZ , MVT::i16 , Custom);
464 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
465 if (Subtarget->is64Bit())
466 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
469 if (Subtarget->hasLZCNT()) {
470 // When promoting the i8 variants, force them to i32 for a shorter
472 setOperationAction(ISD::CTLZ , MVT::i8 , Promote);
473 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32);
474 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote);
475 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
476 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand);
477 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand);
478 if (Subtarget->is64Bit())
479 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand);
481 setOperationAction(ISD::CTLZ , MVT::i8 , Custom);
482 setOperationAction(ISD::CTLZ , MVT::i16 , Custom);
483 setOperationAction(ISD::CTLZ , MVT::i32 , Custom);
484 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom);
485 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom);
486 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom);
487 if (Subtarget->is64Bit()) {
488 setOperationAction(ISD::CTLZ , MVT::i64 , Custom);
489 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);
493 // Special handling for half-precision floating point conversions.
494 // If we don't have F16C support, then lower half float conversions
495 // into library calls.
496 if (TM.Options.UseSoftFloat || !Subtarget->hasF16C()) {
497 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
498 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
501 // There's never any support for operations beyond MVT::f32.
502 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
503 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand);
504 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
505 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand);
507 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
508 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
509 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand);
510 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
511 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
512 setTruncStoreAction(MVT::f80, MVT::f16, Expand);
514 if (Subtarget->hasPOPCNT()) {
515 setOperationAction(ISD::CTPOP , MVT::i8 , Promote);
517 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
518 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
519 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
520 if (Subtarget->is64Bit())
521 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
524 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
526 if (!Subtarget->hasMOVBE())
527 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
529 // These should be promoted to a larger select which is supported.
530 setOperationAction(ISD::SELECT , MVT::i1 , Promote);
531 // X86 wants to expand cmov itself.
532 setOperationAction(ISD::SELECT , MVT::i8 , Custom);
533 setOperationAction(ISD::SELECT , MVT::i16 , Custom);
534 setOperationAction(ISD::SELECT , MVT::i32 , Custom);
535 setOperationAction(ISD::SELECT , MVT::f32 , Custom);
536 setOperationAction(ISD::SELECT , MVT::f64 , Custom);
537 setOperationAction(ISD::SELECT , MVT::f80 , Custom);
538 setOperationAction(ISD::SETCC , MVT::i8 , Custom);
539 setOperationAction(ISD::SETCC , MVT::i16 , Custom);
540 setOperationAction(ISD::SETCC , MVT::i32 , Custom);
541 setOperationAction(ISD::SETCC , MVT::f32 , Custom);
542 setOperationAction(ISD::SETCC , MVT::f64 , Custom);
543 setOperationAction(ISD::SETCC , MVT::f80 , Custom);
544 if (Subtarget->is64Bit()) {
545 setOperationAction(ISD::SELECT , MVT::i64 , Custom);
546 setOperationAction(ISD::SETCC , MVT::i64 , Custom);
548 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
549 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
550 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
551 // support continuation, user-level threading, and etc.. As a result, no
552 // other SjLj exception interfaces are implemented and please don't build
553 // your own exception handling based on them.
554 // LLVM/Clang supports zero-cost DWARF exception handling.
555 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
556 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
559 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom);
560 setOperationAction(ISD::JumpTable , MVT::i32 , Custom);
561 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom);
562 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom);
563 if (Subtarget->is64Bit())
564 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
565 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom);
566 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom);
567 if (Subtarget->is64Bit()) {
568 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom);
569 setOperationAction(ISD::JumpTable , MVT::i64 , Custom);
570 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom);
571 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom);
572 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom);
574 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
575 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom);
576 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom);
577 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom);
578 if (Subtarget->is64Bit()) {
579 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom);
580 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom);
581 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom);
584 if (Subtarget->hasSSE1())
585 setOperationAction(ISD::PREFETCH , MVT::Other, Legal);
587 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
589 // Expand certain atomics
590 for (unsigned i = 0; i != array_lengthof(IntVTs); ++i) {
592 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
593 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
594 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
597 if (Subtarget->hasCmpxchg16b()) {
598 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
601 // FIXME - use subtarget debug flags
602 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetELF() &&
603 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWin64()) {
604 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
607 if (Subtarget->is64Bit()) {
608 setExceptionPointerRegister(X86::RAX);
609 setExceptionSelectorRegister(X86::RDX);
611 setExceptionPointerRegister(X86::EAX);
612 setExceptionSelectorRegister(X86::EDX);
614 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
615 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
617 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
618 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
620 setOperationAction(ISD::TRAP, MVT::Other, Legal);
621 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
623 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
624 setOperationAction(ISD::VASTART , MVT::Other, Custom);
625 setOperationAction(ISD::VAEND , MVT::Other, Expand);
626 if (Subtarget->is64Bit() && !Subtarget->isTargetWin64()) {
627 // TargetInfo::X86_64ABIBuiltinVaList
628 setOperationAction(ISD::VAARG , MVT::Other, Custom);
629 setOperationAction(ISD::VACOPY , MVT::Other, Custom);
631 // TargetInfo::CharPtrBuiltinVaList
632 setOperationAction(ISD::VAARG , MVT::Other, Expand);
633 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
636 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
637 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
639 setOperationAction(ISD::DYNAMIC_STACKALLOC, getPointerTy(), Custom);
641 if (!TM.Options.UseSoftFloat && X86ScalarSSEf64) {
642 // f32 and f64 use SSE.
643 // Set up the FP register classes.
644 addRegisterClass(MVT::f32, &X86::FR32RegClass);
645 addRegisterClass(MVT::f64, &X86::FR64RegClass);
647 // Use ANDPD to simulate FABS.
648 setOperationAction(ISD::FABS , MVT::f64, Custom);
649 setOperationAction(ISD::FABS , MVT::f32, Custom);
651 // Use XORP to simulate FNEG.
652 setOperationAction(ISD::FNEG , MVT::f64, Custom);
653 setOperationAction(ISD::FNEG , MVT::f32, Custom);
655 // Use ANDPD and ORPD to simulate FCOPYSIGN.
656 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
657 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
659 // Lower this to FGETSIGNx86 plus an AND.
660 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
661 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
663 // We don't support sin/cos/fmod
664 setOperationAction(ISD::FSIN , MVT::f64, Expand);
665 setOperationAction(ISD::FCOS , MVT::f64, Expand);
666 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
667 setOperationAction(ISD::FSIN , MVT::f32, Expand);
668 setOperationAction(ISD::FCOS , MVT::f32, Expand);
669 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
671 // Expand FP immediates into loads from the stack, except for the special
673 addLegalFPImmediate(APFloat(+0.0)); // xorpd
674 addLegalFPImmediate(APFloat(+0.0f)); // xorps
675 } else if (!TM.Options.UseSoftFloat && X86ScalarSSEf32) {
676 // Use SSE for f32, x87 for f64.
677 // Set up the FP register classes.
678 addRegisterClass(MVT::f32, &X86::FR32RegClass);
679 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
681 // Use ANDPS to simulate FABS.
682 setOperationAction(ISD::FABS , MVT::f32, Custom);
684 // Use XORP to simulate FNEG.
685 setOperationAction(ISD::FNEG , MVT::f32, Custom);
687 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
689 // Use ANDPS and ORPS to simulate FCOPYSIGN.
690 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
691 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
693 // We don't support sin/cos/fmod
694 setOperationAction(ISD::FSIN , MVT::f32, Expand);
695 setOperationAction(ISD::FCOS , MVT::f32, Expand);
696 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
698 // Special cases we handle for FP constants.
699 addLegalFPImmediate(APFloat(+0.0f)); // xorps
700 addLegalFPImmediate(APFloat(+0.0)); // FLD0
701 addLegalFPImmediate(APFloat(+1.0)); // FLD1
702 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
703 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
705 if (!TM.Options.UnsafeFPMath) {
706 setOperationAction(ISD::FSIN , MVT::f64, Expand);
707 setOperationAction(ISD::FCOS , MVT::f64, Expand);
708 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
710 } else if (!TM.Options.UseSoftFloat) {
711 // f32 and f64 in x87.
712 // Set up the FP register classes.
713 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
714 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
716 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
717 setOperationAction(ISD::UNDEF, MVT::f32, Expand);
718 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
719 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
721 if (!TM.Options.UnsafeFPMath) {
722 setOperationAction(ISD::FSIN , MVT::f64, Expand);
723 setOperationAction(ISD::FSIN , MVT::f32, Expand);
724 setOperationAction(ISD::FCOS , MVT::f64, Expand);
725 setOperationAction(ISD::FCOS , MVT::f32, Expand);
726 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
727 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
729 addLegalFPImmediate(APFloat(+0.0)); // FLD0
730 addLegalFPImmediate(APFloat(+1.0)); // FLD1
731 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
732 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
733 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
734 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
735 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
736 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
739 // We don't support FMA.
740 setOperationAction(ISD::FMA, MVT::f64, Expand);
741 setOperationAction(ISD::FMA, MVT::f32, Expand);
743 // Long double always uses X87.
744 if (!TM.Options.UseSoftFloat) {
745 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
746 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
747 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
749 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended);
750 addLegalFPImmediate(TmpFlt); // FLD0
752 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
755 APFloat TmpFlt2(+1.0);
756 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven,
758 addLegalFPImmediate(TmpFlt2); // FLD1
759 TmpFlt2.changeSign();
760 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
763 if (!TM.Options.UnsafeFPMath) {
764 setOperationAction(ISD::FSIN , MVT::f80, Expand);
765 setOperationAction(ISD::FCOS , MVT::f80, Expand);
766 setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
769 setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
770 setOperationAction(ISD::FCEIL, MVT::f80, Expand);
771 setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
772 setOperationAction(ISD::FRINT, MVT::f80, Expand);
773 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
774 setOperationAction(ISD::FMA, MVT::f80, Expand);
777 // Always use a library call for pow.
778 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
779 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
780 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
782 setOperationAction(ISD::FLOG, MVT::f80, Expand);
783 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
784 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
785 setOperationAction(ISD::FEXP, MVT::f80, Expand);
786 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
787 setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
788 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
790 // First set operation action for all vector types to either promote
791 // (for widening) or expand (for scalarization). Then we will selectively
792 // turn on ones that can be effectively codegen'd.
793 for (MVT VT : MVT::vector_valuetypes()) {
794 setOperationAction(ISD::ADD , VT, Expand);
795 setOperationAction(ISD::SUB , VT, Expand);
796 setOperationAction(ISD::FADD, VT, Expand);
797 setOperationAction(ISD::FNEG, VT, Expand);
798 setOperationAction(ISD::FSUB, VT, Expand);
799 setOperationAction(ISD::MUL , VT, Expand);
800 setOperationAction(ISD::FMUL, VT, Expand);
801 setOperationAction(ISD::SDIV, VT, Expand);
802 setOperationAction(ISD::UDIV, VT, Expand);
803 setOperationAction(ISD::FDIV, VT, Expand);
804 setOperationAction(ISD::SREM, VT, Expand);
805 setOperationAction(ISD::UREM, VT, Expand);
806 setOperationAction(ISD::LOAD, VT, Expand);
807 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
808 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
809 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
810 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
811 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
812 setOperationAction(ISD::FABS, VT, Expand);
813 setOperationAction(ISD::FSIN, VT, Expand);
814 setOperationAction(ISD::FSINCOS, VT, Expand);
815 setOperationAction(ISD::FCOS, VT, Expand);
816 setOperationAction(ISD::FSINCOS, VT, Expand);
817 setOperationAction(ISD::FREM, VT, Expand);
818 setOperationAction(ISD::FMA, VT, Expand);
819 setOperationAction(ISD::FPOWI, VT, Expand);
820 setOperationAction(ISD::FSQRT, VT, Expand);
821 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
822 setOperationAction(ISD::FFLOOR, VT, Expand);
823 setOperationAction(ISD::FCEIL, VT, Expand);
824 setOperationAction(ISD::FTRUNC, VT, Expand);
825 setOperationAction(ISD::FRINT, VT, Expand);
826 setOperationAction(ISD::FNEARBYINT, VT, Expand);
827 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
828 setOperationAction(ISD::MULHS, VT, Expand);
829 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
830 setOperationAction(ISD::MULHU, VT, Expand);
831 setOperationAction(ISD::SDIVREM, VT, Expand);
832 setOperationAction(ISD::UDIVREM, VT, Expand);
833 setOperationAction(ISD::FPOW, VT, Expand);
834 setOperationAction(ISD::CTPOP, VT, Expand);
835 setOperationAction(ISD::CTTZ, VT, Expand);
836 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
837 setOperationAction(ISD::CTLZ, VT, Expand);
838 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
839 setOperationAction(ISD::SHL, VT, Expand);
840 setOperationAction(ISD::SRA, VT, Expand);
841 setOperationAction(ISD::SRL, VT, Expand);
842 setOperationAction(ISD::ROTL, VT, Expand);
843 setOperationAction(ISD::ROTR, VT, Expand);
844 setOperationAction(ISD::BSWAP, VT, Expand);
845 setOperationAction(ISD::SETCC, VT, Expand);
846 setOperationAction(ISD::FLOG, VT, Expand);
847 setOperationAction(ISD::FLOG2, VT, Expand);
848 setOperationAction(ISD::FLOG10, VT, Expand);
849 setOperationAction(ISD::FEXP, VT, Expand);
850 setOperationAction(ISD::FEXP2, VT, Expand);
851 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
852 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
853 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
854 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
855 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
856 setOperationAction(ISD::TRUNCATE, VT, Expand);
857 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
858 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
859 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
860 setOperationAction(ISD::VSELECT, VT, Expand);
861 setOperationAction(ISD::SELECT_CC, VT, Expand);
862 for (MVT InnerVT : MVT::vector_valuetypes()) {
863 setTruncStoreAction(InnerVT, VT, Expand);
865 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
866 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
868 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
869 // types, we have to deal with them whether we ask for Expansion or not.
870 // Setting Expand causes its own optimisation problems though, so leave
872 if (VT.getVectorElementType() == MVT::i1)
873 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
877 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
878 // with -msoft-float, disable use of MMX as well.
879 if (!TM.Options.UseSoftFloat && Subtarget->hasMMX()) {
880 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
881 // No operations on x86mmx supported, everything uses intrinsics.
884 // MMX-sized vectors (other than x86mmx) are expected to be expanded
885 // into smaller operations.
886 setOperationAction(ISD::MULHS, MVT::v8i8, Expand);
887 setOperationAction(ISD::MULHS, MVT::v4i16, Expand);
888 setOperationAction(ISD::MULHS, MVT::v2i32, Expand);
889 setOperationAction(ISD::MULHS, MVT::v1i64, Expand);
890 setOperationAction(ISD::AND, MVT::v8i8, Expand);
891 setOperationAction(ISD::AND, MVT::v4i16, Expand);
892 setOperationAction(ISD::AND, MVT::v2i32, Expand);
893 setOperationAction(ISD::AND, MVT::v1i64, Expand);
894 setOperationAction(ISD::OR, MVT::v8i8, Expand);
895 setOperationAction(ISD::OR, MVT::v4i16, Expand);
896 setOperationAction(ISD::OR, MVT::v2i32, Expand);
897 setOperationAction(ISD::OR, MVT::v1i64, Expand);
898 setOperationAction(ISD::XOR, MVT::v8i8, Expand);
899 setOperationAction(ISD::XOR, MVT::v4i16, Expand);
900 setOperationAction(ISD::XOR, MVT::v2i32, Expand);
901 setOperationAction(ISD::XOR, MVT::v1i64, Expand);
902 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Expand);
903 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Expand);
904 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i32, Expand);
905 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Expand);
906 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand);
907 setOperationAction(ISD::SELECT, MVT::v8i8, Expand);
908 setOperationAction(ISD::SELECT, MVT::v4i16, Expand);
909 setOperationAction(ISD::SELECT, MVT::v2i32, Expand);
910 setOperationAction(ISD::SELECT, MVT::v1i64, Expand);
911 setOperationAction(ISD::BITCAST, MVT::v8i8, Expand);
912 setOperationAction(ISD::BITCAST, MVT::v4i16, Expand);
913 setOperationAction(ISD::BITCAST, MVT::v2i32, Expand);
914 setOperationAction(ISD::BITCAST, MVT::v1i64, Expand);
916 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE1()) {
917 addRegisterClass(MVT::v4f32, &X86::VR128RegClass);
919 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
920 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
921 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
922 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
923 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
924 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
925 setOperationAction(ISD::FABS, MVT::v4f32, Custom);
926 setOperationAction(ISD::LOAD, MVT::v4f32, Legal);
927 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
928 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
929 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
930 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
931 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
934 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE2()) {
935 addRegisterClass(MVT::v2f64, &X86::VR128RegClass);
937 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
938 // registers cannot be used even for integer operations.
939 addRegisterClass(MVT::v16i8, &X86::VR128RegClass);
940 addRegisterClass(MVT::v8i16, &X86::VR128RegClass);
941 addRegisterClass(MVT::v4i32, &X86::VR128RegClass);
942 addRegisterClass(MVT::v2i64, &X86::VR128RegClass);
944 setOperationAction(ISD::ADD, MVT::v16i8, Legal);
945 setOperationAction(ISD::ADD, MVT::v8i16, Legal);
946 setOperationAction(ISD::ADD, MVT::v4i32, Legal);
947 setOperationAction(ISD::ADD, MVT::v2i64, Legal);
948 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
949 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
950 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom);
951 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom);
952 setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
953 setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
954 setOperationAction(ISD::SUB, MVT::v16i8, Legal);
955 setOperationAction(ISD::SUB, MVT::v8i16, Legal);
956 setOperationAction(ISD::SUB, MVT::v4i32, Legal);
957 setOperationAction(ISD::SUB, MVT::v2i64, Legal);
958 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
959 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
960 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
961 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
962 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
963 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
964 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
965 setOperationAction(ISD::FABS, MVT::v2f64, Custom);
967 setOperationAction(ISD::SETCC, MVT::v2i64, Custom);
968 setOperationAction(ISD::SETCC, MVT::v16i8, Custom);
969 setOperationAction(ISD::SETCC, MVT::v8i16, Custom);
970 setOperationAction(ISD::SETCC, MVT::v4i32, Custom);
972 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
973 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
974 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
975 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
976 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
978 // Only provide customized ctpop vector bit twiddling for vector types we
979 // know to perform better than using the popcnt instructions on each vector
980 // element. If popcnt isn't supported, always provide the custom version.
981 if (!Subtarget->hasPOPCNT()) {
982 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
983 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom);
986 // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
987 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) {
988 MVT VT = (MVT::SimpleValueType)i;
989 // Do not attempt to custom lower non-power-of-2 vectors
990 if (!isPowerOf2_32(VT.getVectorNumElements()))
992 // Do not attempt to custom lower non-128-bit vectors
993 if (!VT.is128BitVector())
995 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
996 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
997 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1000 // We support custom legalizing of sext and anyext loads for specific
1001 // memory vector types which we can load as a scalar (or sequence of
1002 // scalars) and extend in-register to a legal 128-bit vector type. For sext
1003 // loads these must work with a single scalar load.
1004 for (MVT VT : MVT::integer_vector_valuetypes()) {
1005 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom);
1006 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom);
1007 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom);
1008 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom);
1009 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom);
1010 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom);
1011 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom);
1012 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom);
1013 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom);
1016 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
1017 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
1018 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
1019 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
1020 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
1021 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
1023 if (Subtarget->is64Bit()) {
1024 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom);
1025 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
1028 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64.
1029 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) {
1030 MVT VT = (MVT::SimpleValueType)i;
1032 // Do not attempt to promote non-128-bit vectors
1033 if (!VT.is128BitVector())
1036 setOperationAction(ISD::AND, VT, Promote);
1037 AddPromotedToType (ISD::AND, VT, MVT::v2i64);
1038 setOperationAction(ISD::OR, VT, Promote);
1039 AddPromotedToType (ISD::OR, VT, MVT::v2i64);
1040 setOperationAction(ISD::XOR, VT, Promote);
1041 AddPromotedToType (ISD::XOR, VT, MVT::v2i64);
1042 setOperationAction(ISD::LOAD, VT, Promote);
1043 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64);
1044 setOperationAction(ISD::SELECT, VT, Promote);
1045 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64);
1048 // Custom lower v2i64 and v2f64 selects.
1049 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
1050 setOperationAction(ISD::LOAD, MVT::v2i64, Legal);
1051 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
1052 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
1054 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
1055 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
1057 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom);
1058 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1059 // As there is no 64-bit GPR available, we need build a special custom
1060 // sequence to convert from v2i32 to v2f32.
1061 if (!Subtarget->is64Bit())
1062 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
1064 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
1065 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
1067 for (MVT VT : MVT::fp_vector_valuetypes())
1068 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal);
1070 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
1071 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
1072 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
1075 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE41()) {
1076 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
1077 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
1078 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
1079 setOperationAction(ISD::FRINT, MVT::f32, Legal);
1080 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
1081 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
1082 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
1083 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
1084 setOperationAction(ISD::FRINT, MVT::f64, Legal);
1085 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
1087 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
1088 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
1089 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
1090 setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
1091 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
1092 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
1093 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
1094 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
1095 setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
1096 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
1098 // FIXME: Do we need to handle scalar-to-vector here?
1099 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1101 setOperationAction(ISD::VSELECT, MVT::v2f64, Custom);
1102 setOperationAction(ISD::VSELECT, MVT::v2i64, Custom);
1103 setOperationAction(ISD::VSELECT, MVT::v4i32, Custom);
1104 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
1105 setOperationAction(ISD::VSELECT, MVT::v8i16, Custom);
1106 // There is no BLENDI for byte vectors. We don't need to custom lower
1107 // some vselects for now.
1108 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1110 // SSE41 brings specific instructions for doing vector sign extend even in
1111 // cases where we don't have SRA.
1112 for (MVT VT : MVT::integer_vector_valuetypes()) {
1113 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom);
1114 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom);
1115 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom);
1118 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1119 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal);
1120 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal);
1121 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal);
1122 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal);
1123 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal);
1124 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal);
1126 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal);
1127 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal);
1128 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal);
1129 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal);
1130 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal);
1131 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal);
1133 // i8 and i16 vectors are custom because the source register and source
1134 // source memory operand types are not the same width. f32 vectors are
1135 // custom since the immediate controlling the insert encodes additional
1137 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1138 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
1139 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
1140 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
1142 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom);
1143 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom);
1144 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom);
1145 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
1147 // FIXME: these should be Legal, but that's only for the case where
1148 // the index is constant. For now custom expand to deal with that.
1149 if (Subtarget->is64Bit()) {
1150 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom);
1151 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
1155 if (Subtarget->hasSSE2()) {
1156 setOperationAction(ISD::SRL, MVT::v8i16, Custom);
1157 setOperationAction(ISD::SRL, MVT::v16i8, Custom);
1159 setOperationAction(ISD::SHL, MVT::v8i16, Custom);
1160 setOperationAction(ISD::SHL, MVT::v16i8, Custom);
1162 setOperationAction(ISD::SRA, MVT::v8i16, Custom);
1163 setOperationAction(ISD::SRA, MVT::v16i8, Custom);
1165 // In the customized shift lowering, the legal cases in AVX2 will be
1167 setOperationAction(ISD::SRL, MVT::v2i64, Custom);
1168 setOperationAction(ISD::SRL, MVT::v4i32, Custom);
1170 setOperationAction(ISD::SHL, MVT::v2i64, Custom);
1171 setOperationAction(ISD::SHL, MVT::v4i32, Custom);
1173 setOperationAction(ISD::SRA, MVT::v4i32, Custom);
1176 if (!TM.Options.UseSoftFloat && Subtarget->hasFp256()) {
1177 addRegisterClass(MVT::v32i8, &X86::VR256RegClass);
1178 addRegisterClass(MVT::v16i16, &X86::VR256RegClass);
1179 addRegisterClass(MVT::v8i32, &X86::VR256RegClass);
1180 addRegisterClass(MVT::v8f32, &X86::VR256RegClass);
1181 addRegisterClass(MVT::v4i64, &X86::VR256RegClass);
1182 addRegisterClass(MVT::v4f64, &X86::VR256RegClass);
1184 setOperationAction(ISD::LOAD, MVT::v8f32, Legal);
1185 setOperationAction(ISD::LOAD, MVT::v4f64, Legal);
1186 setOperationAction(ISD::LOAD, MVT::v4i64, Legal);
1188 setOperationAction(ISD::FADD, MVT::v8f32, Legal);
1189 setOperationAction(ISD::FSUB, MVT::v8f32, Legal);
1190 setOperationAction(ISD::FMUL, MVT::v8f32, Legal);
1191 setOperationAction(ISD::FDIV, MVT::v8f32, Legal);
1192 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal);
1193 setOperationAction(ISD::FFLOOR, MVT::v8f32, Legal);
1194 setOperationAction(ISD::FCEIL, MVT::v8f32, Legal);
1195 setOperationAction(ISD::FTRUNC, MVT::v8f32, Legal);
1196 setOperationAction(ISD::FRINT, MVT::v8f32, Legal);
1197 setOperationAction(ISD::FNEARBYINT, MVT::v8f32, Legal);
1198 setOperationAction(ISD::FNEG, MVT::v8f32, Custom);
1199 setOperationAction(ISD::FABS, MVT::v8f32, Custom);
1201 setOperationAction(ISD::FADD, MVT::v4f64, Legal);
1202 setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
1203 setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
1204 setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
1205 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
1206 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal);
1207 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal);
1208 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal);
1209 setOperationAction(ISD::FRINT, MVT::v4f64, Legal);
1210 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Legal);
1211 setOperationAction(ISD::FNEG, MVT::v4f64, Custom);
1212 setOperationAction(ISD::FABS, MVT::v4f64, Custom);
1214 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1215 // even though v8i16 is a legal type.
1216 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote);
1217 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote);
1218 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal);
1220 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote);
1221 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal);
1222 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal);
1224 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom);
1225 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
1227 for (MVT VT : MVT::fp_vector_valuetypes())
1228 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal);
1230 setOperationAction(ISD::SRL, MVT::v16i16, Custom);
1231 setOperationAction(ISD::SRL, MVT::v32i8, Custom);
1233 setOperationAction(ISD::SHL, MVT::v16i16, Custom);
1234 setOperationAction(ISD::SHL, MVT::v32i8, Custom);
1236 setOperationAction(ISD::SRA, MVT::v16i16, Custom);
1237 setOperationAction(ISD::SRA, MVT::v32i8, Custom);
1239 setOperationAction(ISD::SETCC, MVT::v32i8, Custom);
1240 setOperationAction(ISD::SETCC, MVT::v16i16, Custom);
1241 setOperationAction(ISD::SETCC, MVT::v8i32, Custom);
1242 setOperationAction(ISD::SETCC, MVT::v4i64, Custom);
1244 setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1245 setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1246 setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1248 setOperationAction(ISD::VSELECT, MVT::v4f64, Custom);
1249 setOperationAction(ISD::VSELECT, MVT::v4i64, Custom);
1250 setOperationAction(ISD::VSELECT, MVT::v8i32, Custom);
1251 setOperationAction(ISD::VSELECT, MVT::v8f32, Custom);
1253 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1254 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom);
1255 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
1256 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom);
1257 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom);
1258 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom);
1259 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom);
1260 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom);
1261 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom);
1262 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1263 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1264 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1266 if (Subtarget->hasFMA() || Subtarget->hasFMA4()) {
1267 setOperationAction(ISD::FMA, MVT::v8f32, Legal);
1268 setOperationAction(ISD::FMA, MVT::v4f64, Legal);
1269 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
1270 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
1271 setOperationAction(ISD::FMA, MVT::f32, Legal);
1272 setOperationAction(ISD::FMA, MVT::f64, Legal);
1275 if (Subtarget->hasInt256()) {
1276 setOperationAction(ISD::ADD, MVT::v4i64, Legal);
1277 setOperationAction(ISD::ADD, MVT::v8i32, Legal);
1278 setOperationAction(ISD::ADD, MVT::v16i16, Legal);
1279 setOperationAction(ISD::ADD, MVT::v32i8, Legal);
1281 setOperationAction(ISD::SUB, MVT::v4i64, Legal);
1282 setOperationAction(ISD::SUB, MVT::v8i32, Legal);
1283 setOperationAction(ISD::SUB, MVT::v16i16, Legal);
1284 setOperationAction(ISD::SUB, MVT::v32i8, Legal);
1286 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1287 setOperationAction(ISD::MUL, MVT::v8i32, Legal);
1288 setOperationAction(ISD::MUL, MVT::v16i16, Legal);
1289 // Don't lower v32i8 because there is no 128-bit byte mul
1291 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom);
1292 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom);
1293 setOperationAction(ISD::MULHU, MVT::v16i16, Legal);
1294 setOperationAction(ISD::MULHS, MVT::v16i16, Legal);
1296 setOperationAction(ISD::VSELECT, MVT::v16i16, Custom);
1297 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1299 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1300 // when we have a 256bit-wide blend with immediate.
1301 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1303 // Only provide customized ctpop vector bit twiddling for vector types we
1304 // know to perform better than using the popcnt instructions on each
1305 // vector element. If popcnt isn't supported, always provide the custom
1307 if (!Subtarget->hasPOPCNT())
1308 setOperationAction(ISD::CTPOP, MVT::v4i64, Custom);
1310 // Custom CTPOP always performs better on natively supported v8i32
1311 setOperationAction(ISD::CTPOP, MVT::v8i32, Custom);
1313 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1314 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
1315 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal);
1316 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal);
1317 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal);
1318 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal);
1319 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal);
1321 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
1322 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal);
1323 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal);
1324 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal);
1325 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal);
1326 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal);
1328 setOperationAction(ISD::ADD, MVT::v4i64, Custom);
1329 setOperationAction(ISD::ADD, MVT::v8i32, Custom);
1330 setOperationAction(ISD::ADD, MVT::v16i16, Custom);
1331 setOperationAction(ISD::ADD, MVT::v32i8, Custom);
1333 setOperationAction(ISD::SUB, MVT::v4i64, Custom);
1334 setOperationAction(ISD::SUB, MVT::v8i32, Custom);
1335 setOperationAction(ISD::SUB, MVT::v16i16, Custom);
1336 setOperationAction(ISD::SUB, MVT::v32i8, Custom);
1338 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1339 setOperationAction(ISD::MUL, MVT::v8i32, Custom);
1340 setOperationAction(ISD::MUL, MVT::v16i16, Custom);
1341 // Don't lower v32i8 because there is no 128-bit byte mul
1344 // In the customized shift lowering, the legal cases in AVX2 will be
1346 setOperationAction(ISD::SRL, MVT::v4i64, Custom);
1347 setOperationAction(ISD::SRL, MVT::v8i32, Custom);
1349 setOperationAction(ISD::SHL, MVT::v4i64, Custom);
1350 setOperationAction(ISD::SHL, MVT::v8i32, Custom);
1352 setOperationAction(ISD::SRA, MVT::v8i32, Custom);
1354 // Custom lower several nodes for 256-bit types.
1355 for (MVT VT : MVT::vector_valuetypes()) {
1356 if (VT.getScalarSizeInBits() >= 32) {
1357 setOperationAction(ISD::MLOAD, VT, Legal);
1358 setOperationAction(ISD::MSTORE, VT, Legal);
1360 // Extract subvector is special because the value type
1361 // (result) is 128-bit but the source is 256-bit wide.
1362 if (VT.is128BitVector()) {
1363 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1365 // Do not attempt to custom lower other non-256-bit vectors
1366 if (!VT.is256BitVector())
1369 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1370 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1371 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1372 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1373 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1374 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1375 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1378 // Promote v32i8, v16i16, v8i32 select, and, or, xor to v4i64.
1379 for (int i = MVT::v32i8; i != MVT::v4i64; ++i) {
1380 MVT VT = (MVT::SimpleValueType)i;
1382 // Do not attempt to promote non-256-bit vectors
1383 if (!VT.is256BitVector())
1386 setOperationAction(ISD::AND, VT, Promote);
1387 AddPromotedToType (ISD::AND, VT, MVT::v4i64);
1388 setOperationAction(ISD::OR, VT, Promote);
1389 AddPromotedToType (ISD::OR, VT, MVT::v4i64);
1390 setOperationAction(ISD::XOR, VT, Promote);
1391 AddPromotedToType (ISD::XOR, VT, MVT::v4i64);
1392 setOperationAction(ISD::LOAD, VT, Promote);
1393 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64);
1394 setOperationAction(ISD::SELECT, VT, Promote);
1395 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64);
1399 if (!TM.Options.UseSoftFloat && Subtarget->hasAVX512()) {
1400 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1401 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1402 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1403 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1405 addRegisterClass(MVT::i1, &X86::VK1RegClass);
1406 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1407 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1409 for (MVT VT : MVT::fp_vector_valuetypes())
1410 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal);
1412 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
1413 setOperationAction(ISD::SETCC, MVT::i1, Custom);
1414 setOperationAction(ISD::XOR, MVT::i1, Legal);
1415 setOperationAction(ISD::OR, MVT::i1, Legal);
1416 setOperationAction(ISD::AND, MVT::i1, Legal);
1417 setOperationAction(ISD::LOAD, MVT::v16f32, Legal);
1418 setOperationAction(ISD::LOAD, MVT::v8f64, Legal);
1419 setOperationAction(ISD::LOAD, MVT::v8i64, Legal);
1420 setOperationAction(ISD::LOAD, MVT::v16i32, Legal);
1421 setOperationAction(ISD::LOAD, MVT::v16i1, Legal);
1423 setOperationAction(ISD::FADD, MVT::v16f32, Legal);
1424 setOperationAction(ISD::FSUB, MVT::v16f32, Legal);
1425 setOperationAction(ISD::FMUL, MVT::v16f32, Legal);
1426 setOperationAction(ISD::FDIV, MVT::v16f32, Legal);
1427 setOperationAction(ISD::FSQRT, MVT::v16f32, Legal);
1428 setOperationAction(ISD::FNEG, MVT::v16f32, Custom);
1430 setOperationAction(ISD::FADD, MVT::v8f64, Legal);
1431 setOperationAction(ISD::FSUB, MVT::v8f64, Legal);
1432 setOperationAction(ISD::FMUL, MVT::v8f64, Legal);
1433 setOperationAction(ISD::FDIV, MVT::v8f64, Legal);
1434 setOperationAction(ISD::FSQRT, MVT::v8f64, Legal);
1435 setOperationAction(ISD::FNEG, MVT::v8f64, Custom);
1436 setOperationAction(ISD::FMA, MVT::v8f64, Legal);
1437 setOperationAction(ISD::FMA, MVT::v16f32, Legal);
1439 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
1440 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
1441 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
1442 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
1443 if (Subtarget->is64Bit()) {
1444 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Legal);
1445 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Legal);
1446 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Legal);
1447 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Legal);
1449 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal);
1450 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal);
1451 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal);
1452 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
1453 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal);
1454 setOperationAction(ISD::SINT_TO_FP, MVT::v8i1, Custom);
1455 setOperationAction(ISD::SINT_TO_FP, MVT::v16i1, Custom);
1456 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Promote);
1457 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Promote);
1458 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal);
1459 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal);
1460 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
1461 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal);
1462 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal);
1464 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom);
1465 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1466 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1467 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom);
1468 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom);
1469 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1470 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1471 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1472 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1473 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1474 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom);
1475 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom);
1476 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
1478 setOperationAction(ISD::FFLOOR, MVT::v16f32, Legal);
1479 setOperationAction(ISD::FFLOOR, MVT::v8f64, Legal);
1480 setOperationAction(ISD::FCEIL, MVT::v16f32, Legal);
1481 setOperationAction(ISD::FCEIL, MVT::v8f64, Legal);
1482 setOperationAction(ISD::FTRUNC, MVT::v16f32, Legal);
1483 setOperationAction(ISD::FTRUNC, MVT::v8f64, Legal);
1484 setOperationAction(ISD::FRINT, MVT::v16f32, Legal);
1485 setOperationAction(ISD::FRINT, MVT::v8f64, Legal);
1486 setOperationAction(ISD::FNEARBYINT, MVT::v16f32, Legal);
1487 setOperationAction(ISD::FNEARBYINT, MVT::v8f64, Legal);
1489 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom);
1490 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom);
1491 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom);
1492 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom);
1493 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom);
1494 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Legal);
1496 setOperationAction(ISD::SETCC, MVT::v16i1, Custom);
1497 setOperationAction(ISD::SETCC, MVT::v8i1, Custom);
1499 setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1501 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom);
1502 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom);
1503 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom);
1504 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom);
1505 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom);
1506 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom);
1507 setOperationAction(ISD::SELECT, MVT::v8f64, Custom);
1508 setOperationAction(ISD::SELECT, MVT::v8i64, Custom);
1509 setOperationAction(ISD::SELECT, MVT::v16f32, Custom);
1511 setOperationAction(ISD::ADD, MVT::v8i64, Legal);
1512 setOperationAction(ISD::ADD, MVT::v16i32, Legal);
1514 setOperationAction(ISD::SUB, MVT::v8i64, Legal);
1515 setOperationAction(ISD::SUB, MVT::v16i32, Legal);
1517 setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1519 setOperationAction(ISD::SRL, MVT::v8i64, Custom);
1520 setOperationAction(ISD::SRL, MVT::v16i32, Custom);
1522 setOperationAction(ISD::SHL, MVT::v8i64, Custom);
1523 setOperationAction(ISD::SHL, MVT::v16i32, Custom);
1525 setOperationAction(ISD::SRA, MVT::v8i64, Custom);
1526 setOperationAction(ISD::SRA, MVT::v16i32, Custom);
1528 setOperationAction(ISD::AND, MVT::v8i64, Legal);
1529 setOperationAction(ISD::OR, MVT::v8i64, Legal);
1530 setOperationAction(ISD::XOR, MVT::v8i64, Legal);
1531 setOperationAction(ISD::AND, MVT::v16i32, Legal);
1532 setOperationAction(ISD::OR, MVT::v16i32, Legal);
1533 setOperationAction(ISD::XOR, MVT::v16i32, Legal);
1535 if (Subtarget->hasCDI()) {
1536 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal);
1537 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal);
1540 // Custom lower several nodes.
1541 for (MVT VT : MVT::vector_valuetypes()) {
1542 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
1543 // Extract subvector is special because the value type
1544 // (result) is 256/128-bit but the source is 512-bit wide.
1545 if (VT.is128BitVector() || VT.is256BitVector()) {
1546 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1548 if (VT.getVectorElementType() == MVT::i1)
1549 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1551 // Do not attempt to custom lower other non-512-bit vectors
1552 if (!VT.is512BitVector())
1555 if ( EltSize >= 32) {
1556 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1557 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1558 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1559 setOperationAction(ISD::VSELECT, VT, Legal);
1560 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1561 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1562 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1563 setOperationAction(ISD::MLOAD, VT, Legal);
1564 setOperationAction(ISD::MSTORE, VT, Legal);
1567 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) {
1568 MVT VT = (MVT::SimpleValueType)i;
1570 // Do not attempt to promote non-512-bit vectors.
1571 if (!VT.is512BitVector())
1574 setOperationAction(ISD::SELECT, VT, Promote);
1575 AddPromotedToType (ISD::SELECT, VT, MVT::v8i64);
1579 if (!TM.Options.UseSoftFloat && Subtarget->hasBWI()) {
1580 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1581 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1583 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
1584 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
1586 setOperationAction(ISD::LOAD, MVT::v32i16, Legal);
1587 setOperationAction(ISD::LOAD, MVT::v64i8, Legal);
1588 setOperationAction(ISD::SETCC, MVT::v32i1, Custom);
1589 setOperationAction(ISD::SETCC, MVT::v64i1, Custom);
1590 setOperationAction(ISD::ADD, MVT::v32i16, Legal);
1591 setOperationAction(ISD::ADD, MVT::v64i8, Legal);
1592 setOperationAction(ISD::SUB, MVT::v32i16, Legal);
1593 setOperationAction(ISD::SUB, MVT::v64i8, Legal);
1594 setOperationAction(ISD::MUL, MVT::v32i16, Legal);
1596 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) {
1597 const MVT VT = (MVT::SimpleValueType)i;
1599 const unsigned EltSize = VT.getVectorElementType().getSizeInBits();
1601 // Do not attempt to promote non-512-bit vectors.
1602 if (!VT.is512BitVector())
1606 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1607 setOperationAction(ISD::VSELECT, VT, Legal);
1612 if (!TM.Options.UseSoftFloat && Subtarget->hasVLX()) {
1613 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1614 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1616 setOperationAction(ISD::SETCC, MVT::v4i1, Custom);
1617 setOperationAction(ISD::SETCC, MVT::v2i1, Custom);
1618 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Legal);
1620 setOperationAction(ISD::AND, MVT::v8i32, Legal);
1621 setOperationAction(ISD::OR, MVT::v8i32, Legal);
1622 setOperationAction(ISD::XOR, MVT::v8i32, Legal);
1623 setOperationAction(ISD::AND, MVT::v4i32, Legal);
1624 setOperationAction(ISD::OR, MVT::v4i32, Legal);
1625 setOperationAction(ISD::XOR, MVT::v4i32, Legal);
1628 // SIGN_EXTEND_INREGs are evaluated by the extend type. Handle the expansion
1629 // of this type with custom code.
1630 for (MVT VT : MVT::vector_valuetypes())
1631 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
1633 // We want to custom lower some of our intrinsics.
1634 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1635 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
1636 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1637 if (!Subtarget->is64Bit())
1638 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
1640 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
1641 // handle type legalization for these operations here.
1643 // FIXME: We really should do custom legalization for addition and
1644 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
1645 // than generic legalization for 64-bit multiplication-with-overflow, though.
1646 for (unsigned i = 0, e = 3+Subtarget->is64Bit(); i != e; ++i) {
1647 // Add/Sub/Mul with overflow operations are custom lowered.
1649 setOperationAction(ISD::SADDO, VT, Custom);
1650 setOperationAction(ISD::UADDO, VT, Custom);
1651 setOperationAction(ISD::SSUBO, VT, Custom);
1652 setOperationAction(ISD::USUBO, VT, Custom);
1653 setOperationAction(ISD::SMULO, VT, Custom);
1654 setOperationAction(ISD::UMULO, VT, Custom);
1658 if (!Subtarget->is64Bit()) {
1659 // These libcalls are not available in 32-bit.
1660 setLibcallName(RTLIB::SHL_I128, nullptr);
1661 setLibcallName(RTLIB::SRL_I128, nullptr);
1662 setLibcallName(RTLIB::SRA_I128, nullptr);
1665 // Combine sin / cos into one node or libcall if possible.
1666 if (Subtarget->hasSinCos()) {
1667 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
1668 setLibcallName(RTLIB::SINCOS_F64, "sincos");
1669 if (Subtarget->isTargetDarwin()) {
1670 // For MacOSX, we don't want the normal expansion of a libcall to sincos.
1671 // We want to issue a libcall to __sincos_stret to avoid memory traffic.
1672 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1673 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1677 if (Subtarget->isTargetWin64()) {
1678 setOperationAction(ISD::SDIV, MVT::i128, Custom);
1679 setOperationAction(ISD::UDIV, MVT::i128, Custom);
1680 setOperationAction(ISD::SREM, MVT::i128, Custom);
1681 setOperationAction(ISD::UREM, MVT::i128, Custom);
1682 setOperationAction(ISD::SDIVREM, MVT::i128, Custom);
1683 setOperationAction(ISD::UDIVREM, MVT::i128, Custom);
1686 // We have target-specific dag combine patterns for the following nodes:
1687 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
1688 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
1689 setTargetDAGCombine(ISD::BITCAST);
1690 setTargetDAGCombine(ISD::VSELECT);
1691 setTargetDAGCombine(ISD::SELECT);
1692 setTargetDAGCombine(ISD::SHL);
1693 setTargetDAGCombine(ISD::SRA);
1694 setTargetDAGCombine(ISD::SRL);
1695 setTargetDAGCombine(ISD::OR);
1696 setTargetDAGCombine(ISD::AND);
1697 setTargetDAGCombine(ISD::ADD);
1698 setTargetDAGCombine(ISD::FADD);
1699 setTargetDAGCombine(ISD::FSUB);
1700 setTargetDAGCombine(ISD::FMA);
1701 setTargetDAGCombine(ISD::SUB);
1702 setTargetDAGCombine(ISD::LOAD);
1703 setTargetDAGCombine(ISD::MLOAD);
1704 setTargetDAGCombine(ISD::STORE);
1705 setTargetDAGCombine(ISD::MSTORE);
1706 setTargetDAGCombine(ISD::ZERO_EXTEND);
1707 setTargetDAGCombine(ISD::ANY_EXTEND);
1708 setTargetDAGCombine(ISD::SIGN_EXTEND);
1709 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1710 setTargetDAGCombine(ISD::TRUNCATE);
1711 setTargetDAGCombine(ISD::SINT_TO_FP);
1712 setTargetDAGCombine(ISD::SETCC);
1713 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
1714 setTargetDAGCombine(ISD::BUILD_VECTOR);
1715 setTargetDAGCombine(ISD::MUL);
1716 setTargetDAGCombine(ISD::XOR);
1718 computeRegisterProperties();
1720 // On Darwin, -Os means optimize for size without hurting performance,
1721 // do not reduce the limit.
1722 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
1723 MaxStoresPerMemsetOptSize = Subtarget->isTargetDarwin() ? 16 : 8;
1724 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
1725 MaxStoresPerMemcpyOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
1726 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
1727 MaxStoresPerMemmoveOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
1728 setPrefLoopAlignment(4); // 2^4 bytes.
1730 // Predictable cmov don't hurt on atom because it's in-order.
1731 PredictableSelectIsExpensive = !Subtarget->isAtom();
1732 EnableExtLdPromotion = true;
1733 setPrefFunctionAlignment(4); // 2^4 bytes.
1735 verifyIntrinsicTables();
1738 // This has so far only been implemented for 64-bit MachO.
1739 bool X86TargetLowering::useLoadStackGuardNode() const {
1740 return Subtarget->isTargetMachO() && Subtarget->is64Bit();
1743 TargetLoweringBase::LegalizeTypeAction
1744 X86TargetLowering::getPreferredVectorAction(EVT VT) const {
1745 if (ExperimentalVectorWideningLegalization &&
1746 VT.getVectorNumElements() != 1 &&
1747 VT.getVectorElementType().getSimpleVT() != MVT::i1)
1748 return TypeWidenVector;
1750 return TargetLoweringBase::getPreferredVectorAction(VT);
1753 EVT X86TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
1755 return Subtarget->hasAVX512() ? MVT::i1: MVT::i8;
1757 const unsigned NumElts = VT.getVectorNumElements();
1758 const EVT EltVT = VT.getVectorElementType();
1759 if (VT.is512BitVector()) {
1760 if (Subtarget->hasAVX512())
1761 if (EltVT == MVT::i32 || EltVT == MVT::i64 ||
1762 EltVT == MVT::f32 || EltVT == MVT::f64)
1764 case 8: return MVT::v8i1;
1765 case 16: return MVT::v16i1;
1767 if (Subtarget->hasBWI())
1768 if (EltVT == MVT::i8 || EltVT == MVT::i16)
1770 case 32: return MVT::v32i1;
1771 case 64: return MVT::v64i1;
1775 if (VT.is256BitVector() || VT.is128BitVector()) {
1776 if (Subtarget->hasVLX())
1777 if (EltVT == MVT::i32 || EltVT == MVT::i64 ||
1778 EltVT == MVT::f32 || EltVT == MVT::f64)
1780 case 2: return MVT::v2i1;
1781 case 4: return MVT::v4i1;
1782 case 8: return MVT::v8i1;
1784 if (Subtarget->hasBWI() && Subtarget->hasVLX())
1785 if (EltVT == MVT::i8 || EltVT == MVT::i16)
1787 case 8: return MVT::v8i1;
1788 case 16: return MVT::v16i1;
1789 case 32: return MVT::v32i1;
1793 return VT.changeVectorElementTypeToInteger();
1796 /// Helper for getByValTypeAlignment to determine
1797 /// the desired ByVal argument alignment.
1798 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
1801 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1802 if (VTy->getBitWidth() == 128)
1804 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
1805 unsigned EltAlign = 0;
1806 getMaxByValAlign(ATy->getElementType(), EltAlign);
1807 if (EltAlign > MaxAlign)
1808 MaxAlign = EltAlign;
1809 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
1810 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1811 unsigned EltAlign = 0;
1812 getMaxByValAlign(STy->getElementType(i), EltAlign);
1813 if (EltAlign > MaxAlign)
1814 MaxAlign = EltAlign;
1821 /// Return the desired alignment for ByVal aggregate
1822 /// function arguments in the caller parameter area. For X86, aggregates
1823 /// that contain SSE vectors are placed at 16-byte boundaries while the rest
1824 /// are at 4-byte boundaries.
1825 unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty) const {
1826 if (Subtarget->is64Bit()) {
1827 // Max of 8 and alignment of type.
1828 unsigned TyAlign = TD->getABITypeAlignment(Ty);
1835 if (Subtarget->hasSSE1())
1836 getMaxByValAlign(Ty, Align);
1840 /// Returns the target specific optimal type for load
1841 /// and store operations as a result of memset, memcpy, and memmove
1842 /// lowering. If DstAlign is zero that means it's safe to destination
1843 /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
1844 /// means there isn't a need to check it against alignment requirement,
1845 /// probably because the source does not need to be loaded. If 'IsMemset' is
1846 /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that
1847 /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy
1848 /// source is constant so it does not need to be loaded.
1849 /// It returns EVT::Other if the type should be determined using generic
1850 /// target-independent logic.
1852 X86TargetLowering::getOptimalMemOpType(uint64_t Size,
1853 unsigned DstAlign, unsigned SrcAlign,
1854 bool IsMemset, bool ZeroMemset,
1856 MachineFunction &MF) const {
1857 const Function *F = MF.getFunction();
1858 if ((!IsMemset || ZeroMemset) &&
1859 !F->hasFnAttribute(Attribute::NoImplicitFloat)) {
1861 (Subtarget->isUnalignedMemAccessFast() ||
1862 ((DstAlign == 0 || DstAlign >= 16) &&
1863 (SrcAlign == 0 || SrcAlign >= 16)))) {
1865 if (Subtarget->hasInt256())
1867 if (Subtarget->hasFp256())
1870 if (Subtarget->hasSSE2())
1872 if (Subtarget->hasSSE1())
1874 } else if (!MemcpyStrSrc && Size >= 8 &&
1875 !Subtarget->is64Bit() &&
1876 Subtarget->hasSSE2()) {
1877 // Do not use f64 to lower memcpy if source is string constant. It's
1878 // better to use i32 to avoid the loads.
1882 if (Subtarget->is64Bit() && Size >= 8)
1887 bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
1889 return X86ScalarSSEf32;
1890 else if (VT == MVT::f64)
1891 return X86ScalarSSEf64;
1896 X86TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
1901 *Fast = Subtarget->isUnalignedMemAccessFast();
1905 /// Return the entry encoding for a jump table in the
1906 /// current function. The returned value is a member of the
1907 /// MachineJumpTableInfo::JTEntryKind enum.
1908 unsigned X86TargetLowering::getJumpTableEncoding() const {
1909 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
1911 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1912 Subtarget->isPICStyleGOT())
1913 return MachineJumpTableInfo::EK_Custom32;
1915 // Otherwise, use the normal jump table encoding heuristics.
1916 return TargetLowering::getJumpTableEncoding();
1920 X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
1921 const MachineBasicBlock *MBB,
1922 unsigned uid,MCContext &Ctx) const{
1923 assert(MBB->getParent()->getTarget().getRelocationModel() == Reloc::PIC_ &&
1924 Subtarget->isPICStyleGOT());
1925 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
1927 return MCSymbolRefExpr::Create(MBB->getSymbol(),
1928 MCSymbolRefExpr::VK_GOTOFF, Ctx);
1931 /// Returns relocation base for the given PIC jumptable.
1932 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
1933 SelectionDAG &DAG) const {
1934 if (!Subtarget->is64Bit())
1935 // This doesn't have SDLoc associated with it, but is not really the
1936 // same as a Register.
1937 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy());
1941 /// This returns the relocation base for the given PIC jumptable,
1942 /// the same as getPICJumpTableRelocBase, but as an MCExpr.
1943 const MCExpr *X86TargetLowering::
1944 getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
1945 MCContext &Ctx) const {
1946 // X86-64 uses RIP relative addressing based on the jump table label.
1947 if (Subtarget->isPICStyleRIPRel())
1948 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
1950 // Otherwise, the reference is relative to the PIC base.
1951 return MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), Ctx);
1954 // FIXME: Why this routine is here? Move to RegInfo!
1955 std::pair<const TargetRegisterClass*, uint8_t>
1956 X86TargetLowering::findRepresentativeClass(MVT VT) const{
1957 const TargetRegisterClass *RRC = nullptr;
1959 switch (VT.SimpleTy) {
1961 return TargetLowering::findRepresentativeClass(VT);
1962 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:
1963 RRC = Subtarget->is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
1966 RRC = &X86::VR64RegClass;
1968 case MVT::f32: case MVT::f64:
1969 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1970 case MVT::v4f32: case MVT::v2f64:
1971 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32:
1973 RRC = &X86::VR128RegClass;
1976 return std::make_pair(RRC, Cost);
1979 bool X86TargetLowering::getStackCookieLocation(unsigned &AddressSpace,
1980 unsigned &Offset) const {
1981 if (!Subtarget->isTargetLinux())
1984 if (Subtarget->is64Bit()) {
1985 // %fs:0x28, unless we're using a Kernel code model, in which case it's %gs:
1987 if (getTargetMachine().getCodeModel() == CodeModel::Kernel)
1999 bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
2000 unsigned DestAS) const {
2001 assert(SrcAS != DestAS && "Expected different address spaces!");
2003 return SrcAS < 256 && DestAS < 256;
2006 //===----------------------------------------------------------------------===//
2007 // Return Value Calling Convention Implementation
2008 //===----------------------------------------------------------------------===//
2010 #include "X86GenCallingConv.inc"
2013 X86TargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2014 MachineFunction &MF, bool isVarArg,
2015 const SmallVectorImpl<ISD::OutputArg> &Outs,
2016 LLVMContext &Context) const {
2017 SmallVector<CCValAssign, 16> RVLocs;
2018 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2019 return CCInfo.CheckReturn(Outs, RetCC_X86);
2022 const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {
2023 static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };
2028 X86TargetLowering::LowerReturn(SDValue Chain,
2029 CallingConv::ID CallConv, bool isVarArg,
2030 const SmallVectorImpl<ISD::OutputArg> &Outs,
2031 const SmallVectorImpl<SDValue> &OutVals,
2032 SDLoc dl, SelectionDAG &DAG) const {
2033 MachineFunction &MF = DAG.getMachineFunction();
2034 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2036 SmallVector<CCValAssign, 16> RVLocs;
2037 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
2038 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
2041 SmallVector<SDValue, 6> RetOps;
2042 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2043 // Operand #1 = Bytes To Pop
2044 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(),
2047 // Copy the result values into the output registers.
2048 for (unsigned i = 0; i != RVLocs.size(); ++i) {
2049 CCValAssign &VA = RVLocs[i];
2050 assert(VA.isRegLoc() && "Can only return in registers!");
2051 SDValue ValToCopy = OutVals[i];
2052 EVT ValVT = ValToCopy.getValueType();
2054 // Promote values to the appropriate types.
2055 if (VA.getLocInfo() == CCValAssign::SExt)
2056 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
2057 else if (VA.getLocInfo() == CCValAssign::ZExt)
2058 ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);
2059 else if (VA.getLocInfo() == CCValAssign::AExt)
2060 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
2061 else if (VA.getLocInfo() == CCValAssign::BCvt)
2062 ValToCopy = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), ValToCopy);
2064 assert(VA.getLocInfo() != CCValAssign::FPExt &&
2065 "Unexpected FP-extend for return value.");
2067 // If this is x86-64, and we disabled SSE, we can't return FP values,
2068 // or SSE or MMX vectors.
2069 if ((ValVT == MVT::f32 || ValVT == MVT::f64 ||
2070 VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) &&
2071 (Subtarget->is64Bit() && !Subtarget->hasSSE1())) {
2072 report_fatal_error("SSE register return with SSE disabled");
2074 // Likewise we can't return F64 values with SSE1 only. gcc does so, but
2075 // llvm-gcc has never done it right and no one has noticed, so this
2076 // should be OK for now.
2077 if (ValVT == MVT::f64 &&
2078 (Subtarget->is64Bit() && !Subtarget->hasSSE2()))
2079 report_fatal_error("SSE2 register return with SSE2 disabled");
2081 // Returns in ST0/ST1 are handled specially: these are pushed as operands to
2082 // the RET instruction and handled by the FP Stackifier.
2083 if (VA.getLocReg() == X86::FP0 ||
2084 VA.getLocReg() == X86::FP1) {
2085 // If this is a copy from an xmm register to ST(0), use an FPExtend to
2086 // change the value to the FP stack register class.
2087 if (isScalarFPTypeInSSEReg(VA.getValVT()))
2088 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
2089 RetOps.push_back(ValToCopy);
2090 // Don't emit a copytoreg.
2094 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
2095 // which is returned in RAX / RDX.
2096 if (Subtarget->is64Bit()) {
2097 if (ValVT == MVT::x86mmx) {
2098 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
2099 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy);
2100 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
2102 // If we don't have SSE2 available, convert to v4f32 so the generated
2103 // register is legal.
2104 if (!Subtarget->hasSSE2())
2105 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy);
2110 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag);
2111 Flag = Chain.getValue(1);
2112 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2115 // The x86-64 ABIs require that for returning structs by value we copy
2116 // the sret argument into %rax/%eax (depending on ABI) for the return.
2117 // Win32 requires us to put the sret argument to %eax as well.
2118 // We saved the argument into a virtual register in the entry block,
2119 // so now we copy the value out and into %rax/%eax.
2121 // Checking Function.hasStructRetAttr() here is insufficient because the IR
2122 // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
2123 // false, then an sret argument may be implicitly inserted in the SelDAG. In
2124 // either case FuncInfo->setSRetReturnReg() will have been called.
2125 if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
2126 assert((Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC()) &&
2127 "No need for an sret register");
2128 SDValue Val = DAG.getCopyFromReg(Chain, dl, SRetReg, getPointerTy());
2131 = (Subtarget->is64Bit() && !Subtarget->isTarget64BitILP32()) ?
2132 X86::RAX : X86::EAX;
2133 Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
2134 Flag = Chain.getValue(1);
2136 // RAX/EAX now acts like a return value.
2137 RetOps.push_back(DAG.getRegister(RetValReg, getPointerTy()));
2140 RetOps[0] = Chain; // Update chain.
2142 // Add the flag if we have it.
2144 RetOps.push_back(Flag);
2146 return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps);
2149 bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2150 if (N->getNumValues() != 1)
2152 if (!N->hasNUsesOfValue(1, 0))
2155 SDValue TCChain = Chain;
2156 SDNode *Copy = *N->use_begin();
2157 if (Copy->getOpcode() == ISD::CopyToReg) {
2158 // If the copy has a glue operand, we conservatively assume it isn't safe to
2159 // perform a tail call.
2160 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2162 TCChain = Copy->getOperand(0);
2163 } else if (Copy->getOpcode() != ISD::FP_EXTEND)
2166 bool HasRet = false;
2167 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
2169 if (UI->getOpcode() != X86ISD::RET_FLAG)
2171 // If we are returning more than one value, we can definitely
2172 // not make a tail call see PR19530
2173 if (UI->getNumOperands() > 4)
2175 if (UI->getNumOperands() == 4 &&
2176 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue)
2189 X86TargetLowering::getTypeForExtArgOrReturn(LLVMContext &Context, EVT VT,
2190 ISD::NodeType ExtendKind) const {
2192 // TODO: Is this also valid on 32-bit?
2193 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND)
2194 ReturnMVT = MVT::i8;
2196 ReturnMVT = MVT::i32;
2198 EVT MinVT = getRegisterType(Context, ReturnMVT);
2199 return VT.bitsLT(MinVT) ? MinVT : VT;
2202 /// Lower the result values of a call into the
2203 /// appropriate copies out of appropriate physical registers.
2206 X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
2207 CallingConv::ID CallConv, bool isVarArg,
2208 const SmallVectorImpl<ISD::InputArg> &Ins,
2209 SDLoc dl, SelectionDAG &DAG,
2210 SmallVectorImpl<SDValue> &InVals) const {
2212 // Assign locations to each value returned by this call.
2213 SmallVector<CCValAssign, 16> RVLocs;
2214 bool Is64Bit = Subtarget->is64Bit();
2215 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2217 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
2219 // Copy all of the result registers out of their specified physreg.
2220 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
2221 CCValAssign &VA = RVLocs[i];
2222 EVT CopyVT = VA.getValVT();
2224 // If this is x86-64, and we disabled SSE, we can't return FP values
2225 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
2226 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
2227 report_fatal_error("SSE register return with SSE disabled");
2230 // If we prefer to use the value in xmm registers, copy it out as f80 and
2231 // use a truncate to move it from fp stack reg to xmm reg.
2232 if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
2233 isScalarFPTypeInSSEReg(VA.getValVT()))
2236 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
2237 CopyVT, InFlag).getValue(1);
2238 SDValue Val = Chain.getValue(0);
2240 if (CopyVT != VA.getValVT())
2241 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
2242 // This truncation won't change the value.
2243 DAG.getIntPtrConstant(1));
2245 InFlag = Chain.getValue(2);
2246 InVals.push_back(Val);
2252 //===----------------------------------------------------------------------===//
2253 // C & StdCall & Fast Calling Convention implementation
2254 //===----------------------------------------------------------------------===//
2255 // StdCall calling convention seems to be standard for many Windows' API
2256 // routines and around. It differs from C calling convention just a little:
2257 // callee should clean up the stack, not caller. Symbols should be also
2258 // decorated in some fancy way :) It doesn't support any vector arguments.
2259 // For info on fast calling convention see Fast Calling Convention (tail call)
2260 // implementation LowerX86_32FastCCCallTo.
2262 /// CallIsStructReturn - Determines whether a call uses struct return
2264 enum StructReturnType {
2269 static StructReturnType
2270 callIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) {
2272 return NotStructReturn;
2274 const ISD::ArgFlagsTy &Flags = Outs[0].Flags;
2275 if (!Flags.isSRet())
2276 return NotStructReturn;
2277 if (Flags.isInReg())
2278 return RegStructReturn;
2279 return StackStructReturn;
2282 /// Determines whether a function uses struct return semantics.
2283 static StructReturnType
2284 argsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) {
2286 return NotStructReturn;
2288 const ISD::ArgFlagsTy &Flags = Ins[0].Flags;
2289 if (!Flags.isSRet())
2290 return NotStructReturn;
2291 if (Flags.isInReg())
2292 return RegStructReturn;
2293 return StackStructReturn;
2296 /// Make a copy of an aggregate at address specified by "Src" to address
2297 /// "Dst" with size and alignment information specified by the specific
2298 /// parameter attribute. The copy will be passed as a byval function parameter.
2300 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
2301 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
2303 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
2305 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
2306 /*isVolatile*/false, /*AlwaysInline=*/true,
2307 MachinePointerInfo(), MachinePointerInfo());
2310 /// Return true if the calling convention is one that
2311 /// supports tail call optimization.
2312 static bool IsTailCallConvention(CallingConv::ID CC) {
2313 return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
2314 CC == CallingConv::HiPE);
2317 /// \brief Return true if the calling convention is a C calling convention.
2318 static bool IsCCallConvention(CallingConv::ID CC) {
2319 return (CC == CallingConv::C || CC == CallingConv::X86_64_Win64 ||
2320 CC == CallingConv::X86_64_SysV);
2323 bool X86TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
2324 if (!CI->isTailCall() || getTargetMachine().Options.DisableTailCalls)
2328 CallingConv::ID CalleeCC = CS.getCallingConv();
2329 if (!IsTailCallConvention(CalleeCC) && !IsCCallConvention(CalleeCC))
2335 /// Return true if the function is being made into
2336 /// a tailcall target by changing its ABI.
2337 static bool FuncIsMadeTailCallSafe(CallingConv::ID CC,
2338 bool GuaranteedTailCallOpt) {
2339 return GuaranteedTailCallOpt && IsTailCallConvention(CC);
2343 X86TargetLowering::LowerMemArgument(SDValue Chain,
2344 CallingConv::ID CallConv,
2345 const SmallVectorImpl<ISD::InputArg> &Ins,
2346 SDLoc dl, SelectionDAG &DAG,
2347 const CCValAssign &VA,
2348 MachineFrameInfo *MFI,
2350 // Create the nodes corresponding to a load from this parameter slot.
2351 ISD::ArgFlagsTy Flags = Ins[i].Flags;
2352 bool AlwaysUseMutable = FuncIsMadeTailCallSafe(
2353 CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt);
2354 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
2357 // If value is passed by pointer we have address passed instead of the value
2359 if (VA.getLocInfo() == CCValAssign::Indirect)
2360 ValVT = VA.getLocVT();
2362 ValVT = VA.getValVT();
2364 // FIXME: For now, all byval parameter objects are marked mutable. This can be
2365 // changed with more analysis.
2366 // In case of tail call optimization mark all arguments mutable. Since they
2367 // could be overwritten by lowering of arguments in case of a tail call.
2368 if (Flags.isByVal()) {
2369 unsigned Bytes = Flags.getByValSize();
2370 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
2371 int FI = MFI->CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable);
2372 return DAG.getFrameIndex(FI, getPointerTy());
2374 int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
2375 VA.getLocMemOffset(), isImmutable);
2376 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
2377 return DAG.getLoad(ValVT, dl, Chain, FIN,
2378 MachinePointerInfo::getFixedStack(FI),
2379 false, false, false, 0);
2383 // FIXME: Get this from tablegen.
2384 static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
2385 const X86Subtarget *Subtarget) {
2386 assert(Subtarget->is64Bit());
2388 if (Subtarget->isCallingConvWin64(CallConv)) {
2389 static const MCPhysReg GPR64ArgRegsWin64[] = {
2390 X86::RCX, X86::RDX, X86::R8, X86::R9
2392 return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
2395 static const MCPhysReg GPR64ArgRegs64Bit[] = {
2396 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
2398 return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
2401 // FIXME: Get this from tablegen.
2402 static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
2403 CallingConv::ID CallConv,
2404 const X86Subtarget *Subtarget) {
2405 assert(Subtarget->is64Bit());
2406 if (Subtarget->isCallingConvWin64(CallConv)) {
2407 // The XMM registers which might contain var arg parameters are shadowed
2408 // in their paired GPR. So we only need to save the GPR to their home
2410 // TODO: __vectorcall will change this.
2414 const Function *Fn = MF.getFunction();
2415 bool NoImplicitFloatOps = Fn->hasFnAttribute(Attribute::NoImplicitFloat);
2416 assert(!(MF.getTarget().Options.UseSoftFloat && NoImplicitFloatOps) &&
2417 "SSE register cannot be used when SSE is disabled!");
2418 if (MF.getTarget().Options.UseSoftFloat || NoImplicitFloatOps ||
2419 !Subtarget->hasSSE1())
2420 // Kernel mode asks for SSE to be disabled, so there are no XMM argument
2424 static const MCPhysReg XMMArgRegs64Bit[] = {
2425 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2426 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
2428 return makeArrayRef(std::begin(XMMArgRegs64Bit), std::end(XMMArgRegs64Bit));
2432 X86TargetLowering::LowerFormalArguments(SDValue Chain,
2433 CallingConv::ID CallConv,
2435 const SmallVectorImpl<ISD::InputArg> &Ins,
2438 SmallVectorImpl<SDValue> &InVals)
2440 MachineFunction &MF = DAG.getMachineFunction();
2441 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2443 const Function* Fn = MF.getFunction();
2444 if (Fn->hasExternalLinkage() &&
2445 Subtarget->isTargetCygMing() &&
2446 Fn->getName() == "main")
2447 FuncInfo->setForceFramePointer(true);
2449 MachineFrameInfo *MFI = MF.getFrameInfo();
2450 bool Is64Bit = Subtarget->is64Bit();
2451 bool IsWin64 = Subtarget->isCallingConvWin64(CallConv);
2453 assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
2454 "Var args not supported with calling convention fastcc, ghc or hipe");
2456 // Assign locations to all of the incoming arguments.
2457 SmallVector<CCValAssign, 16> ArgLocs;
2458 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
2460 // Allocate shadow area for Win64
2462 CCInfo.AllocateStack(32, 8);
2464 CCInfo.AnalyzeFormalArguments(Ins, CC_X86);
2466 unsigned LastVal = ~0U;
2468 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2469 CCValAssign &VA = ArgLocs[i];
2470 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
2472 assert(VA.getValNo() != LastVal &&
2473 "Don't support value assigned to multiple locs yet");
2475 LastVal = VA.getValNo();
2477 if (VA.isRegLoc()) {
2478 EVT RegVT = VA.getLocVT();
2479 const TargetRegisterClass *RC;
2480 if (RegVT == MVT::i32)
2481 RC = &X86::GR32RegClass;
2482 else if (Is64Bit && RegVT == MVT::i64)
2483 RC = &X86::GR64RegClass;
2484 else if (RegVT == MVT::f32)
2485 RC = &X86::FR32RegClass;
2486 else if (RegVT == MVT::f64)
2487 RC = &X86::FR64RegClass;
2488 else if (RegVT.is512BitVector())
2489 RC = &X86::VR512RegClass;
2490 else if (RegVT.is256BitVector())
2491 RC = &X86::VR256RegClass;
2492 else if (RegVT.is128BitVector())
2493 RC = &X86::VR128RegClass;
2494 else if (RegVT == MVT::x86mmx)
2495 RC = &X86::VR64RegClass;
2496 else if (RegVT == MVT::i1)
2497 RC = &X86::VK1RegClass;
2498 else if (RegVT == MVT::v8i1)
2499 RC = &X86::VK8RegClass;
2500 else if (RegVT == MVT::v16i1)
2501 RC = &X86::VK16RegClass;
2502 else if (RegVT == MVT::v32i1)
2503 RC = &X86::VK32RegClass;
2504 else if (RegVT == MVT::v64i1)
2505 RC = &X86::VK64RegClass;
2507 llvm_unreachable("Unknown argument type!");
2509 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
2510 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
2512 // If this is an 8 or 16-bit value, it is really passed promoted to 32
2513 // bits. Insert an assert[sz]ext to capture this, then truncate to the
2515 if (VA.getLocInfo() == CCValAssign::SExt)
2516 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
2517 DAG.getValueType(VA.getValVT()));
2518 else if (VA.getLocInfo() == CCValAssign::ZExt)
2519 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
2520 DAG.getValueType(VA.getValVT()));
2521 else if (VA.getLocInfo() == CCValAssign::BCvt)
2522 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
2524 if (VA.isExtInLoc()) {
2525 // Handle MMX values passed in XMM regs.
2526 if (RegVT.isVector())
2527 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
2529 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
2532 assert(VA.isMemLoc());
2533 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i);
2536 // If value is passed via pointer - do a load.
2537 if (VA.getLocInfo() == CCValAssign::Indirect)
2538 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue,
2539 MachinePointerInfo(), false, false, false, 0);
2541 InVals.push_back(ArgValue);
2544 if (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC()) {
2545 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2546 // The x86-64 ABIs require that for returning structs by value we copy
2547 // the sret argument into %rax/%eax (depending on ABI) for the return.
2548 // Win32 requires us to put the sret argument to %eax as well.
2549 // Save the argument into a virtual register so that we can access it
2550 // from the return points.
2551 if (Ins[i].Flags.isSRet()) {
2552 unsigned Reg = FuncInfo->getSRetReturnReg();
2554 MVT PtrTy = getPointerTy();
2555 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
2556 FuncInfo->setSRetReturnReg(Reg);
2558 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[i]);
2559 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
2565 unsigned StackSize = CCInfo.getNextStackOffset();
2566 // Align stack specially for tail calls.
2567 if (FuncIsMadeTailCallSafe(CallConv,
2568 MF.getTarget().Options.GuaranteedTailCallOpt))
2569 StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
2571 // If the function takes variable number of arguments, make a frame index for
2572 // the start of the first vararg value... for expansion of llvm.va_start. We
2573 // can skip this if there are no va_start calls.
2574 if (MFI->hasVAStart() &&
2575 (Is64Bit || (CallConv != CallingConv::X86_FastCall &&
2576 CallConv != CallingConv::X86_ThisCall))) {
2577 FuncInfo->setVarArgsFrameIndex(
2578 MFI->CreateFixedObject(1, StackSize, true));
2581 // Figure out if XMM registers are in use.
2582 assert(!(MF.getTarget().Options.UseSoftFloat &&
2583 Fn->hasFnAttribute(Attribute::NoImplicitFloat)) &&
2584 "SSE register cannot be used when SSE is disabled!");
2586 // 64-bit calling conventions support varargs and register parameters, so we
2587 // have to do extra work to spill them in the prologue.
2588 if (Is64Bit && isVarArg && MFI->hasVAStart()) {
2589 // Find the first unallocated argument registers.
2590 ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);
2591 ArrayRef<MCPhysReg> ArgXMMs = get64BitArgumentXMMs(MF, CallConv, Subtarget);
2592 unsigned NumIntRegs =
2593 CCInfo.getFirstUnallocated(ArgGPRs.data(), ArgGPRs.size());
2594 unsigned NumXMMRegs =
2595 CCInfo.getFirstUnallocated(ArgXMMs.data(), ArgXMMs.size());
2596 assert(!(NumXMMRegs && !Subtarget->hasSSE1()) &&
2597 "SSE register cannot be used when SSE is disabled!");
2599 // Gather all the live in physical registers.
2600 SmallVector<SDValue, 6> LiveGPRs;
2601 SmallVector<SDValue, 8> LiveXMMRegs;
2603 for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {
2604 unsigned GPR = MF.addLiveIn(Reg, &X86::GR64RegClass);
2606 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64));
2608 if (!ArgXMMs.empty()) {
2609 unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
2610 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8);
2611 for (MCPhysReg Reg : ArgXMMs.slice(NumXMMRegs)) {
2612 unsigned XMMReg = MF.addLiveIn(Reg, &X86::VR128RegClass);
2613 LiveXMMRegs.push_back(
2614 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32));
2619 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
2620 // Get to the caller-allocated home save location. Add 8 to account
2621 // for the return address.
2622 int HomeOffset = TFI.getOffsetOfLocalArea() + 8;
2623 FuncInfo->setRegSaveFrameIndex(
2624 MFI->CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false));
2625 // Fixup to set vararg frame on shadow area (4 x i64).
2627 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());
2629 // For X86-64, if there are vararg parameters that are passed via
2630 // registers, then we must store them to their spots on the stack so
2631 // they may be loaded by deferencing the result of va_next.
2632 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
2633 FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
2634 FuncInfo->setRegSaveFrameIndex(MFI->CreateStackObject(
2635 ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false));
2638 // Store the integer parameter registers.
2639 SmallVector<SDValue, 8> MemOps;
2640 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
2642 unsigned Offset = FuncInfo->getVarArgsGPOffset();
2643 for (SDValue Val : LiveGPRs) {
2644 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN,
2645 DAG.getIntPtrConstant(Offset));
2647 DAG.getStore(Val.getValue(1), dl, Val, FIN,
2648 MachinePointerInfo::getFixedStack(
2649 FuncInfo->getRegSaveFrameIndex(), Offset),
2651 MemOps.push_back(Store);
2655 if (!ArgXMMs.empty() && NumXMMRegs != ArgXMMs.size()) {
2656 // Now store the XMM (fp + vector) parameter registers.
2657 SmallVector<SDValue, 12> SaveXMMOps;
2658 SaveXMMOps.push_back(Chain);
2659 SaveXMMOps.push_back(ALVal);
2660 SaveXMMOps.push_back(DAG.getIntPtrConstant(
2661 FuncInfo->getRegSaveFrameIndex()));
2662 SaveXMMOps.push_back(DAG.getIntPtrConstant(
2663 FuncInfo->getVarArgsFPOffset()));
2664 SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(),
2666 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
2667 MVT::Other, SaveXMMOps));
2670 if (!MemOps.empty())
2671 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
2674 if (isVarArg && MFI->hasMustTailInVarArgFunc()) {
2675 // Find the largest legal vector type.
2676 MVT VecVT = MVT::Other;
2677 // FIXME: Only some x86_32 calling conventions support AVX512.
2678 if (Subtarget->hasAVX512() &&
2679 (Is64Bit || (CallConv == CallingConv::X86_VectorCall ||
2680 CallConv == CallingConv::Intel_OCL_BI)))
2681 VecVT = MVT::v16f32;
2682 else if (Subtarget->hasAVX())
2684 else if (Subtarget->hasSSE2())
2687 // We forward some GPRs and some vector types.
2688 SmallVector<MVT, 2> RegParmTypes;
2689 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32;
2690 RegParmTypes.push_back(IntVT);
2691 if (VecVT != MVT::Other)
2692 RegParmTypes.push_back(VecVT);
2694 // Compute the set of forwarded registers. The rest are scratch.
2695 SmallVectorImpl<ForwardedRegister> &Forwards =
2696 FuncInfo->getForwardedMustTailRegParms();
2697 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86);
2699 // Conservatively forward AL on x86_64, since it might be used for varargs.
2700 if (Is64Bit && !CCInfo.isAllocated(X86::AL)) {
2701 unsigned ALVReg = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
2702 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));
2705 // Copy all forwards from physical to virtual registers.
2706 for (ForwardedRegister &F : Forwards) {
2707 // FIXME: Can we use a less constrained schedule?
2708 SDValue RegVal = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
2709 F.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(F.VT));
2710 Chain = DAG.getCopyToReg(Chain, dl, F.VReg, RegVal);
2714 // Some CCs need callee pop.
2715 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
2716 MF.getTarget().Options.GuaranteedTailCallOpt)) {
2717 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
2719 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
2720 // If this is an sret function, the return should pop the hidden pointer.
2721 if (!Is64Bit && !IsTailCallConvention(CallConv) &&
2722 !Subtarget->getTargetTriple().isOSMSVCRT() &&
2723 argsAreStructReturn(Ins) == StackStructReturn)
2724 FuncInfo->setBytesToPopOnReturn(4);
2728 // RegSaveFrameIndex is X86-64 only.
2729 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
2730 if (CallConv == CallingConv::X86_FastCall ||
2731 CallConv == CallingConv::X86_ThisCall)
2732 // fastcc functions can't have varargs.
2733 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
2736 FuncInfo->setArgumentStackSize(StackSize);
2742 X86TargetLowering::LowerMemOpCallTo(SDValue Chain,
2743 SDValue StackPtr, SDValue Arg,
2744 SDLoc dl, SelectionDAG &DAG,
2745 const CCValAssign &VA,
2746 ISD::ArgFlagsTy Flags) const {
2747 unsigned LocMemOffset = VA.getLocMemOffset();
2748 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
2749 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
2750 if (Flags.isByVal())
2751 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
2753 return DAG.getStore(Chain, dl, Arg, PtrOff,
2754 MachinePointerInfo::getStack(LocMemOffset),
2758 /// Emit a load of return address if tail call
2759 /// optimization is performed and it is required.
2761 X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG,
2762 SDValue &OutRetAddr, SDValue Chain,
2763 bool IsTailCall, bool Is64Bit,
2764 int FPDiff, SDLoc dl) const {
2765 // Adjust the Return address stack slot.
2766 EVT VT = getPointerTy();
2767 OutRetAddr = getReturnAddressFrameIndex(DAG);
2769 // Load the "old" Return address.
2770 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo(),
2771 false, false, false, 0);
2772 return SDValue(OutRetAddr.getNode(), 1);
2775 /// Emit a store of the return address if tail call
2776 /// optimization is performed and it is required (FPDiff!=0).
2777 static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
2778 SDValue Chain, SDValue RetAddrFrIdx,
2779 EVT PtrVT, unsigned SlotSize,
2780 int FPDiff, SDLoc dl) {
2781 // Store the return address to the appropriate stack slot.
2782 if (!FPDiff) return Chain;
2783 // Calculate the new stack slot for the return address.
2784 int NewReturnAddrFI =
2785 MF.getFrameInfo()->CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
2787 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
2788 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
2789 MachinePointerInfo::getFixedStack(NewReturnAddrFI),
2795 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2796 SmallVectorImpl<SDValue> &InVals) const {
2797 SelectionDAG &DAG = CLI.DAG;
2799 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2800 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2801 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2802 SDValue Chain = CLI.Chain;
2803 SDValue Callee = CLI.Callee;
2804 CallingConv::ID CallConv = CLI.CallConv;
2805 bool &isTailCall = CLI.IsTailCall;
2806 bool isVarArg = CLI.IsVarArg;
2808 MachineFunction &MF = DAG.getMachineFunction();
2809 bool Is64Bit = Subtarget->is64Bit();
2810 bool IsWin64 = Subtarget->isCallingConvWin64(CallConv);
2811 StructReturnType SR = callIsStructReturn(Outs);
2812 bool IsSibcall = false;
2813 X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
2815 if (MF.getTarget().Options.DisableTailCalls)
2818 bool IsMustTail = CLI.CS && CLI.CS->isMustTailCall();
2820 // Force this to be a tail call. The verifier rules are enough to ensure
2821 // that we can lower this successfully without moving the return address
2824 } else if (isTailCall) {
2825 // Check if it's really possible to do a tail call.
2826 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
2827 isVarArg, SR != NotStructReturn,
2828 MF.getFunction()->hasStructRetAttr(), CLI.RetTy,
2829 Outs, OutVals, Ins, DAG);
2831 // Sibcalls are automatically detected tailcalls which do not require
2833 if (!MF.getTarget().Options.GuaranteedTailCallOpt && isTailCall)
2840 assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
2841 "Var args not supported with calling convention fastcc, ghc or hipe");
2843 // Analyze operands of the call, assigning locations to each operand.
2844 SmallVector<CCValAssign, 16> ArgLocs;
2845 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
2847 // Allocate shadow area for Win64
2849 CCInfo.AllocateStack(32, 8);
2851 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
2853 // Get a count of how many bytes are to be pushed on the stack.
2854 unsigned NumBytes = CCInfo.getNextStackOffset();
2856 // This is a sibcall. The memory operands are available in caller's
2857 // own caller's stack.
2859 else if (MF.getTarget().Options.GuaranteedTailCallOpt &&
2860 IsTailCallConvention(CallConv))
2861 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
2864 if (isTailCall && !IsSibcall && !IsMustTail) {
2865 // Lower arguments at fp - stackoffset + fpdiff.
2866 unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
2868 FPDiff = NumBytesCallerPushed - NumBytes;
2870 // Set the delta of movement of the returnaddr stackslot.
2871 // But only set if delta is greater than previous delta.
2872 if (FPDiff < X86Info->getTCReturnAddrDelta())
2873 X86Info->setTCReturnAddrDelta(FPDiff);
2876 unsigned NumBytesToPush = NumBytes;
2877 unsigned NumBytesToPop = NumBytes;
2879 // If we have an inalloca argument, all stack space has already been allocated
2880 // for us and be right at the top of the stack. We don't support multiple
2881 // arguments passed in memory when using inalloca.
2882 if (!Outs.empty() && Outs.back().Flags.isInAlloca()) {
2884 if (!ArgLocs.back().isMemLoc())
2885 report_fatal_error("cannot use inalloca attribute on a register "
2887 if (ArgLocs.back().getLocMemOffset() != 0)
2888 report_fatal_error("any parameter with the inalloca attribute must be "
2889 "the only memory argument");
2893 Chain = DAG.getCALLSEQ_START(
2894 Chain, DAG.getIntPtrConstant(NumBytesToPush, true), dl);
2896 SDValue RetAddrFrIdx;
2897 // Load return address for tail calls.
2898 if (isTailCall && FPDiff)
2899 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
2900 Is64Bit, FPDiff, dl);
2902 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2903 SmallVector<SDValue, 8> MemOpChains;
2906 // Walk the register/memloc assignments, inserting copies/loads. In the case
2907 // of tail call optimization arguments are handle later.
2908 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2909 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2910 // Skip inalloca arguments, they have already been written.
2911 ISD::ArgFlagsTy Flags = Outs[i].Flags;
2912 if (Flags.isInAlloca())
2915 CCValAssign &VA = ArgLocs[i];
2916 EVT RegVT = VA.getLocVT();
2917 SDValue Arg = OutVals[i];
2918 bool isByVal = Flags.isByVal();
2920 // Promote the value if needed.
2921 switch (VA.getLocInfo()) {
2922 default: llvm_unreachable("Unknown loc info!");
2923 case CCValAssign::Full: break;
2924 case CCValAssign::SExt:
2925 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
2927 case CCValAssign::ZExt:
2928 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
2930 case CCValAssign::AExt:
2931 if (RegVT.is128BitVector()) {
2932 // Special case: passing MMX values in XMM registers.
2933 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg);
2934 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
2935 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
2937 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
2939 case CCValAssign::BCvt:
2940 Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg);
2942 case CCValAssign::Indirect: {
2943 // Store the argument.
2944 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
2945 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2946 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot,
2947 MachinePointerInfo::getFixedStack(FI),
2954 if (VA.isRegLoc()) {
2955 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2956 if (isVarArg && IsWin64) {
2957 // Win64 ABI requires argument XMM reg to be copied to the corresponding
2958 // shadow reg if callee is a varargs function.
2959 unsigned ShadowReg = 0;
2960 switch (VA.getLocReg()) {
2961 case X86::XMM0: ShadowReg = X86::RCX; break;
2962 case X86::XMM1: ShadowReg = X86::RDX; break;
2963 case X86::XMM2: ShadowReg = X86::R8; break;
2964 case X86::XMM3: ShadowReg = X86::R9; break;
2967 RegsToPass.push_back(std::make_pair(ShadowReg, Arg));
2969 } else if (!IsSibcall && (!isTailCall || isByVal)) {
2970 assert(VA.isMemLoc());
2971 if (!StackPtr.getNode())
2972 StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
2974 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
2975 dl, DAG, VA, Flags));
2979 if (!MemOpChains.empty())
2980 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
2982 if (Subtarget->isPICStyleGOT()) {
2983 // ELF / PIC requires GOT in the EBX register before function calls via PLT
2986 RegsToPass.push_back(std::make_pair(unsigned(X86::EBX),
2987 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy())));
2989 // If we are tail calling and generating PIC/GOT style code load the
2990 // address of the callee into ECX. The value in ecx is used as target of
2991 // the tail jump. This is done to circumvent the ebx/callee-saved problem
2992 // for tail calls on PIC/GOT architectures. Normally we would just put the
2993 // address of GOT into ebx and then call target@PLT. But for tail calls
2994 // ebx would be restored (since ebx is callee saved) before jumping to the
2997 // Note: The actual moving to ECX is done further down.
2998 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
2999 if (G && !G->getGlobal()->hasHiddenVisibility() &&
3000 !G->getGlobal()->hasProtectedVisibility())
3001 Callee = LowerGlobalAddress(Callee, DAG);
3002 else if (isa<ExternalSymbolSDNode>(Callee))
3003 Callee = LowerExternalSymbol(Callee, DAG);
3007 if (Is64Bit && isVarArg && !IsWin64 && !IsMustTail) {
3008 // From AMD64 ABI document:
3009 // For calls that may call functions that use varargs or stdargs
3010 // (prototype-less calls or calls to functions containing ellipsis (...) in
3011 // the declaration) %al is used as hidden argument to specify the number
3012 // of SSE registers used. The contents of %al do not need to match exactly
3013 // the number of registers, but must be an ubound on the number of SSE
3014 // registers used and is in the range 0 - 8 inclusive.
3016 // Count the number of XMM registers allocated.
3017 static const MCPhysReg XMMArgRegs[] = {
3018 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3019 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3021 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);
3022 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3023 && "SSE registers cannot be used when SSE is disabled");
3025 RegsToPass.push_back(std::make_pair(unsigned(X86::AL),
3026 DAG.getConstant(NumXMMRegs, MVT::i8)));
3029 if (isVarArg && IsMustTail) {
3030 const auto &Forwards = X86Info->getForwardedMustTailRegParms();
3031 for (const auto &F : Forwards) {
3032 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
3033 RegsToPass.push_back(std::make_pair(unsigned(F.PReg), Val));
3037 // For tail calls lower the arguments to the 'real' stack slots. Sibcalls
3038 // don't need this because the eligibility check rejects calls that require
3039 // shuffling arguments passed in memory.
3040 if (!IsSibcall && isTailCall) {
3041 // Force all the incoming stack arguments to be loaded from the stack
3042 // before any new outgoing arguments are stored to the stack, because the
3043 // outgoing stack slots may alias the incoming argument stack slots, and
3044 // the alias isn't otherwise explicit. This is slightly more conservative
3045 // than necessary, because it means that each store effectively depends
3046 // on every argument instead of just those arguments it would clobber.
3047 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
3049 SmallVector<SDValue, 8> MemOpChains2;
3052 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3053 CCValAssign &VA = ArgLocs[i];
3056 assert(VA.isMemLoc());
3057 SDValue Arg = OutVals[i];
3058 ISD::ArgFlagsTy Flags = Outs[i].Flags;
3059 // Skip inalloca arguments. They don't require any work.
3060 if (Flags.isInAlloca())
3062 // Create frame index.
3063 int32_t Offset = VA.getLocMemOffset()+FPDiff;
3064 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
3065 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true);
3066 FIN = DAG.getFrameIndex(FI, getPointerTy());
3068 if (Flags.isByVal()) {
3069 // Copy relative to framepointer.
3070 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
3071 if (!StackPtr.getNode())
3072 StackPtr = DAG.getCopyFromReg(Chain, dl,
3073 RegInfo->getStackRegister(),
3075 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source);
3077 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
3081 // Store relative to framepointer.
3082 MemOpChains2.push_back(
3083 DAG.getStore(ArgChain, dl, Arg, FIN,
3084 MachinePointerInfo::getFixedStack(FI),
3089 if (!MemOpChains2.empty())
3090 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
3092 // Store the return address to the appropriate stack slot.
3093 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
3094 getPointerTy(), RegInfo->getSlotSize(),
3098 // Build a sequence of copy-to-reg nodes chained together with token chain
3099 // and flag operands which copy the outgoing args into registers.
3101 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
3102 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
3103 RegsToPass[i].second, InFlag);
3104 InFlag = Chain.getValue(1);
3107 if (DAG.getTarget().getCodeModel() == CodeModel::Large) {
3108 assert(Is64Bit && "Large code model is only legal in 64-bit mode.");
3109 // In the 64-bit large code model, we have to make all calls
3110 // through a register, since the call instruction's 32-bit
3111 // pc-relative offset may not be large enough to hold the whole
3113 } else if (Callee->getOpcode() == ISD::GlobalAddress) {
3114 // If the callee is a GlobalAddress node (quite common, every direct call
3115 // is) turn it into a TargetGlobalAddress node so that legalize doesn't hack
3117 GlobalAddressSDNode* G = cast<GlobalAddressSDNode>(Callee);
3119 // We should use extra load for direct calls to dllimported functions in
3121 const GlobalValue *GV = G->getGlobal();
3122 if (!GV->hasDLLImportStorageClass()) {
3123 unsigned char OpFlags = 0;
3124 bool ExtraLoad = false;
3125 unsigned WrapperKind = ISD::DELETED_NODE;
3127 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to
3128 // external symbols most go through the PLT in PIC mode. If the symbol
3129 // has hidden or protected visibility, or if it is static or local, then
3130 // we don't need to use the PLT - we can directly call it.
3131 if (Subtarget->isTargetELF() &&
3132 DAG.getTarget().getRelocationModel() == Reloc::PIC_ &&
3133 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
3134 OpFlags = X86II::MO_PLT;
3135 } else if (Subtarget->isPICStyleStubAny() &&
3136 (GV->isDeclaration() || GV->isWeakForLinker()) &&
3137 (!Subtarget->getTargetTriple().isMacOSX() ||
3138 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
3139 // PC-relative references to external symbols should go through $stub,
3140 // unless we're building with the leopard linker or later, which
3141 // automatically synthesizes these stubs.
3142 OpFlags = X86II::MO_DARWIN_STUB;
3143 } else if (Subtarget->isPICStyleRIPRel() && isa<Function>(GV) &&
3144 cast<Function>(GV)->hasFnAttribute(Attribute::NonLazyBind)) {
3145 // If the function is marked as non-lazy, generate an indirect call
3146 // which loads from the GOT directly. This avoids runtime overhead
3147 // at the cost of eager binding (and one extra byte of encoding).
3148 OpFlags = X86II::MO_GOTPCREL;
3149 WrapperKind = X86ISD::WrapperRIP;
3153 Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(),
3154 G->getOffset(), OpFlags);
3156 // Add a wrapper if needed.
3157 if (WrapperKind != ISD::DELETED_NODE)
3158 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee);
3159 // Add extra indirection if needed.
3161 Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee,
3162 MachinePointerInfo::getGOT(),
3163 false, false, false, 0);
3165 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
3166 unsigned char OpFlags = 0;
3168 // On ELF targets, in either X86-64 or X86-32 mode, direct calls to
3169 // external symbols should go through the PLT.
3170 if (Subtarget->isTargetELF() &&
3171 DAG.getTarget().getRelocationModel() == Reloc::PIC_) {
3172 OpFlags = X86II::MO_PLT;
3173 } else if (Subtarget->isPICStyleStubAny() &&
3174 (!Subtarget->getTargetTriple().isMacOSX() ||
3175 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
3176 // PC-relative references to external symbols should go through $stub,
3177 // unless we're building with the leopard linker or later, which
3178 // automatically synthesizes these stubs.
3179 OpFlags = X86II::MO_DARWIN_STUB;
3182 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(),
3184 } else if (Subtarget->isTarget64BitILP32() &&
3185 Callee->getValueType(0) == MVT::i32) {
3186 // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
3187 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee);
3190 // Returns a chain & a flag for retval copy to use.
3191 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3192 SmallVector<SDValue, 8> Ops;
3194 if (!IsSibcall && isTailCall) {
3195 Chain = DAG.getCALLSEQ_END(Chain,
3196 DAG.getIntPtrConstant(NumBytesToPop, true),
3197 DAG.getIntPtrConstant(0, true), InFlag, dl);
3198 InFlag = Chain.getValue(1);
3201 Ops.push_back(Chain);
3202 Ops.push_back(Callee);
3205 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32));
3207 // Add argument registers to the end of the list so that they are known live
3209 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
3210 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
3211 RegsToPass[i].second.getValueType()));
3213 // Add a register mask operand representing the call-preserved registers.
3214 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
3215 const uint32_t *Mask = TRI->getCallPreservedMask(CallConv);
3216 assert(Mask && "Missing call preserved mask for calling convention");
3217 Ops.push_back(DAG.getRegisterMask(Mask));
3219 if (InFlag.getNode())
3220 Ops.push_back(InFlag);
3224 //// If this is the first return lowered for this function, add the regs
3225 //// to the liveout set for the function.
3226 // This isn't right, although it's probably harmless on x86; liveouts
3227 // should be computed from returns not tail calls. Consider a void
3228 // function making a tail call to a function returning int.
3229 return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
3232 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
3233 InFlag = Chain.getValue(1);
3235 // Create the CALLSEQ_END node.
3236 unsigned NumBytesForCalleeToPop;
3237 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
3238 DAG.getTarget().Options.GuaranteedTailCallOpt))
3239 NumBytesForCalleeToPop = NumBytes; // Callee pops everything
3240 else if (!Is64Bit && !IsTailCallConvention(CallConv) &&
3241 !Subtarget->getTargetTriple().isOSMSVCRT() &&
3242 SR == StackStructReturn)
3243 // If this is a call to a struct-return function, the callee
3244 // pops the hidden struct pointer, so we have to push it back.
3245 // This is common for Darwin/X86, Linux & Mingw32 targets.
3246 // For MSVC Win32 targets, the caller pops the hidden struct pointer.
3247 NumBytesForCalleeToPop = 4;
3249 NumBytesForCalleeToPop = 0; // Callee pops nothing.
3251 // Returns a flag for retval copy to use.
3253 Chain = DAG.getCALLSEQ_END(Chain,
3254 DAG.getIntPtrConstant(NumBytesToPop, true),
3255 DAG.getIntPtrConstant(NumBytesForCalleeToPop,
3258 InFlag = Chain.getValue(1);
3261 // Handle result values, copying them out of physregs into vregs that we
3263 return LowerCallResult(Chain, InFlag, CallConv, isVarArg,
3264 Ins, dl, DAG, InVals);
3267 //===----------------------------------------------------------------------===//
3268 // Fast Calling Convention (tail call) implementation
3269 //===----------------------------------------------------------------------===//
3271 // Like std call, callee cleans arguments, convention except that ECX is
3272 // reserved for storing the tail called function address. Only 2 registers are
3273 // free for argument passing (inreg). Tail call optimization is performed
3275 // * tailcallopt is enabled
3276 // * caller/callee are fastcc
3277 // On X86_64 architecture with GOT-style position independent code only local
3278 // (within module) calls are supported at the moment.
3279 // To keep the stack aligned according to platform abi the function
3280 // GetAlignedArgumentStackSize ensures that argument delta is always multiples
3281 // of stack alignment. (Dynamic linkers need this - darwin's dyld for example)
3282 // If a tail called function callee has more arguments than the caller the
3283 // caller needs to make sure that there is room to move the RETADDR to. This is
3284 // achieved by reserving an area the size of the argument delta right after the
3285 // original RETADDR, but before the saved framepointer or the spilled registers
3286 // e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
3298 /// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned
3299 /// for a 16 byte align requirement.
3301 X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
3302 SelectionDAG& DAG) const {
3303 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3304 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
3305 unsigned StackAlignment = TFI.getStackAlignment();
3306 uint64_t AlignMask = StackAlignment - 1;
3307 int64_t Offset = StackSize;
3308 unsigned SlotSize = RegInfo->getSlotSize();
3309 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) {
3310 // Number smaller than 12 so just add the difference.
3311 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask));
3313 // Mask out lower bits, add stackalignment once plus the 12 bytes.
3314 Offset = ((~AlignMask) & Offset) + StackAlignment +
3315 (StackAlignment-SlotSize);
3320 /// MatchingStackOffset - Return true if the given stack call argument is
3321 /// already available in the same position (relatively) of the caller's
3322 /// incoming argument stack.
3324 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
3325 MachineFrameInfo *MFI, const MachineRegisterInfo *MRI,
3326 const X86InstrInfo *TII) {
3327 unsigned Bytes = Arg.getValueType().getSizeInBits() / 8;
3329 if (Arg.getOpcode() == ISD::CopyFromReg) {
3330 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
3331 if (!TargetRegisterInfo::isVirtualRegister(VR))
3333 MachineInstr *Def = MRI->getVRegDef(VR);
3336 if (!Flags.isByVal()) {
3337 if (!TII->isLoadFromStackSlot(Def, FI))
3340 unsigned Opcode = Def->getOpcode();
3341 if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r ||
3342 Opcode == X86::LEA64_32r) &&
3343 Def->getOperand(1).isFI()) {
3344 FI = Def->getOperand(1).getIndex();
3345 Bytes = Flags.getByValSize();
3349 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
3350 if (Flags.isByVal())
3351 // ByVal argument is passed in as a pointer but it's now being
3352 // dereferenced. e.g.
3353 // define @foo(%struct.X* %A) {
3354 // tail call @bar(%struct.X* byval %A)
3357 SDValue Ptr = Ld->getBasePtr();
3358 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
3361 FI = FINode->getIndex();
3362 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {
3363 FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg);
3364 FI = FINode->getIndex();
3365 Bytes = Flags.getByValSize();
3369 assert(FI != INT_MAX);
3370 if (!MFI->isFixedObjectIndex(FI))
3372 return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI);
3375 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
3376 /// for tail call optimization. Targets which want to do tail call
3377 /// optimization should implement this function.
3379 X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
3380 CallingConv::ID CalleeCC,
3382 bool isCalleeStructRet,
3383 bool isCallerStructRet,
3385 const SmallVectorImpl<ISD::OutputArg> &Outs,
3386 const SmallVectorImpl<SDValue> &OutVals,
3387 const SmallVectorImpl<ISD::InputArg> &Ins,
3388 SelectionDAG &DAG) const {
3389 if (!IsTailCallConvention(CalleeCC) && !IsCCallConvention(CalleeCC))
3392 // If -tailcallopt is specified, make fastcc functions tail-callable.
3393 const MachineFunction &MF = DAG.getMachineFunction();
3394 const Function *CallerF = MF.getFunction();
3396 // If the function return type is x86_fp80 and the callee return type is not,
3397 // then the FP_EXTEND of the call result is not a nop. It's not safe to
3398 // perform a tailcall optimization here.
3399 if (CallerF->getReturnType()->isX86_FP80Ty() && !RetTy->isX86_FP80Ty())
3402 CallingConv::ID CallerCC = CallerF->getCallingConv();
3403 bool CCMatch = CallerCC == CalleeCC;
3404 bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CalleeCC);
3405 bool IsCallerWin64 = Subtarget->isCallingConvWin64(CallerCC);
3407 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
3408 if (IsTailCallConvention(CalleeCC) && CCMatch)
3413 // Look for obvious safe cases to perform tail call optimization that do not
3414 // require ABI changes. This is what gcc calls sibcall.
3416 // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to
3417 // emit a special epilogue.
3418 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3419 if (RegInfo->needsStackRealignment(MF))
3422 // Also avoid sibcall optimization if either caller or callee uses struct
3423 // return semantics.
3424 if (isCalleeStructRet || isCallerStructRet)
3427 // An stdcall/thiscall caller is expected to clean up its arguments; the
3428 // callee isn't going to do that.
3429 // FIXME: this is more restrictive than needed. We could produce a tailcall
3430 // when the stack adjustment matches. For example, with a thiscall that takes
3431 // only one argument.
3432 if (!CCMatch && (CallerCC == CallingConv::X86_StdCall ||
3433 CallerCC == CallingConv::X86_ThisCall))
3436 // Do not sibcall optimize vararg calls unless all arguments are passed via
3438 if (isVarArg && !Outs.empty()) {
3440 // Optimizing for varargs on Win64 is unlikely to be safe without
3441 // additional testing.
3442 if (IsCalleeWin64 || IsCallerWin64)
3445 SmallVector<CCValAssign, 16> ArgLocs;
3446 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), ArgLocs,
3449 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
3450 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i)
3451 if (!ArgLocs[i].isRegLoc())
3455 // If the call result is in ST0 / ST1, it needs to be popped off the x87
3456 // stack. Therefore, if it's not used by the call it is not safe to optimize
3457 // this into a sibcall.
3458 bool Unused = false;
3459 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
3466 SmallVector<CCValAssign, 16> RVLocs;
3467 CCState CCInfo(CalleeCC, false, DAG.getMachineFunction(), RVLocs,
3469 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
3470 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
3471 CCValAssign &VA = RVLocs[i];
3472 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
3477 // If the calling conventions do not match, then we'd better make sure the
3478 // results are returned in the same way as what the caller expects.
3480 SmallVector<CCValAssign, 16> RVLocs1;
3481 CCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(), RVLocs1,
3483 CCInfo1.AnalyzeCallResult(Ins, RetCC_X86);
3485 SmallVector<CCValAssign, 16> RVLocs2;
3486 CCState CCInfo2(CallerCC, false, DAG.getMachineFunction(), RVLocs2,
3488 CCInfo2.AnalyzeCallResult(Ins, RetCC_X86);
3490 if (RVLocs1.size() != RVLocs2.size())
3492 for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) {
3493 if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc())
3495 if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo())
3497 if (RVLocs1[i].isRegLoc()) {
3498 if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg())
3501 if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset())
3507 // If the callee takes no arguments then go on to check the results of the
3509 if (!Outs.empty()) {
3510 // Check if stack adjustment is needed. For now, do not do this if any
3511 // argument is passed on the stack.
3512 SmallVector<CCValAssign, 16> ArgLocs;
3513 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), ArgLocs,
3516 // Allocate shadow area for Win64
3518 CCInfo.AllocateStack(32, 8);
3520 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
3521 if (CCInfo.getNextStackOffset()) {
3522 MachineFunction &MF = DAG.getMachineFunction();
3523 if (MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn())
3526 // Check if the arguments are already laid out in the right way as
3527 // the caller's fixed stack objects.
3528 MachineFrameInfo *MFI = MF.getFrameInfo();
3529 const MachineRegisterInfo *MRI = &MF.getRegInfo();
3530 const X86InstrInfo *TII = Subtarget->getInstrInfo();
3531 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3532 CCValAssign &VA = ArgLocs[i];
3533 SDValue Arg = OutVals[i];
3534 ISD::ArgFlagsTy Flags = Outs[i].Flags;
3535 if (VA.getLocInfo() == CCValAssign::Indirect)
3537 if (!VA.isRegLoc()) {
3538 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
3545 // If the tailcall address may be in a register, then make sure it's
3546 // possible to register allocate for it. In 32-bit, the call address can
3547 // only target EAX, EDX, or ECX since the tail call must be scheduled after
3548 // callee-saved registers are restored. These happen to be the same
3549 // registers used to pass 'inreg' arguments so watch out for those.
3550 if (!Subtarget->is64Bit() &&
3551 ((!isa<GlobalAddressSDNode>(Callee) &&
3552 !isa<ExternalSymbolSDNode>(Callee)) ||
3553 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
3554 unsigned NumInRegs = 0;
3555 // In PIC we need an extra register to formulate the address computation
3557 unsigned MaxInRegs =
3558 (DAG.getTarget().getRelocationModel() == Reloc::PIC_) ? 2 : 3;
3560 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3561 CCValAssign &VA = ArgLocs[i];
3564 unsigned Reg = VA.getLocReg();
3567 case X86::EAX: case X86::EDX: case X86::ECX:
3568 if (++NumInRegs == MaxInRegs)
3580 X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
3581 const TargetLibraryInfo *libInfo) const {
3582 return X86::createFastISel(funcInfo, libInfo);
3585 //===----------------------------------------------------------------------===//
3586 // Other Lowering Hooks
3587 //===----------------------------------------------------------------------===//
3589 static bool MayFoldLoad(SDValue Op) {
3590 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
3593 static bool MayFoldIntoStore(SDValue Op) {
3594 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
3597 static bool isTargetShuffle(unsigned Opcode) {
3599 default: return false;
3600 case X86ISD::BLENDI:
3601 case X86ISD::PSHUFB:
3602 case X86ISD::PSHUFD:
3603 case X86ISD::PSHUFHW:
3604 case X86ISD::PSHUFLW:
3606 case X86ISD::PALIGNR:
3607 case X86ISD::MOVLHPS:
3608 case X86ISD::MOVLHPD:
3609 case X86ISD::MOVHLPS:
3610 case X86ISD::MOVLPS:
3611 case X86ISD::MOVLPD:
3612 case X86ISD::MOVSHDUP:
3613 case X86ISD::MOVSLDUP:
3614 case X86ISD::MOVDDUP:
3617 case X86ISD::UNPCKL:
3618 case X86ISD::UNPCKH:
3619 case X86ISD::VPERMILPI:
3620 case X86ISD::VPERM2X128:
3621 case X86ISD::VPERMI:
3626 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3627 SDValue V1, SelectionDAG &DAG) {
3629 default: llvm_unreachable("Unknown x86 shuffle node");
3630 case X86ISD::MOVSHDUP:
3631 case X86ISD::MOVSLDUP:
3632 case X86ISD::MOVDDUP:
3633 return DAG.getNode(Opc, dl, VT, V1);
3637 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3638 SDValue V1, unsigned TargetMask,
3639 SelectionDAG &DAG) {
3641 default: llvm_unreachable("Unknown x86 shuffle node");
3642 case X86ISD::PSHUFD:
3643 case X86ISD::PSHUFHW:
3644 case X86ISD::PSHUFLW:
3645 case X86ISD::VPERMILPI:
3646 case X86ISD::VPERMI:
3647 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8));
3651 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3652 SDValue V1, SDValue V2, unsigned TargetMask,
3653 SelectionDAG &DAG) {
3655 default: llvm_unreachable("Unknown x86 shuffle node");
3656 case X86ISD::PALIGNR:
3657 case X86ISD::VALIGN:
3659 case X86ISD::VPERM2X128:
3660 return DAG.getNode(Opc, dl, VT, V1, V2,
3661 DAG.getConstant(TargetMask, MVT::i8));
3665 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3666 SDValue V1, SDValue V2, SelectionDAG &DAG) {
3668 default: llvm_unreachable("Unknown x86 shuffle node");
3669 case X86ISD::MOVLHPS:
3670 case X86ISD::MOVLHPD:
3671 case X86ISD::MOVHLPS:
3672 case X86ISD::MOVLPS:
3673 case X86ISD::MOVLPD:
3676 case X86ISD::UNPCKL:
3677 case X86ISD::UNPCKH:
3678 return DAG.getNode(Opc, dl, VT, V1, V2);
3682 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
3683 MachineFunction &MF = DAG.getMachineFunction();
3684 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3685 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
3686 int ReturnAddrIndex = FuncInfo->getRAIndex();
3688 if (ReturnAddrIndex == 0) {
3689 // Set up a frame object for the return address.
3690 unsigned SlotSize = RegInfo->getSlotSize();
3691 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize,
3694 FuncInfo->setRAIndex(ReturnAddrIndex);
3697 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
3700 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
3701 bool hasSymbolicDisplacement) {
3702 // Offset should fit into 32 bit immediate field.
3703 if (!isInt<32>(Offset))
3706 // If we don't have a symbolic displacement - we don't have any extra
3708 if (!hasSymbolicDisplacement)
3711 // FIXME: Some tweaks might be needed for medium code model.
3712 if (M != CodeModel::Small && M != CodeModel::Kernel)
3715 // For small code model we assume that latest object is 16MB before end of 31
3716 // bits boundary. We may also accept pretty large negative constants knowing
3717 // that all objects are in the positive half of address space.
3718 if (M == CodeModel::Small && Offset < 16*1024*1024)
3721 // For kernel code model we know that all object resist in the negative half
3722 // of 32bits address space. We may not accept negative offsets, since they may
3723 // be just off and we may accept pretty large positive ones.
3724 if (M == CodeModel::Kernel && Offset >= 0)
3730 /// isCalleePop - Determines whether the callee is required to pop its
3731 /// own arguments. Callee pop is necessary to support tail calls.
3732 bool X86::isCalleePop(CallingConv::ID CallingConv,
3733 bool is64Bit, bool IsVarArg, bool TailCallOpt) {
3734 switch (CallingConv) {
3737 case CallingConv::X86_StdCall:
3738 case CallingConv::X86_FastCall:
3739 case CallingConv::X86_ThisCall:
3741 case CallingConv::Fast:
3742 case CallingConv::GHC:
3743 case CallingConv::HiPE:
3750 /// \brief Return true if the condition is an unsigned comparison operation.
3751 static bool isX86CCUnsigned(unsigned X86CC) {
3753 default: llvm_unreachable("Invalid integer condition!");
3754 case X86::COND_E: return true;
3755 case X86::COND_G: return false;
3756 case X86::COND_GE: return false;
3757 case X86::COND_L: return false;
3758 case X86::COND_LE: return false;
3759 case X86::COND_NE: return true;
3760 case X86::COND_B: return true;
3761 case X86::COND_A: return true;
3762 case X86::COND_BE: return true;
3763 case X86::COND_AE: return true;
3765 llvm_unreachable("covered switch fell through?!");
3768 /// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86
3769 /// specific condition code, returning the condition code and the LHS/RHS of the
3770 /// comparison to make.
3771 static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
3772 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) {
3774 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
3775 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
3776 // X > -1 -> X == 0, jump !sign.
3777 RHS = DAG.getConstant(0, RHS.getValueType());
3778 return X86::COND_NS;
3780 if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
3781 // X < 0 -> X == 0, jump on sign.
3784 if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) {
3786 RHS = DAG.getConstant(0, RHS.getValueType());
3787 return X86::COND_LE;
3791 switch (SetCCOpcode) {
3792 default: llvm_unreachable("Invalid integer condition!");
3793 case ISD::SETEQ: return X86::COND_E;
3794 case ISD::SETGT: return X86::COND_G;
3795 case ISD::SETGE: return X86::COND_GE;
3796 case ISD::SETLT: return X86::COND_L;
3797 case ISD::SETLE: return X86::COND_LE;
3798 case ISD::SETNE: return X86::COND_NE;
3799 case ISD::SETULT: return X86::COND_B;
3800 case ISD::SETUGT: return X86::COND_A;
3801 case ISD::SETULE: return X86::COND_BE;
3802 case ISD::SETUGE: return X86::COND_AE;
3806 // First determine if it is required or is profitable to flip the operands.
3808 // If LHS is a foldable load, but RHS is not, flip the condition.
3809 if (ISD::isNON_EXTLoad(LHS.getNode()) &&
3810 !ISD::isNON_EXTLoad(RHS.getNode())) {
3811 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
3812 std::swap(LHS, RHS);
3815 switch (SetCCOpcode) {
3821 std::swap(LHS, RHS);
3825 // On a floating point condition, the flags are set as follows:
3827 // 0 | 0 | 0 | X > Y
3828 // 0 | 0 | 1 | X < Y
3829 // 1 | 0 | 0 | X == Y
3830 // 1 | 1 | 1 | unordered
3831 switch (SetCCOpcode) {
3832 default: llvm_unreachable("Condcode should be pre-legalized away");
3834 case ISD::SETEQ: return X86::COND_E;
3835 case ISD::SETOLT: // flipped
3837 case ISD::SETGT: return X86::COND_A;
3838 case ISD::SETOLE: // flipped
3840 case ISD::SETGE: return X86::COND_AE;
3841 case ISD::SETUGT: // flipped
3843 case ISD::SETLT: return X86::COND_B;
3844 case ISD::SETUGE: // flipped
3846 case ISD::SETLE: return X86::COND_BE;
3848 case ISD::SETNE: return X86::COND_NE;
3849 case ISD::SETUO: return X86::COND_P;
3850 case ISD::SETO: return X86::COND_NP;
3852 case ISD::SETUNE: return X86::COND_INVALID;
3856 /// hasFPCMov - is there a floating point cmov for the specific X86 condition
3857 /// code. Current x86 isa includes the following FP cmov instructions:
3858 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
3859 static bool hasFPCMov(unsigned X86CC) {
3875 /// isFPImmLegal - Returns true if the target can instruction select the
3876 /// specified FP immediate natively. If false, the legalizer will
3877 /// materialize the FP immediate as a load from a constant pool.
3878 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
3879 for (unsigned i = 0, e = LegalFPImmediates.size(); i != e; ++i) {
3880 if (Imm.bitwiseIsEqual(LegalFPImmediates[i]))
3886 bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
3887 ISD::LoadExtType ExtTy,
3889 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3890 // relocation target a movq or addq instruction: don't let the load shrink.
3891 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3892 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3893 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3894 return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3898 /// \brief Returns true if it is beneficial to convert a load of a constant
3899 /// to just the constant itself.
3900 bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3902 assert(Ty->isIntegerTy());
3904 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3905 if (BitSize == 0 || BitSize > 64)
3910 bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT,
3911 unsigned Index) const {
3912 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
3915 return (Index == 0 || Index == ResVT.getVectorNumElements());
3918 bool X86TargetLowering::isCheapToSpeculateCttz() const {
3919 // Speculate cttz only if we can directly use TZCNT.
3920 return Subtarget->hasBMI();
3923 bool X86TargetLowering::isCheapToSpeculateCtlz() const {
3924 // Speculate ctlz only if we can directly use LZCNT.
3925 return Subtarget->hasLZCNT();
3928 /// isUndefOrInRange - Return true if Val is undef or if its value falls within
3929 /// the specified range (L, H].
3930 static bool isUndefOrInRange(int Val, int Low, int Hi) {
3931 return (Val < 0) || (Val >= Low && Val < Hi);
3934 /// isUndefOrEqual - Val is either less than zero (undef) or equal to the
3935 /// specified value.
3936 static bool isUndefOrEqual(int Val, int CmpVal) {
3937 return (Val < 0 || Val == CmpVal);
3940 /// isSequentialOrUndefInRange - Return true if every element in Mask, beginning
3941 /// from position Pos and ending in Pos+Size, falls within the specified
3942 /// sequential range (Low, Low+Size]. or is undef.
3943 static bool isSequentialOrUndefInRange(ArrayRef<int> Mask,
3944 unsigned Pos, unsigned Size, int Low) {
3945 for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
3946 if (!isUndefOrEqual(Mask[i], Low))
3951 /// isPSHUFDMask - Return true if the node specifies a shuffle of elements that
3952 /// is suitable for input to PSHUFD. That is, it doesn't reference the other
3953 /// operand - by default will match for first operand.
3954 static bool isPSHUFDMask(ArrayRef<int> Mask, MVT VT,
3955 bool TestSecondOperand = false) {
3956 if (VT != MVT::v4f32 && VT != MVT::v4i32 &&
3957 VT != MVT::v2f64 && VT != MVT::v2i64)
3960 unsigned NumElems = VT.getVectorNumElements();
3961 unsigned Lo = TestSecondOperand ? NumElems : 0;
3962 unsigned Hi = Lo + NumElems;
3964 for (unsigned i = 0; i < NumElems; ++i)
3965 if (!isUndefOrInRange(Mask[i], (int)Lo, (int)Hi))
3971 /// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that
3972 /// is suitable for input to PSHUFHW.
3973 static bool isPSHUFHWMask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
3974 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
3977 // Lower quadword copied in order or undef.
3978 if (!isSequentialOrUndefInRange(Mask, 0, 4, 0))
3981 // Upper quadword shuffled.
3982 for (unsigned i = 4; i != 8; ++i)
3983 if (!isUndefOrInRange(Mask[i], 4, 8))
3986 if (VT == MVT::v16i16) {
3987 // Lower quadword copied in order or undef.
3988 if (!isSequentialOrUndefInRange(Mask, 8, 4, 8))
3991 // Upper quadword shuffled.
3992 for (unsigned i = 12; i != 16; ++i)
3993 if (!isUndefOrInRange(Mask[i], 12, 16))
4000 /// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that
4001 /// is suitable for input to PSHUFLW.
4002 static bool isPSHUFLWMask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
4003 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
4006 // Upper quadword copied in order.
4007 if (!isSequentialOrUndefInRange(Mask, 4, 4, 4))
4010 // Lower quadword shuffled.
4011 for (unsigned i = 0; i != 4; ++i)
4012 if (!isUndefOrInRange(Mask[i], 0, 4))
4015 if (VT == MVT::v16i16) {
4016 // Upper quadword copied in order.
4017 if (!isSequentialOrUndefInRange(Mask, 12, 4, 12))
4020 // Lower quadword shuffled.
4021 for (unsigned i = 8; i != 12; ++i)
4022 if (!isUndefOrInRange(Mask[i], 8, 12))
4029 /// \brief Return true if the mask specifies a shuffle of elements that is
4030 /// suitable for input to intralane (palignr) or interlane (valign) vector
4032 static bool isAlignrMask(ArrayRef<int> Mask, MVT VT, bool InterLane) {
4033 unsigned NumElts = VT.getVectorNumElements();
4034 unsigned NumLanes = InterLane ? 1: VT.getSizeInBits()/128;
4035 unsigned NumLaneElts = NumElts/NumLanes;
4037 // Do not handle 64-bit element shuffles with palignr.
4038 if (NumLaneElts == 2)
4041 for (unsigned l = 0; l != NumElts; l+=NumLaneElts) {
4043 for (i = 0; i != NumLaneElts; ++i) {
4048 // Lane is all undef, go to next lane
4049 if (i == NumLaneElts)
4052 int Start = Mask[i+l];
4054 // Make sure its in this lane in one of the sources
4055 if (!isUndefOrInRange(Start, l, l+NumLaneElts) &&
4056 !isUndefOrInRange(Start, l+NumElts, l+NumElts+NumLaneElts))
4059 // If not lane 0, then we must match lane 0
4060 if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Start, Mask[i]+l))
4063 // Correct second source to be contiguous with first source
4064 if (Start >= (int)NumElts)
4065 Start -= NumElts - NumLaneElts;
4067 // Make sure we're shifting in the right direction.
4068 if (Start <= (int)(i+l))
4073 // Check the rest of the elements to see if they are consecutive.
4074 for (++i; i != NumLaneElts; ++i) {
4075 int Idx = Mask[i+l];
4077 // Make sure its in this lane
4078 if (!isUndefOrInRange(Idx, l, l+NumLaneElts) &&
4079 !isUndefOrInRange(Idx, l+NumElts, l+NumElts+NumLaneElts))
4082 // If not lane 0, then we must match lane 0
4083 if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Idx, Mask[i]+l))
4086 if (Idx >= (int)NumElts)
4087 Idx -= NumElts - NumLaneElts;
4089 if (!isUndefOrEqual(Idx, Start+i))
4098 /// \brief Return true if the node specifies a shuffle of elements that is
4099 /// suitable for input to PALIGNR.
4100 static bool isPALIGNRMask(ArrayRef<int> Mask, MVT VT,
4101 const X86Subtarget *Subtarget) {
4102 if ((VT.is128BitVector() && !Subtarget->hasSSSE3()) ||
4103 (VT.is256BitVector() && !Subtarget->hasInt256()) ||
4104 VT.is512BitVector())
4105 // FIXME: Add AVX512BW.
4108 return isAlignrMask(Mask, VT, false);
4111 /// \brief Return true if the node specifies a shuffle of elements that is
4112 /// suitable for input to VALIGN.
4113 static bool isVALIGNMask(ArrayRef<int> Mask, MVT VT,
4114 const X86Subtarget *Subtarget) {
4115 // FIXME: Add AVX512VL.
4116 if (!VT.is512BitVector() || !Subtarget->hasAVX512())
4118 return isAlignrMask(Mask, VT, true);
4121 /// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming
4122 /// the two vector operands have swapped position.
4123 static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask,
4124 unsigned NumElems) {
4125 for (unsigned i = 0; i != NumElems; ++i) {
4129 else if (idx < (int)NumElems)
4130 Mask[i] = idx + NumElems;
4132 Mask[i] = idx - NumElems;
4136 /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
4137 /// specifies a shuffle of elements that is suitable for input to 128/256-bit
4138 /// SHUFPS and SHUFPD. If Commuted is true, then it checks for sources to be
4139 /// reverse of what x86 shuffles want.
4140 static bool isSHUFPMask(ArrayRef<int> Mask, MVT VT, bool Commuted = false) {
4142 unsigned NumElems = VT.getVectorNumElements();
4143 unsigned NumLanes = VT.getSizeInBits()/128;
4144 unsigned NumLaneElems = NumElems/NumLanes;
4146 if (NumLaneElems != 2 && NumLaneElems != 4)
4149 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4150 bool symmetricMaskRequired =
4151 (VT.getSizeInBits() >= 256) && (EltSize == 32);
4153 // VSHUFPSY divides the resulting vector into 4 chunks.
4154 // The sources are also splitted into 4 chunks, and each destination
4155 // chunk must come from a different source chunk.
4157 // SRC1 => X7 X6 X5 X4 X3 X2 X1 X0
4158 // SRC2 => Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y9
4160 // DST => Y7..Y4, Y7..Y4, X7..X4, X7..X4,
4161 // Y3..Y0, Y3..Y0, X3..X0, X3..X0
4163 // VSHUFPDY divides the resulting vector into 4 chunks.
4164 // The sources are also splitted into 4 chunks, and each destination
4165 // chunk must come from a different source chunk.
4167 // SRC1 => X3 X2 X1 X0
4168 // SRC2 => Y3 Y2 Y1 Y0
4170 // DST => Y3..Y2, X3..X2, Y1..Y0, X1..X0
4172 SmallVector<int, 4> MaskVal(NumLaneElems, -1);
4173 unsigned HalfLaneElems = NumLaneElems/2;
4174 for (unsigned l = 0; l != NumElems; l += NumLaneElems) {
4175 for (unsigned i = 0; i != NumLaneElems; ++i) {
4176 int Idx = Mask[i+l];
4177 unsigned RngStart = l + ((Commuted == (i<HalfLaneElems)) ? NumElems : 0);
4178 if (!isUndefOrInRange(Idx, RngStart, RngStart+NumLaneElems))
4180 // For VSHUFPSY, the mask of the second half must be the same as the
4181 // first but with the appropriate offsets. This works in the same way as
4182 // VPERMILPS works with masks.
4183 if (!symmetricMaskRequired || Idx < 0)
4185 if (MaskVal[i] < 0) {
4186 MaskVal[i] = Idx - l;
4189 if ((signed)(Idx - l) != MaskVal[i])
4197 /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
4198 /// specifies a shuffle of elements that is suitable for input to MOVHLPS.
4199 static bool isMOVHLPSMask(ArrayRef<int> Mask, MVT VT) {
4200 if (!VT.is128BitVector())
4203 unsigned NumElems = VT.getVectorNumElements();
4208 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
4209 return isUndefOrEqual(Mask[0], 6) &&
4210 isUndefOrEqual(Mask[1], 7) &&
4211 isUndefOrEqual(Mask[2], 2) &&
4212 isUndefOrEqual(Mask[3], 3);
4215 /// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
4216 /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
4218 static bool isMOVHLPS_v_undef_Mask(ArrayRef<int> Mask, MVT VT) {
4219 if (!VT.is128BitVector())
4222 unsigned NumElems = VT.getVectorNumElements();
4227 return isUndefOrEqual(Mask[0], 2) &&
4228 isUndefOrEqual(Mask[1], 3) &&
4229 isUndefOrEqual(Mask[2], 2) &&
4230 isUndefOrEqual(Mask[3], 3);
4233 /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
4234 /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
4235 static bool isMOVLPMask(ArrayRef<int> Mask, MVT VT) {
4236 if (!VT.is128BitVector())
4239 unsigned NumElems = VT.getVectorNumElements();
4241 if (NumElems != 2 && NumElems != 4)
4244 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
4245 if (!isUndefOrEqual(Mask[i], i + NumElems))
4248 for (unsigned i = NumElems/2, e = NumElems; i != e; ++i)
4249 if (!isUndefOrEqual(Mask[i], i))
4255 /// isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand
4256 /// specifies a shuffle of elements that is suitable for input to MOVLHPS.
4257 static bool isMOVLHPSMask(ArrayRef<int> Mask, MVT VT) {
4258 if (!VT.is128BitVector())
4261 unsigned NumElems = VT.getVectorNumElements();
4263 if (NumElems != 2 && NumElems != 4)
4266 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
4267 if (!isUndefOrEqual(Mask[i], i))
4270 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
4271 if (!isUndefOrEqual(Mask[i + e], i + NumElems))
4277 /// isINSERTPSMask - Return true if the specified VECTOR_SHUFFLE operand
4278 /// specifies a shuffle of elements that is suitable for input to INSERTPS.
4279 /// i. e: If all but one element come from the same vector.
4280 static bool isINSERTPSMask(ArrayRef<int> Mask, MVT VT) {
4281 // TODO: Deal with AVX's VINSERTPS
4282 if (!VT.is128BitVector() || (VT != MVT::v4f32 && VT != MVT::v4i32))
4285 unsigned CorrectPosV1 = 0;
4286 unsigned CorrectPosV2 = 0;
4287 for (int i = 0, e = (int)VT.getVectorNumElements(); i != e; ++i) {
4288 if (Mask[i] == -1) {
4296 else if (Mask[i] == i + 4)
4300 if (CorrectPosV1 == 3 || CorrectPosV2 == 3)
4301 // We have 3 elements (undefs count as elements from any vector) from one
4302 // vector, and one from another.
4309 // Some special combinations that can be optimized.
4312 SDValue Compact8x32ShuffleNode(ShuffleVectorSDNode *SVOp,
4313 SelectionDAG &DAG) {
4314 MVT VT = SVOp->getSimpleValueType(0);
4317 if (VT != MVT::v8i32 && VT != MVT::v8f32)
4320 ArrayRef<int> Mask = SVOp->getMask();
4322 // These are the special masks that may be optimized.
4323 static const int MaskToOptimizeEven[] = {0, 8, 2, 10, 4, 12, 6, 14};
4324 static const int MaskToOptimizeOdd[] = {1, 9, 3, 11, 5, 13, 7, 15};
4325 bool MatchEvenMask = true;
4326 bool MatchOddMask = true;
4327 for (int i=0; i<8; ++i) {
4328 if (!isUndefOrEqual(Mask[i], MaskToOptimizeEven[i]))
4329 MatchEvenMask = false;
4330 if (!isUndefOrEqual(Mask[i], MaskToOptimizeOdd[i]))
4331 MatchOddMask = false;
4334 if (!MatchEvenMask && !MatchOddMask)
4337 SDValue UndefNode = DAG.getNode(ISD::UNDEF, dl, VT);
4339 SDValue Op0 = SVOp->getOperand(0);
4340 SDValue Op1 = SVOp->getOperand(1);
4342 if (MatchEvenMask) {
4343 // Shift the second operand right to 32 bits.
4344 static const int ShiftRightMask[] = {-1, 0, -1, 2, -1, 4, -1, 6 };
4345 Op1 = DAG.getVectorShuffle(VT, dl, Op1, UndefNode, ShiftRightMask);
4347 // Shift the first operand left to 32 bits.
4348 static const int ShiftLeftMask[] = {1, -1, 3, -1, 5, -1, 7, -1 };
4349 Op0 = DAG.getVectorShuffle(VT, dl, Op0, UndefNode, ShiftLeftMask);
4351 static const int BlendMask[] = {0, 9, 2, 11, 4, 13, 6, 15};
4352 return DAG.getVectorShuffle(VT, dl, Op0, Op1, BlendMask);
4355 /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
4356 /// specifies a shuffle of elements that is suitable for input to UNPCKL.
4357 static bool isUNPCKLMask(ArrayRef<int> Mask, MVT VT,
4358 bool HasInt256, bool V2IsSplat = false) {
4360 assert(VT.getSizeInBits() >= 128 &&
4361 "Unsupported vector type for unpckl");
4363 unsigned NumElts = VT.getVectorNumElements();
4364 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
4365 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4368 assert((!VT.is512BitVector() || VT.getScalarType().getSizeInBits() >= 32) &&
4369 "Unsupported vector type for unpckh");
4371 // AVX defines UNPCK* to operate independently on 128-bit lanes.
4372 unsigned NumLanes = VT.getSizeInBits()/128;
4373 unsigned NumLaneElts = NumElts/NumLanes;
4375 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4376 for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) {
4377 int BitI = Mask[l+i];
4378 int BitI1 = Mask[l+i+1];
4379 if (!isUndefOrEqual(BitI, j))
4382 if (!isUndefOrEqual(BitI1, NumElts))
4385 if (!isUndefOrEqual(BitI1, j + NumElts))
4394 /// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
4395 /// specifies a shuffle of elements that is suitable for input to UNPCKH.
4396 static bool isUNPCKHMask(ArrayRef<int> Mask, MVT VT,
4397 bool HasInt256, bool V2IsSplat = false) {
4398 assert(VT.getSizeInBits() >= 128 &&
4399 "Unsupported vector type for unpckh");
4401 unsigned NumElts = VT.getVectorNumElements();
4402 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
4403 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4406 assert((!VT.is512BitVector() || VT.getScalarType().getSizeInBits() >= 32) &&
4407 "Unsupported vector type for unpckh");
4409 // AVX defines UNPCK* to operate independently on 128-bit lanes.
4410 unsigned NumLanes = VT.getSizeInBits()/128;
4411 unsigned NumLaneElts = NumElts/NumLanes;
4413 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4414 for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) {
4415 int BitI = Mask[l+i];
4416 int BitI1 = Mask[l+i+1];
4417 if (!isUndefOrEqual(BitI, j))
4420 if (isUndefOrEqual(BitI1, NumElts))
4423 if (!isUndefOrEqual(BitI1, j+NumElts))
4431 /// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
4432 /// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
4434 static bool isUNPCKL_v_undef_Mask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
4435 unsigned NumElts = VT.getVectorNumElements();
4436 bool Is256BitVec = VT.is256BitVector();
4438 if (VT.is512BitVector())
4440 assert((VT.is128BitVector() || VT.is256BitVector()) &&
4441 "Unsupported vector type for unpckh");
4443 if (Is256BitVec && NumElts != 4 && NumElts != 8 &&
4444 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4447 // For 256-bit i64/f64, use MOVDDUPY instead, so reject the matching pattern
4448 // FIXME: Need a better way to get rid of this, there's no latency difference
4449 // between UNPCKLPD and MOVDDUP, the later should always be checked first and
4450 // the former later. We should also remove the "_undef" special mask.
4451 if (NumElts == 4 && Is256BitVec)
4454 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
4455 // independently on 128-bit lanes.
4456 unsigned NumLanes = VT.getSizeInBits()/128;
4457 unsigned NumLaneElts = NumElts/NumLanes;
4459 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4460 for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) {
4461 int BitI = Mask[l+i];
4462 int BitI1 = Mask[l+i+1];
4464 if (!isUndefOrEqual(BitI, j))
4466 if (!isUndefOrEqual(BitI1, j))
4474 /// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form
4475 /// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef,
4477 static bool isUNPCKH_v_undef_Mask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
4478 unsigned NumElts = VT.getVectorNumElements();
4480 if (VT.is512BitVector())
4483 assert((VT.is128BitVector() || VT.is256BitVector()) &&
4484 "Unsupported vector type for unpckh");
4486 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
4487 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4490 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
4491 // independently on 128-bit lanes.
4492 unsigned NumLanes = VT.getSizeInBits()/128;
4493 unsigned NumLaneElts = NumElts/NumLanes;
4495 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4496 for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) {
4497 int BitI = Mask[l+i];
4498 int BitI1 = Mask[l+i+1];
4499 if (!isUndefOrEqual(BitI, j))
4501 if (!isUndefOrEqual(BitI1, j))
4508 // Match for INSERTI64x4 INSERTF64x4 instructions (src0[0], src1[0]) or
4509 // (src1[0], src0[1]), manipulation with 256-bit sub-vectors
4510 static bool isINSERT64x4Mask(ArrayRef<int> Mask, MVT VT, unsigned int *Imm) {
4511 if (!VT.is512BitVector())
4514 unsigned NumElts = VT.getVectorNumElements();
4515 unsigned HalfSize = NumElts/2;
4516 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, 0)) {
4517 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, NumElts)) {
4522 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, NumElts)) {
4523 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, HalfSize)) {
4531 /// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
4532 /// specifies a shuffle of elements that is suitable for input to MOVSS,
4533 /// MOVSD, and MOVD, i.e. setting the lowest element.
4534 static bool isMOVLMask(ArrayRef<int> Mask, EVT VT) {
4535 if (VT.getVectorElementType().getSizeInBits() < 32)
4537 if (!VT.is128BitVector())
4540 unsigned NumElts = VT.getVectorNumElements();
4542 if (!isUndefOrEqual(Mask[0], NumElts))
4545 for (unsigned i = 1; i != NumElts; ++i)
4546 if (!isUndefOrEqual(Mask[i], i))
4552 /// isVPERM2X128Mask - Match 256-bit shuffles where the elements are considered
4553 /// as permutations between 128-bit chunks or halves. As an example: this
4555 /// vector_shuffle <4, 5, 6, 7, 12, 13, 14, 15>
4556 /// The first half comes from the second half of V1 and the second half from the
4557 /// the second half of V2.
4558 static bool isVPERM2X128Mask(ArrayRef<int> Mask, MVT VT, bool HasFp256) {
4559 if (!HasFp256 || !VT.is256BitVector())
4562 // The shuffle result is divided into half A and half B. In total the two
4563 // sources have 4 halves, namely: C, D, E, F. The final values of A and
4564 // B must come from C, D, E or F.
4565 unsigned HalfSize = VT.getVectorNumElements()/2;
4566 bool MatchA = false, MatchB = false;
4568 // Check if A comes from one of C, D, E, F.
4569 for (unsigned Half = 0; Half != 4; ++Half) {
4570 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, Half*HalfSize)) {
4576 // Check if B comes from one of C, D, E, F.
4577 for (unsigned Half = 0; Half != 4; ++Half) {
4578 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, Half*HalfSize)) {
4584 return MatchA && MatchB;
4587 /// getShuffleVPERM2X128Immediate - Return the appropriate immediate to shuffle
4588 /// the specified VECTOR_MASK mask with VPERM2F128/VPERM2I128 instructions.
4589 static unsigned getShuffleVPERM2X128Immediate(ShuffleVectorSDNode *SVOp) {
4590 MVT VT = SVOp->getSimpleValueType(0);
4592 unsigned HalfSize = VT.getVectorNumElements()/2;
4594 unsigned FstHalf = 0, SndHalf = 0;
4595 for (unsigned i = 0; i < HalfSize; ++i) {
4596 if (SVOp->getMaskElt(i) > 0) {
4597 FstHalf = SVOp->getMaskElt(i)/HalfSize;
4601 for (unsigned i = HalfSize; i < HalfSize*2; ++i) {
4602 if (SVOp->getMaskElt(i) > 0) {
4603 SndHalf = SVOp->getMaskElt(i)/HalfSize;
4608 return (FstHalf | (SndHalf << 4));
4611 // Symmetric in-lane mask. Each lane has 4 elements (for imm8)
4612 static bool isPermImmMask(ArrayRef<int> Mask, MVT VT, unsigned& Imm8) {
4613 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4617 unsigned NumElts = VT.getVectorNumElements();
4619 if (VT.is128BitVector() || (VT.is256BitVector() && EltSize == 64)) {
4620 for (unsigned i = 0; i != NumElts; ++i) {
4623 Imm8 |= Mask[i] << (i*2);
4628 unsigned LaneSize = 4;
4629 SmallVector<int, 4> MaskVal(LaneSize, -1);
4631 for (unsigned l = 0; l != NumElts; l += LaneSize) {
4632 for (unsigned i = 0; i != LaneSize; ++i) {
4633 if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize))
4637 if (MaskVal[i] < 0) {
4638 MaskVal[i] = Mask[i+l] - l;
4639 Imm8 |= MaskVal[i] << (i*2);
4642 if (Mask[i+l] != (signed)(MaskVal[i]+l))
4649 /// isVPERMILPMask - Return true if the specified VECTOR_SHUFFLE operand
4650 /// specifies a shuffle of elements that is suitable for input to VPERMILPD*.
4651 /// Note that VPERMIL mask matching is different depending whether theunderlying
4652 /// type is 32 or 64. In the VPERMILPS the high half of the mask should point
4653 /// to the same elements of the low, but to the higher half of the source.
4654 /// In VPERMILPD the two lanes could be shuffled independently of each other
4655 /// with the same restriction that lanes can't be crossed. Also handles PSHUFDY.
4656 static bool isVPERMILPMask(ArrayRef<int> Mask, MVT VT) {
4657 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4658 if (VT.getSizeInBits() < 256 || EltSize < 32)
4660 bool symmetricMaskRequired = (EltSize == 32);
4661 unsigned NumElts = VT.getVectorNumElements();
4663 unsigned NumLanes = VT.getSizeInBits()/128;
4664 unsigned LaneSize = NumElts/NumLanes;
4665 // 2 or 4 elements in one lane
4667 SmallVector<int, 4> ExpectedMaskVal(LaneSize, -1);
4668 for (unsigned l = 0; l != NumElts; l += LaneSize) {
4669 for (unsigned i = 0; i != LaneSize; ++i) {
4670 if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize))
4672 if (symmetricMaskRequired) {
4673 if (ExpectedMaskVal[i] < 0 && Mask[i+l] >= 0) {
4674 ExpectedMaskVal[i] = Mask[i+l] - l;
4677 if (!isUndefOrEqual(Mask[i+l], ExpectedMaskVal[i]+l))
4685 /// isCommutedMOVLMask - Returns true if the shuffle mask is except the reverse
4686 /// of what x86 movss want. X86 movs requires the lowest element to be lowest
4687 /// element of vector 2 and the other elements to come from vector 1 in order.
4688 static bool isCommutedMOVLMask(ArrayRef<int> Mask, MVT VT,
4689 bool V2IsSplat = false, bool V2IsUndef = false) {
4690 if (!VT.is128BitVector())
4693 unsigned NumOps = VT.getVectorNumElements();
4694 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
4697 if (!isUndefOrEqual(Mask[0], 0))
4700 for (unsigned i = 1; i != NumOps; ++i)
4701 if (!(isUndefOrEqual(Mask[i], i+NumOps) ||
4702 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) ||
4703 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps))))
4709 /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
4710 /// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
4711 /// Masks to match: <1, 1, 3, 3> or <1, 1, 3, 3, 5, 5, 7, 7>
4712 static bool isMOVSHDUPMask(ArrayRef<int> Mask, MVT VT,
4713 const X86Subtarget *Subtarget) {
4714 if (!Subtarget->hasSSE3())
4717 unsigned NumElems = VT.getVectorNumElements();
4719 if ((VT.is128BitVector() && NumElems != 4) ||
4720 (VT.is256BitVector() && NumElems != 8) ||
4721 (VT.is512BitVector() && NumElems != 16))
4724 // "i+1" is the value the indexed mask element must have
4725 for (unsigned i = 0; i != NumElems; i += 2)
4726 if (!isUndefOrEqual(Mask[i], i+1) ||
4727 !isUndefOrEqual(Mask[i+1], i+1))
4733 /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
4734 /// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
4735 /// Masks to match: <0, 0, 2, 2> or <0, 0, 2, 2, 4, 4, 6, 6>
4736 static bool isMOVSLDUPMask(ArrayRef<int> Mask, MVT VT,
4737 const X86Subtarget *Subtarget) {
4738 if (!Subtarget->hasSSE3())
4741 unsigned NumElems = VT.getVectorNumElements();
4743 if ((VT.is128BitVector() && NumElems != 4) ||
4744 (VT.is256BitVector() && NumElems != 8) ||
4745 (VT.is512BitVector() && NumElems != 16))
4748 // "i" is the value the indexed mask element must have
4749 for (unsigned i = 0; i != NumElems; i += 2)
4750 if (!isUndefOrEqual(Mask[i], i) ||
4751 !isUndefOrEqual(Mask[i+1], i))
4757 /// isMOVDDUPYMask - Return true if the specified VECTOR_SHUFFLE operand
4758 /// specifies a shuffle of elements that is suitable for input to 256-bit
4759 /// version of MOVDDUP.
4760 static bool isMOVDDUPYMask(ArrayRef<int> Mask, MVT VT, bool HasFp256) {
4761 if (!HasFp256 || !VT.is256BitVector())
4764 unsigned NumElts = VT.getVectorNumElements();
4768 for (unsigned i = 0; i != NumElts/2; ++i)
4769 if (!isUndefOrEqual(Mask[i], 0))
4771 for (unsigned i = NumElts/2; i != NumElts; ++i)
4772 if (!isUndefOrEqual(Mask[i], NumElts/2))
4777 /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
4778 /// specifies a shuffle of elements that is suitable for input to 128-bit
4779 /// version of MOVDDUP.
4780 static bool isMOVDDUPMask(ArrayRef<int> Mask, MVT VT) {
4781 if (!VT.is128BitVector())
4784 unsigned e = VT.getVectorNumElements() / 2;
4785 for (unsigned i = 0; i != e; ++i)
4786 if (!isUndefOrEqual(Mask[i], i))
4788 for (unsigned i = 0; i != e; ++i)
4789 if (!isUndefOrEqual(Mask[e+i], i))
4794 /// isVEXTRACTIndex - Return true if the specified
4795 /// EXTRACT_SUBVECTOR operand specifies a vector extract that is
4796 /// suitable for instruction that extract 128 or 256 bit vectors
4797 static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) {
4798 assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
4799 if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
4802 // The index should be aligned on a vecWidth-bit boundary.
4804 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
4806 MVT VT = N->getSimpleValueType(0);
4807 unsigned ElSize = VT.getVectorElementType().getSizeInBits();
4808 bool Result = (Index * ElSize) % vecWidth == 0;
4813 /// isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR
4814 /// operand specifies a subvector insert that is suitable for input to
4815 /// insertion of 128 or 256-bit subvectors
4816 static bool isVINSERTIndex(SDNode *N, unsigned vecWidth) {
4817 assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
4818 if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
4820 // The index should be aligned on a vecWidth-bit boundary.
4822 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
4824 MVT VT = N->getSimpleValueType(0);
4825 unsigned ElSize = VT.getVectorElementType().getSizeInBits();
4826 bool Result = (Index * ElSize) % vecWidth == 0;
4831 bool X86::isVINSERT128Index(SDNode *N) {
4832 return isVINSERTIndex(N, 128);
4835 bool X86::isVINSERT256Index(SDNode *N) {
4836 return isVINSERTIndex(N, 256);
4839 bool X86::isVEXTRACT128Index(SDNode *N) {
4840 return isVEXTRACTIndex(N, 128);
4843 bool X86::isVEXTRACT256Index(SDNode *N) {
4844 return isVEXTRACTIndex(N, 256);
4847 /// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
4848 /// the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions.
4849 /// Handles 128-bit and 256-bit.
4850 static unsigned getShuffleSHUFImmediate(ShuffleVectorSDNode *N) {
4851 MVT VT = N->getSimpleValueType(0);
4853 assert((VT.getSizeInBits() >= 128) &&
4854 "Unsupported vector type for PSHUF/SHUFP");
4856 // Handle 128 and 256-bit vector lengths. AVX defines PSHUF/SHUFP to operate
4857 // independently on 128-bit lanes.
4858 unsigned NumElts = VT.getVectorNumElements();
4859 unsigned NumLanes = VT.getSizeInBits()/128;
4860 unsigned NumLaneElts = NumElts/NumLanes;
4862 assert((NumLaneElts == 2 || NumLaneElts == 4 || NumLaneElts == 8) &&
4863 "Only supports 2, 4 or 8 elements per lane");
4865 unsigned Shift = (NumLaneElts >= 4) ? 1 : 0;
4867 for (unsigned i = 0; i != NumElts; ++i) {
4868 int Elt = N->getMaskElt(i);
4869 if (Elt < 0) continue;
4870 Elt &= NumLaneElts - 1;
4871 unsigned ShAmt = (i << Shift) % 8;
4872 Mask |= Elt << ShAmt;
4878 /// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle
4879 /// the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction.
4880 static unsigned getShufflePSHUFHWImmediate(ShuffleVectorSDNode *N) {
4881 MVT VT = N->getSimpleValueType(0);
4883 assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
4884 "Unsupported vector type for PSHUFHW");
4886 unsigned NumElts = VT.getVectorNumElements();
4889 for (unsigned l = 0; l != NumElts; l += 8) {
4890 // 8 nodes per lane, but we only care about the last 4.
4891 for (unsigned i = 0; i < 4; ++i) {
4892 int Elt = N->getMaskElt(l+i+4);
4893 if (Elt < 0) continue;
4894 Elt &= 0x3; // only 2-bits.
4895 Mask |= Elt << (i * 2);
4902 /// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle
4903 /// the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction.
4904 static unsigned getShufflePSHUFLWImmediate(ShuffleVectorSDNode *N) {
4905 MVT VT = N->getSimpleValueType(0);
4907 assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
4908 "Unsupported vector type for PSHUFHW");
4910 unsigned NumElts = VT.getVectorNumElements();
4913 for (unsigned l = 0; l != NumElts; l += 8) {
4914 // 8 nodes per lane, but we only care about the first 4.
4915 for (unsigned i = 0; i < 4; ++i) {
4916 int Elt = N->getMaskElt(l+i);
4917 if (Elt < 0) continue;
4918 Elt &= 0x3; // only 2-bits
4919 Mask |= Elt << (i * 2);
4926 /// \brief Return the appropriate immediate to shuffle the specified
4927 /// VECTOR_SHUFFLE mask with the PALIGNR (if InterLane is false) or with
4928 /// VALIGN (if Interlane is true) instructions.
4929 static unsigned getShuffleAlignrImmediate(ShuffleVectorSDNode *SVOp,
4931 MVT VT = SVOp->getSimpleValueType(0);
4932 unsigned EltSize = InterLane ? 1 :
4933 VT.getVectorElementType().getSizeInBits() >> 3;
4935 unsigned NumElts = VT.getVectorNumElements();
4936 unsigned NumLanes = VT.is512BitVector() ? 1 : VT.getSizeInBits()/128;
4937 unsigned NumLaneElts = NumElts/NumLanes;
4941 for (i = 0; i != NumElts; ++i) {
4942 Val = SVOp->getMaskElt(i);
4946 if (Val >= (int)NumElts)
4947 Val -= NumElts - NumLaneElts;
4949 assert(Val - i > 0 && "PALIGNR imm should be positive");
4950 return (Val - i) * EltSize;
4953 /// \brief Return the appropriate immediate to shuffle the specified
4954 /// VECTOR_SHUFFLE mask with the PALIGNR instruction.
4955 static unsigned getShufflePALIGNRImmediate(ShuffleVectorSDNode *SVOp) {
4956 return getShuffleAlignrImmediate(SVOp, false);
4959 /// \brief Return the appropriate immediate to shuffle the specified
4960 /// VECTOR_SHUFFLE mask with the VALIGN instruction.
4961 static unsigned getShuffleVALIGNImmediate(ShuffleVectorSDNode *SVOp) {
4962 return getShuffleAlignrImmediate(SVOp, true);
4966 static unsigned getExtractVEXTRACTImmediate(SDNode *N, unsigned vecWidth) {
4967 assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width");
4968 if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
4969 llvm_unreachable("Illegal extract subvector for VEXTRACT");
4972 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
4974 MVT VecVT = N->getOperand(0).getSimpleValueType();
4975 MVT ElVT = VecVT.getVectorElementType();
4977 unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
4978 return Index / NumElemsPerChunk;
4981 static unsigned getInsertVINSERTImmediate(SDNode *N, unsigned vecWidth) {
4982 assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width");
4983 if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
4984 llvm_unreachable("Illegal insert subvector for VINSERT");
4987 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
4989 MVT VecVT = N->getSimpleValueType(0);
4990 MVT ElVT = VecVT.getVectorElementType();
4992 unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
4993 return Index / NumElemsPerChunk;
4996 /// getExtractVEXTRACT128Immediate - Return the appropriate immediate
4997 /// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128
4998 /// and VINSERTI128 instructions.
4999 unsigned X86::getExtractVEXTRACT128Immediate(SDNode *N) {
5000 return getExtractVEXTRACTImmediate(N, 128);
5003 /// getExtractVEXTRACT256Immediate - Return the appropriate immediate
5004 /// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF64x4
5005 /// and VINSERTI64x4 instructions.
5006 unsigned X86::getExtractVEXTRACT256Immediate(SDNode *N) {
5007 return getExtractVEXTRACTImmediate(N, 256);
5010 /// getInsertVINSERT128Immediate - Return the appropriate immediate
5011 /// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128
5012 /// and VINSERTI128 instructions.
5013 unsigned X86::getInsertVINSERT128Immediate(SDNode *N) {
5014 return getInsertVINSERTImmediate(N, 128);
5017 /// getInsertVINSERT256Immediate - Return the appropriate immediate
5018 /// to insert at the specified INSERT_SUBVECTOR index with VINSERTF46x4
5019 /// and VINSERTI64x4 instructions.
5020 unsigned X86::getInsertVINSERT256Immediate(SDNode *N) {
5021 return getInsertVINSERTImmediate(N, 256);
5024 /// isZero - Returns true if Elt is a constant integer zero
5025 static bool isZero(SDValue V) {
5026 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
5027 return C && C->isNullValue();
5030 /// isZeroNode - Returns true if Elt is a constant zero or a floating point
5032 bool X86::isZeroNode(SDValue Elt) {
5035 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Elt))
5036 return CFP->getValueAPF().isPosZero();
5040 /// ShouldXformToMOVHLPS - Return true if the node should be transformed to
5041 /// match movhlps. The lower half elements should come from upper half of
5042 /// V1 (and in order), and the upper half elements should come from the upper
5043 /// half of V2 (and in order).
5044 static bool ShouldXformToMOVHLPS(ArrayRef<int> Mask, MVT VT) {
5045 if (!VT.is128BitVector())
5047 if (VT.getVectorNumElements() != 4)
5049 for (unsigned i = 0, e = 2; i != e; ++i)
5050 if (!isUndefOrEqual(Mask[i], i+2))
5052 for (unsigned i = 2; i != 4; ++i)
5053 if (!isUndefOrEqual(Mask[i], i+4))
5058 /// isScalarLoadToVector - Returns true if the node is a scalar load that
5059 /// is promoted to a vector. It also returns the LoadSDNode by reference if
5061 static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = nullptr) {
5062 if (N->getOpcode() != ISD::SCALAR_TO_VECTOR)
5064 N = N->getOperand(0).getNode();
5065 if (!ISD::isNON_EXTLoad(N))
5068 *LD = cast<LoadSDNode>(N);
5072 // Test whether the given value is a vector value which will be legalized
5074 static bool WillBeConstantPoolLoad(SDNode *N) {
5075 if (N->getOpcode() != ISD::BUILD_VECTOR)
5078 // Check for any non-constant elements.
5079 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
5080 switch (N->getOperand(i).getNode()->getOpcode()) {
5082 case ISD::ConstantFP:
5089 // Vectors of all-zeros and all-ones are materialized with special
5090 // instructions rather than being loaded.
5091 return !ISD::isBuildVectorAllZeros(N) &&
5092 !ISD::isBuildVectorAllOnes(N);
5095 /// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to
5096 /// match movlp{s|d}. The lower half elements should come from lower half of
5097 /// V1 (and in order), and the upper half elements should come from the upper
5098 /// half of V2 (and in order). And since V1 will become the source of the
5099 /// MOVLP, it must be either a vector load or a scalar load to vector.
5100 static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
5101 ArrayRef<int> Mask, MVT VT) {
5102 if (!VT.is128BitVector())
5105 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
5107 // Is V2 is a vector load, don't do this transformation. We will try to use
5108 // load folding shufps op.
5109 if (ISD::isNON_EXTLoad(V2) || WillBeConstantPoolLoad(V2))
5112 unsigned NumElems = VT.getVectorNumElements();
5114 if (NumElems != 2 && NumElems != 4)
5116 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
5117 if (!isUndefOrEqual(Mask[i], i))
5119 for (unsigned i = NumElems/2, e = NumElems; i != e; ++i)
5120 if (!isUndefOrEqual(Mask[i], i+NumElems))
5125 /// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
5126 /// to an zero vector.
5127 /// FIXME: move to dag combiner / method on ShuffleVectorSDNode
5128 static bool isZeroShuffle(ShuffleVectorSDNode *N) {
5129 SDValue V1 = N->getOperand(0);
5130 SDValue V2 = N->getOperand(1);
5131 unsigned NumElems = N->getValueType(0).getVectorNumElements();
5132 for (unsigned i = 0; i != NumElems; ++i) {
5133 int Idx = N->getMaskElt(i);
5134 if (Idx >= (int)NumElems) {
5135 unsigned Opc = V2.getOpcode();
5136 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
5138 if (Opc != ISD::BUILD_VECTOR ||
5139 !X86::isZeroNode(V2.getOperand(Idx-NumElems)))
5141 } else if (Idx >= 0) {
5142 unsigned Opc = V1.getOpcode();
5143 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode()))
5145 if (Opc != ISD::BUILD_VECTOR ||
5146 !X86::isZeroNode(V1.getOperand(Idx)))
5153 /// getZeroVector - Returns a vector of specified type with all zero elements.
5155 static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget,
5156 SelectionDAG &DAG, SDLoc dl) {
5157 assert(VT.isVector() && "Expected a vector type");
5159 // Always build SSE zero vectors as <4 x i32> bitcasted
5160 // to their dest type. This ensures they get CSE'd.
5162 if (VT.is128BitVector()) { // SSE
5163 if (Subtarget->hasSSE2()) { // SSE2
5164 SDValue Cst = DAG.getConstant(0, MVT::i32);
5165 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
5167 SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32);
5168 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst);
5170 } else if (VT.is256BitVector()) { // AVX
5171 if (Subtarget->hasInt256()) { // AVX2
5172 SDValue Cst = DAG.getConstant(0, MVT::i32);
5173 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5174 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
5176 // 256-bit logic and arithmetic instructions in AVX are all
5177 // floating-point, no support for integer ops. Emit fp zeroed vectors.
5178 SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32);
5179 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5180 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops);
5182 } else if (VT.is512BitVector()) { // AVX-512
5183 SDValue Cst = DAG.getConstant(0, MVT::i32);
5184 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst,
5185 Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5186 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops);
5187 } else if (VT.getScalarType() == MVT::i1) {
5188 assert(VT.getVectorNumElements() <= 16 && "Unexpected vector type");
5189 SDValue Cst = DAG.getConstant(0, MVT::i1);
5190 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
5191 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
5193 llvm_unreachable("Unexpected vector type");
5195 return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
5198 /// getOnesVector - Returns a vector of specified type with all bits set.
5199 /// Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with
5200 /// no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately.
5201 /// Then bitcast to their original type, ensuring they get CSE'd.
5202 static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG,
5204 assert(VT.isVector() && "Expected a vector type");
5206 SDValue Cst = DAG.getConstant(~0U, MVT::i32);
5208 if (VT.is256BitVector()) {
5209 if (HasInt256) { // AVX2
5210 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5211 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
5213 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
5214 Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl);
5216 } else if (VT.is128BitVector()) {
5217 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
5219 llvm_unreachable("Unexpected vector type");
5221 return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
5224 /// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements
5225 /// that point to V2 points to its first element.
5226 static void NormalizeMask(SmallVectorImpl<int> &Mask, unsigned NumElems) {
5227 for (unsigned i = 0; i != NumElems; ++i) {
5228 if (Mask[i] > (int)NumElems) {
5234 /// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
5235 /// operation of specified width.
5236 static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
5238 unsigned NumElems = VT.getVectorNumElements();
5239 SmallVector<int, 8> Mask;
5240 Mask.push_back(NumElems);
5241 for (unsigned i = 1; i != NumElems; ++i)
5243 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
5246 /// getUnpackl - Returns a vector_shuffle node for an unpackl operation.
5247 static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
5249 unsigned NumElems = VT.getVectorNumElements();
5250 SmallVector<int, 8> Mask;
5251 for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
5253 Mask.push_back(i + NumElems);
5255 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
5258 /// getUnpackh - Returns a vector_shuffle node for an unpackh operation.
5259 static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
5261 unsigned NumElems = VT.getVectorNumElements();
5262 SmallVector<int, 8> Mask;
5263 for (unsigned i = 0, Half = NumElems/2; i != Half; ++i) {
5264 Mask.push_back(i + Half);
5265 Mask.push_back(i + NumElems + Half);
5267 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
5270 // PromoteSplati8i16 - All i16 and i8 vector types can't be used directly by
5271 // a generic shuffle instruction because the target has no such instructions.
5272 // Generate shuffles which repeat i16 and i8 several times until they can be
5273 // represented by v4f32 and then be manipulated by target suported shuffles.
5274 static SDValue PromoteSplati8i16(SDValue V, SelectionDAG &DAG, int &EltNo) {
5275 MVT VT = V.getSimpleValueType();
5276 int NumElems = VT.getVectorNumElements();
5279 while (NumElems > 4) {
5280 if (EltNo < NumElems/2) {
5281 V = getUnpackl(DAG, dl, VT, V, V);
5283 V = getUnpackh(DAG, dl, VT, V, V);
5284 EltNo -= NumElems/2;
5291 /// getLegalSplat - Generate a legal splat with supported x86 shuffles
5292 static SDValue getLegalSplat(SelectionDAG &DAG, SDValue V, int EltNo) {
5293 MVT VT = V.getSimpleValueType();
5296 if (VT.is128BitVector()) {
5297 V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V);
5298 int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
5299 V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32),
5301 } else if (VT.is256BitVector()) {
5302 // To use VPERMILPS to splat scalars, the second half of indicies must
5303 // refer to the higher part, which is a duplication of the lower one,
5304 // because VPERMILPS can only handle in-lane permutations.
5305 int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo,
5306 EltNo+4, EltNo+4, EltNo+4, EltNo+4 };
5308 V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V);
5309 V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32),
5312 llvm_unreachable("Vector size not supported");
5314 return DAG.getNode(ISD::BITCAST, dl, VT, V);
5317 /// PromoteSplat - Splat is promoted to target supported vector shuffles.
5318 static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) {
5319 MVT SrcVT = SV->getSimpleValueType(0);
5320 SDValue V1 = SV->getOperand(0);
5323 int EltNo = SV->getSplatIndex();
5324 int NumElems = SrcVT.getVectorNumElements();
5325 bool Is256BitVec = SrcVT.is256BitVector();
5327 assert(((SrcVT.is128BitVector() && NumElems > 4) || Is256BitVec) &&
5328 "Unknown how to promote splat for type");
5330 // Extract the 128-bit part containing the splat element and update
5331 // the splat element index when it refers to the higher register.
5333 V1 = Extract128BitVector(V1, EltNo, DAG, dl);
5334 if (EltNo >= NumElems/2)
5335 EltNo -= NumElems/2;
5338 // All i16 and i8 vector types can't be used directly by a generic shuffle
5339 // instruction because the target has no such instruction. Generate shuffles
5340 // which repeat i16 and i8 several times until they fit in i32, and then can
5341 // be manipulated by target suported shuffles.
5342 MVT EltVT = SrcVT.getVectorElementType();
5343 if (EltVT == MVT::i8 || EltVT == MVT::i16)
5344 V1 = PromoteSplati8i16(V1, DAG, EltNo);
5346 // Recreate the 256-bit vector and place the same 128-bit vector
5347 // into the low and high part. This is necessary because we want
5348 // to use VPERM* to shuffle the vectors
5350 V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, V1, V1);
5353 return getLegalSplat(DAG, V1, EltNo);
5356 /// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
5357 /// vector of zero or undef vector. This produces a shuffle where the low
5358 /// element of V2 is swizzled into the zero/undef vector, landing at element
5359 /// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
5360 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
5362 const X86Subtarget *Subtarget,
5363 SelectionDAG &DAG) {
5364 MVT VT = V2.getSimpleValueType();
5366 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
5367 unsigned NumElems = VT.getVectorNumElements();
5368 SmallVector<int, 16> MaskVec;
5369 for (unsigned i = 0; i != NumElems; ++i)
5370 // If this is the insertion idx, put the low elt of V2 here.
5371 MaskVec.push_back(i == Idx ? NumElems : i);
5372 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]);
5375 /// getTargetShuffleMask - Calculates the shuffle mask corresponding to the
5376 /// target specific opcode. Returns true if the Mask could be calculated. Sets
5377 /// IsUnary to true if only uses one source. Note that this will set IsUnary for
5378 /// shuffles which use a single input multiple times, and in those cases it will
5379 /// adjust the mask to only have indices within that single input.
5380 static bool getTargetShuffleMask(SDNode *N, MVT VT,
5381 SmallVectorImpl<int> &Mask, bool &IsUnary) {
5382 unsigned NumElems = VT.getVectorNumElements();
5386 bool IsFakeUnary = false;
5387 switch(N->getOpcode()) {
5388 case X86ISD::BLENDI:
5389 ImmN = N->getOperand(N->getNumOperands()-1);
5390 DecodeBLENDMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5393 ImmN = N->getOperand(N->getNumOperands()-1);
5394 DecodeSHUFPMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5395 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5397 case X86ISD::UNPCKH:
5398 DecodeUNPCKHMask(VT, Mask);
5399 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5401 case X86ISD::UNPCKL:
5402 DecodeUNPCKLMask(VT, Mask);
5403 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5405 case X86ISD::MOVHLPS:
5406 DecodeMOVHLPSMask(NumElems, Mask);
5407 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5409 case X86ISD::MOVLHPS:
5410 DecodeMOVLHPSMask(NumElems, Mask);
5411 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5413 case X86ISD::PALIGNR:
5414 ImmN = N->getOperand(N->getNumOperands()-1);
5415 DecodePALIGNRMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5417 case X86ISD::PSHUFD:
5418 case X86ISD::VPERMILPI:
5419 ImmN = N->getOperand(N->getNumOperands()-1);
5420 DecodePSHUFMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5423 case X86ISD::PSHUFHW:
5424 ImmN = N->getOperand(N->getNumOperands()-1);
5425 DecodePSHUFHWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5428 case X86ISD::PSHUFLW:
5429 ImmN = N->getOperand(N->getNumOperands()-1);
5430 DecodePSHUFLWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5433 case X86ISD::PSHUFB: {
5435 SDValue MaskNode = N->getOperand(1);
5436 while (MaskNode->getOpcode() == ISD::BITCAST)
5437 MaskNode = MaskNode->getOperand(0);
5439 if (MaskNode->getOpcode() == ISD::BUILD_VECTOR) {
5440 // If we have a build-vector, then things are easy.
5441 EVT VT = MaskNode.getValueType();
5442 assert(VT.isVector() &&
5443 "Can't produce a non-vector with a build_vector!");
5444 if (!VT.isInteger())
5447 int NumBytesPerElement = VT.getVectorElementType().getSizeInBits() / 8;
5449 SmallVector<uint64_t, 32> RawMask;
5450 for (int i = 0, e = MaskNode->getNumOperands(); i < e; ++i) {
5451 SDValue Op = MaskNode->getOperand(i);
5452 if (Op->getOpcode() == ISD::UNDEF) {
5453 RawMask.push_back((uint64_t)SM_SentinelUndef);
5456 auto *CN = dyn_cast<ConstantSDNode>(Op.getNode());
5459 APInt MaskElement = CN->getAPIntValue();
5461 // We now have to decode the element which could be any integer size and
5462 // extract each byte of it.
5463 for (int j = 0; j < NumBytesPerElement; ++j) {
5464 // Note that this is x86 and so always little endian: the low byte is
5465 // the first byte of the mask.
5466 RawMask.push_back(MaskElement.getLoBits(8).getZExtValue());
5467 MaskElement = MaskElement.lshr(8);
5470 DecodePSHUFBMask(RawMask, Mask);
5474 auto *MaskLoad = dyn_cast<LoadSDNode>(MaskNode);
5478 SDValue Ptr = MaskLoad->getBasePtr();
5479 if (Ptr->getOpcode() == X86ISD::Wrapper)
5480 Ptr = Ptr->getOperand(0);
5482 auto *MaskCP = dyn_cast<ConstantPoolSDNode>(Ptr);
5483 if (!MaskCP || MaskCP->isMachineConstantPoolEntry())
5486 if (auto *C = dyn_cast<Constant>(MaskCP->getConstVal())) {
5487 DecodePSHUFBMask(C, Mask);
5495 case X86ISD::VPERMI:
5496 ImmN = N->getOperand(N->getNumOperands()-1);
5497 DecodeVPERMMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5502 DecodeScalarMoveMask(VT, /* IsLoad */ false, Mask);
5504 case X86ISD::VPERM2X128:
5505 ImmN = N->getOperand(N->getNumOperands()-1);
5506 DecodeVPERM2X128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5507 if (Mask.empty()) return false;
5509 case X86ISD::MOVSLDUP:
5510 DecodeMOVSLDUPMask(VT, Mask);
5513 case X86ISD::MOVSHDUP:
5514 DecodeMOVSHDUPMask(VT, Mask);
5517 case X86ISD::MOVDDUP:
5518 DecodeMOVDDUPMask(VT, Mask);
5521 case X86ISD::MOVLHPD:
5522 case X86ISD::MOVLPD:
5523 case X86ISD::MOVLPS:
5524 // Not yet implemented
5526 default: llvm_unreachable("unknown target shuffle node");
5529 // If we have a fake unary shuffle, the shuffle mask is spread across two
5530 // inputs that are actually the same node. Re-map the mask to always point
5531 // into the first input.
5534 if (M >= (int)Mask.size())
5540 /// getShuffleScalarElt - Returns the scalar element that will make up the ith
5541 /// element of the result of the vector shuffle.
5542 static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
5545 return SDValue(); // Limit search depth.
5547 SDValue V = SDValue(N, 0);
5548 EVT VT = V.getValueType();
5549 unsigned Opcode = V.getOpcode();
5551 // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
5552 if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
5553 int Elt = SV->getMaskElt(Index);
5556 return DAG.getUNDEF(VT.getVectorElementType());
5558 unsigned NumElems = VT.getVectorNumElements();
5559 SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0)
5560 : SV->getOperand(1);
5561 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1);
5564 // Recurse into target specific vector shuffles to find scalars.
5565 if (isTargetShuffle(Opcode)) {
5566 MVT ShufVT = V.getSimpleValueType();
5567 unsigned NumElems = ShufVT.getVectorNumElements();
5568 SmallVector<int, 16> ShuffleMask;
5571 if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary))
5574 int Elt = ShuffleMask[Index];
5576 return DAG.getUNDEF(ShufVT.getVectorElementType());
5578 SDValue NewV = (Elt < (int)NumElems) ? N->getOperand(0)
5580 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG,
5584 // Actual nodes that may contain scalar elements
5585 if (Opcode == ISD::BITCAST) {
5586 V = V.getOperand(0);
5587 EVT SrcVT = V.getValueType();
5588 unsigned NumElems = VT.getVectorNumElements();
5590 if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems)
5594 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
5595 return (Index == 0) ? V.getOperand(0)
5596 : DAG.getUNDEF(VT.getVectorElementType());
5598 if (V.getOpcode() == ISD::BUILD_VECTOR)
5599 return V.getOperand(Index);
5604 /// getNumOfConsecutiveZeros - Return the number of elements of a vector
5605 /// shuffle operation which come from a consecutively from a zero. The
5606 /// search can start in two different directions, from left or right.
5607 /// We count undefs as zeros until PreferredNum is reached.
5608 static unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp,
5609 unsigned NumElems, bool ZerosFromLeft,
5611 unsigned PreferredNum = -1U) {
5612 unsigned NumZeros = 0;
5613 for (unsigned i = 0; i != NumElems; ++i) {
5614 unsigned Index = ZerosFromLeft ? i : NumElems - i - 1;
5615 SDValue Elt = getShuffleScalarElt(SVOp, Index, DAG, 0);
5619 if (X86::isZeroNode(Elt))
5621 else if (Elt.getOpcode() == ISD::UNDEF) // Undef as zero up to PreferredNum.
5622 NumZeros = std::min(NumZeros + 1, PreferredNum);
5630 /// isShuffleMaskConsecutive - Check if the shuffle mask indicies [MaskI, MaskE)
5631 /// correspond consecutively to elements from one of the vector operands,
5632 /// starting from its index OpIdx. Also tell OpNum which source vector operand.
5634 bool isShuffleMaskConsecutive(ShuffleVectorSDNode *SVOp,
5635 unsigned MaskI, unsigned MaskE, unsigned OpIdx,
5636 unsigned NumElems, unsigned &OpNum) {
5637 bool SeenV1 = false;
5638 bool SeenV2 = false;
5640 for (unsigned i = MaskI; i != MaskE; ++i, ++OpIdx) {
5641 int Idx = SVOp->getMaskElt(i);
5642 // Ignore undef indicies
5646 if (Idx < (int)NumElems)
5651 // Only accept consecutive elements from the same vector
5652 if ((Idx % NumElems != OpIdx) || (SeenV1 && SeenV2))
5656 OpNum = SeenV1 ? 0 : 1;
5660 /// isVectorShiftRight - Returns true if the shuffle can be implemented as a
5661 /// logical left shift of a vector.
5662 static bool isVectorShiftRight(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
5663 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
5665 SVOp->getSimpleValueType(0).getVectorNumElements();
5666 unsigned NumZeros = getNumOfConsecutiveZeros(
5667 SVOp, NumElems, false /* check zeros from right */, DAG,
5668 SVOp->getMaskElt(0));
5674 // Considering the elements in the mask that are not consecutive zeros,
5675 // check if they consecutively come from only one of the source vectors.
5677 // V1 = {X, A, B, C} 0
5679 // vector_shuffle V1, V2 <1, 2, 3, X>
5681 if (!isShuffleMaskConsecutive(SVOp,
5682 0, // Mask Start Index
5683 NumElems-NumZeros, // Mask End Index(exclusive)
5684 NumZeros, // Where to start looking in the src vector
5685 NumElems, // Number of elements in vector
5686 OpSrc)) // Which source operand ?
5691 ShVal = SVOp->getOperand(OpSrc);
5695 /// isVectorShiftLeft - Returns true if the shuffle can be implemented as a
5696 /// logical left shift of a vector.
5697 static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
5698 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
5700 SVOp->getSimpleValueType(0).getVectorNumElements();
5701 unsigned NumZeros = getNumOfConsecutiveZeros(
5702 SVOp, NumElems, true /* check zeros from left */, DAG,
5703 NumElems - SVOp->getMaskElt(NumElems - 1) - 1);
5709 // Considering the elements in the mask that are not consecutive zeros,
5710 // check if they consecutively come from only one of the source vectors.
5712 // 0 { A, B, X, X } = V2
5714 // vector_shuffle V1, V2 <X, X, 4, 5>
5716 if (!isShuffleMaskConsecutive(SVOp,
5717 NumZeros, // Mask Start Index
5718 NumElems, // Mask End Index(exclusive)
5719 0, // Where to start looking in the src vector
5720 NumElems, // Number of elements in vector
5721 OpSrc)) // Which source operand ?
5726 ShVal = SVOp->getOperand(OpSrc);
5730 /// isVectorShift - Returns true if the shuffle can be implemented as a
5731 /// logical left or right shift of a vector.
5732 static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
5733 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
5734 // Although the logic below support any bitwidth size, there are no
5735 // shift instructions which handle more than 128-bit vectors.
5736 if (!SVOp->getSimpleValueType(0).is128BitVector())
5739 if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) ||
5740 isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt))
5746 /// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
5748 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
5749 unsigned NumNonZero, unsigned NumZero,
5751 const X86Subtarget* Subtarget,
5752 const TargetLowering &TLI) {
5759 for (unsigned i = 0; i < 16; ++i) {
5760 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
5761 if (ThisIsNonZero && First) {
5763 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
5765 V = DAG.getUNDEF(MVT::v8i16);
5770 SDValue ThisElt, LastElt;
5771 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
5772 if (LastIsNonZero) {
5773 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl,
5774 MVT::i16, Op.getOperand(i-1));
5776 if (ThisIsNonZero) {
5777 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
5778 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16,
5779 ThisElt, DAG.getConstant(8, MVT::i8));
5781 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt);
5785 if (ThisElt.getNode())
5786 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt,
5787 DAG.getIntPtrConstant(i/2));
5791 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V);
5794 /// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.
5796 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
5797 unsigned NumNonZero, unsigned NumZero,
5799 const X86Subtarget* Subtarget,
5800 const TargetLowering &TLI) {
5807 for (unsigned i = 0; i < 8; ++i) {
5808 bool isNonZero = (NonZeros & (1 << i)) != 0;
5812 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
5814 V = DAG.getUNDEF(MVT::v8i16);
5817 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
5818 MVT::v8i16, V, Op.getOperand(i),
5819 DAG.getIntPtrConstant(i));
5826 /// LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32.
5827 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
5828 const X86Subtarget *Subtarget,
5829 const TargetLowering &TLI) {
5830 // Find all zeroable elements.
5832 for (int i=0; i < 4; ++i) {
5833 SDValue Elt = Op->getOperand(i);
5834 Zeroable[i] = (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt));
5836 assert(std::count_if(&Zeroable[0], &Zeroable[4],
5837 [](bool M) { return !M; }) > 1 &&
5838 "We expect at least two non-zero elements!");
5840 // We only know how to deal with build_vector nodes where elements are either
5841 // zeroable or extract_vector_elt with constant index.
5842 SDValue FirstNonZero;
5843 unsigned FirstNonZeroIdx;
5844 for (unsigned i=0; i < 4; ++i) {
5847 SDValue Elt = Op->getOperand(i);
5848 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5849 !isa<ConstantSDNode>(Elt.getOperand(1)))
5851 // Make sure that this node is extracting from a 128-bit vector.
5852 MVT VT = Elt.getOperand(0).getSimpleValueType();
5853 if (!VT.is128BitVector())
5855 if (!FirstNonZero.getNode()) {
5857 FirstNonZeroIdx = i;
5861 assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!");
5862 SDValue V1 = FirstNonZero.getOperand(0);
5863 MVT VT = V1.getSimpleValueType();
5865 // See if this build_vector can be lowered as a blend with zero.
5867 unsigned EltMaskIdx, EltIdx;
5869 for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
5870 if (Zeroable[EltIdx]) {
5871 // The zero vector will be on the right hand side.
5872 Mask[EltIdx] = EltIdx+4;
5876 Elt = Op->getOperand(EltIdx);
5877 // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index.
5878 EltMaskIdx = cast<ConstantSDNode>(Elt.getOperand(1))->getZExtValue();
5879 if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx)
5881 Mask[EltIdx] = EltIdx;
5885 // Let the shuffle legalizer deal with blend operations.
5886 SDValue VZero = getZeroVector(VT, Subtarget, DAG, SDLoc(Op));
5887 if (V1.getSimpleValueType() != VT)
5888 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), VT, V1);
5889 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZero, &Mask[0]);
5892 // See if we can lower this build_vector to a INSERTPS.
5893 if (!Subtarget->hasSSE41())
5896 SDValue V2 = Elt.getOperand(0);
5897 if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx)
5900 bool CanFold = true;
5901 for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) {
5905 SDValue Current = Op->getOperand(i);
5906 SDValue SrcVector = Current->getOperand(0);
5909 CanFold = SrcVector == V1 &&
5910 cast<ConstantSDNode>(Current.getOperand(1))->getZExtValue() == i;
5916 assert(V1.getNode() && "Expected at least two non-zero elements!");
5917 if (V1.getSimpleValueType() != MVT::v4f32)
5918 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1);
5919 if (V2.getSimpleValueType() != MVT::v4f32)
5920 V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2);
5922 // Ok, we can emit an INSERTPS instruction.
5924 for (int i = 0; i < 4; ++i)
5928 unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
5929 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
5930 SDValue Result = DAG.getNode(X86ISD::INSERTPS, SDLoc(Op), MVT::v4f32, V1, V2,
5931 DAG.getIntPtrConstant(InsertPSMask));
5932 return DAG.getNode(ISD::BITCAST, SDLoc(Op), VT, Result);
5935 /// Return a vector logical shift node.
5936 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
5937 unsigned NumBits, SelectionDAG &DAG,
5938 const TargetLowering &TLI, SDLoc dl) {
5939 assert(VT.is128BitVector() && "Unknown type for VShift");
5940 MVT ShVT = MVT::v2i64;
5941 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
5942 SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp);
5943 MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(SrcOp.getValueType());
5944 assert(NumBits % 8 == 0 && "Only support byte sized shifts");
5945 SDValue ShiftVal = DAG.getConstant(NumBits/8, ScalarShiftTy);
5946 return DAG.getNode(ISD::BITCAST, dl, VT,
5947 DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
5951 LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) {
5953 // Check if the scalar load can be widened into a vector load. And if
5954 // the address is "base + cst" see if the cst can be "absorbed" into
5955 // the shuffle mask.
5956 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
5957 SDValue Ptr = LD->getBasePtr();
5958 if (!ISD::isNormalLoad(LD) || LD->isVolatile())
5960 EVT PVT = LD->getValueType(0);
5961 if (PVT != MVT::i32 && PVT != MVT::f32)
5966 if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
5967 FI = FINode->getIndex();
5969 } else if (DAG.isBaseWithConstantOffset(Ptr) &&
5970 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
5971 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
5972 Offset = Ptr.getConstantOperandVal(1);
5973 Ptr = Ptr.getOperand(0);
5978 // FIXME: 256-bit vector instructions don't require a strict alignment,
5979 // improve this code to support it better.
5980 unsigned RequiredAlign = VT.getSizeInBits()/8;
5981 SDValue Chain = LD->getChain();
5982 // Make sure the stack object alignment is at least 16 or 32.
5983 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
5984 if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) {
5985 if (MFI->isFixedObjectIndex(FI)) {
5986 // Can't change the alignment. FIXME: It's possible to compute
5987 // the exact stack offset and reference FI + adjust offset instead.
5988 // If someone *really* cares about this. That's the way to implement it.
5991 MFI->setObjectAlignment(FI, RequiredAlign);
5995 // (Offset % 16 or 32) must be multiple of 4. Then address is then
5996 // Ptr + (Offset & ~15).
5999 if ((Offset % RequiredAlign) & 3)
6001 int64_t StartOffset = Offset & ~(RequiredAlign-1);
6003 Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(),
6004 Ptr,DAG.getConstant(StartOffset, Ptr.getValueType()));
6006 int EltNo = (Offset - StartOffset) >> 2;
6007 unsigned NumElems = VT.getVectorNumElements();
6009 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
6010 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
6011 LD->getPointerInfo().getWithOffset(StartOffset),
6012 false, false, false, 0);
6014 SmallVector<int, 8> Mask(NumElems, EltNo);
6016 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &Mask[0]);
6022 /// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
6023 /// elements can be replaced by a single large load which has the same value as
6024 /// a build_vector or insert_subvector whose loaded operands are 'Elts'.
6026 /// Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a
6028 /// FIXME: we'd also like to handle the case where the last elements are zero
6029 /// rather than undef via VZEXT_LOAD, but we do not detect that case today.
6030 /// There's even a handy isZeroNode for that purpose.
6031 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
6032 SDLoc &DL, SelectionDAG &DAG,
6033 bool isAfterLegalize) {
6034 unsigned NumElems = Elts.size();
6036 LoadSDNode *LDBase = nullptr;
6037 unsigned LastLoadedElt = -1U;
6039 // For each element in the initializer, see if we've found a load or an undef.
6040 // If we don't find an initial load element, or later load elements are
6041 // non-consecutive, bail out.
6042 for (unsigned i = 0; i < NumElems; ++i) {
6043 SDValue Elt = Elts[i];
6044 // Look through a bitcast.
6045 if (Elt.getNode() && Elt.getOpcode() == ISD::BITCAST)
6046 Elt = Elt.getOperand(0);
6047 if (!Elt.getNode() ||
6048 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode())))
6051 if (Elt.getNode()->getOpcode() == ISD::UNDEF)
6053 LDBase = cast<LoadSDNode>(Elt.getNode());
6057 if (Elt.getOpcode() == ISD::UNDEF)
6060 LoadSDNode *LD = cast<LoadSDNode>(Elt);
6061 EVT LdVT = Elt.getValueType();
6062 // Each loaded element must be the correct fractional portion of the
6063 // requested vector load.
6064 if (LdVT.getSizeInBits() != VT.getSizeInBits() / NumElems)
6066 if (!DAG.isConsecutiveLoad(LD, LDBase, LdVT.getSizeInBits() / 8, i))
6071 // If we have found an entire vector of loads and undefs, then return a large
6072 // load of the entire vector width starting at the base pointer. If we found
6073 // consecutive loads for the low half, generate a vzext_load node.
6074 if (LastLoadedElt == NumElems - 1) {
6075 assert(LDBase && "Did not find base load for merging consecutive loads");
6076 EVT EltVT = LDBase->getValueType(0);
6077 // Ensure that the input vector size for the merged loads matches the
6078 // cumulative size of the input elements.
6079 if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems)
6082 if (isAfterLegalize &&
6083 !DAG.getTargetLoweringInfo().isOperationLegal(ISD::LOAD, VT))
6086 SDValue NewLd = SDValue();
6088 NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
6089 LDBase->getPointerInfo(), LDBase->isVolatile(),
6090 LDBase->isNonTemporal(), LDBase->isInvariant(),
6091 LDBase->getAlignment());
6093 if (LDBase->hasAnyUseOfValue(1)) {
6094 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
6096 SDValue(NewLd.getNode(), 1));
6097 DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
6098 DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
6099 SDValue(NewLd.getNode(), 1));
6105 //TODO: The code below fires only for for loading the low v2i32 / v2f32
6106 //of a v4i32 / v4f32. It's probably worth generalizing.
6107 EVT EltVT = VT.getVectorElementType();
6108 if (NumElems == 4 && LastLoadedElt == 1 && (EltVT.getSizeInBits() == 32) &&
6109 DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) {
6110 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
6111 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
6113 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64,
6114 LDBase->getPointerInfo(),
6115 LDBase->getAlignment(),
6116 false/*isVolatile*/, true/*ReadMem*/,
6119 // Make sure the newly-created LOAD is in the same position as LDBase in
6120 // terms of dependency. We create a TokenFactor for LDBase and ResNode, and
6121 // update uses of LDBase's output chain to use the TokenFactor.
6122 if (LDBase->hasAnyUseOfValue(1)) {
6123 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
6124 SDValue(LDBase, 1), SDValue(ResNode.getNode(), 1));
6125 DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
6126 DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
6127 SDValue(ResNode.getNode(), 1));
6130 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
6135 /// LowerVectorBroadcast - Attempt to use the vbroadcast instruction
6136 /// to generate a splat value for the following cases:
6137 /// 1. A splat BUILD_VECTOR which uses a single scalar load, or a constant.
6138 /// 2. A splat shuffle which uses a scalar_to_vector node which comes from
6139 /// a scalar load, or a constant.
6140 /// The VBROADCAST node is returned when a pattern is found,
6141 /// or SDValue() otherwise.
6142 static SDValue LowerVectorBroadcast(SDValue Op, const X86Subtarget* Subtarget,
6143 SelectionDAG &DAG) {
6144 // VBROADCAST requires AVX.
6145 // TODO: Splats could be generated for non-AVX CPUs using SSE
6146 // instructions, but there's less potential gain for only 128-bit vectors.
6147 if (!Subtarget->hasAVX())
6150 MVT VT = Op.getSimpleValueType();
6153 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
6154 "Unsupported vector type for broadcast.");
6159 switch (Op.getOpcode()) {
6161 // Unknown pattern found.
6164 case ISD::BUILD_VECTOR: {
6165 auto *BVOp = cast<BuildVectorSDNode>(Op.getNode());
6166 BitVector UndefElements;
6167 SDValue Splat = BVOp->getSplatValue(&UndefElements);
6169 // We need a splat of a single value to use broadcast, and it doesn't
6170 // make any sense if the value is only in one element of the vector.
6171 if (!Splat || (VT.getVectorNumElements() - UndefElements.count()) <= 1)
6175 ConstSplatVal = (Ld.getOpcode() == ISD::Constant ||
6176 Ld.getOpcode() == ISD::ConstantFP);
6178 // Make sure that all of the users of a non-constant load are from the
6179 // BUILD_VECTOR node.
6180 if (!ConstSplatVal && !BVOp->isOnlyUserOf(Ld.getNode()))
6185 case ISD::VECTOR_SHUFFLE: {
6186 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
6188 // Shuffles must have a splat mask where the first element is
6190 if ((!SVOp->isSplat()) || SVOp->getMaskElt(0) != 0)
6193 SDValue Sc = Op.getOperand(0);
6194 if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR &&
6195 Sc.getOpcode() != ISD::BUILD_VECTOR) {
6197 if (!Subtarget->hasInt256())
6200 // Use the register form of the broadcast instruction available on AVX2.
6201 if (VT.getSizeInBits() >= 256)
6202 Sc = Extract128BitVector(Sc, 0, DAG, dl);
6203 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc);
6206 Ld = Sc.getOperand(0);
6207 ConstSplatVal = (Ld.getOpcode() == ISD::Constant ||
6208 Ld.getOpcode() == ISD::ConstantFP);
6210 // The scalar_to_vector node and the suspected
6211 // load node must have exactly one user.
6212 // Constants may have multiple users.
6214 // AVX-512 has register version of the broadcast
6215 bool hasRegVer = Subtarget->hasAVX512() && VT.is512BitVector() &&
6216 Ld.getValueType().getSizeInBits() >= 32;
6217 if (!ConstSplatVal && ((!Sc.hasOneUse() || !Ld.hasOneUse()) &&
6224 unsigned ScalarSize = Ld.getValueType().getSizeInBits();
6225 bool IsGE256 = (VT.getSizeInBits() >= 256);
6227 // When optimizing for size, generate up to 5 extra bytes for a broadcast
6228 // instruction to save 8 or more bytes of constant pool data.
6229 // TODO: If multiple splats are generated to load the same constant,
6230 // it may be detrimental to overall size. There needs to be a way to detect
6231 // that condition to know if this is truly a size win.
6232 const Function *F = DAG.getMachineFunction().getFunction();
6233 bool OptForSize = F->hasFnAttribute(Attribute::OptimizeForSize);
6235 // Handle broadcasting a single constant scalar from the constant pool
6237 // On Sandybridge (no AVX2), it is still better to load a constant vector
6238 // from the constant pool and not to broadcast it from a scalar.
6239 // But override that restriction when optimizing for size.
6240 // TODO: Check if splatting is recommended for other AVX-capable CPUs.
6241 if (ConstSplatVal && (Subtarget->hasAVX2() || OptForSize)) {
6242 EVT CVT = Ld.getValueType();
6243 assert(!CVT.isVector() && "Must not broadcast a vector type");
6245 // Splat f32, i32, v4f64, v4i64 in all cases with AVX2.
6246 // For size optimization, also splat v2f64 and v2i64, and for size opt
6247 // with AVX2, also splat i8 and i16.
6248 // With pattern matching, the VBROADCAST node may become a VMOVDDUP.
6249 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
6250 (OptForSize && (ScalarSize == 64 || Subtarget->hasAVX2()))) {
6251 const Constant *C = nullptr;
6252 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
6253 C = CI->getConstantIntValue();
6254 else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
6255 C = CF->getConstantFPValue();
6257 assert(C && "Invalid constant type");
6259 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6260 SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
6261 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
6262 Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP,
6263 MachinePointerInfo::getConstantPool(),
6264 false, false, false, Alignment);
6266 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6270 bool IsLoad = ISD::isNormalLoad(Ld.getNode());
6272 // Handle AVX2 in-register broadcasts.
6273 if (!IsLoad && Subtarget->hasInt256() &&
6274 (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
6275 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6277 // The scalar source must be a normal load.
6281 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
6282 (Subtarget->hasVLX() && ScalarSize == 64))
6283 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6285 // The integer check is needed for the 64-bit into 128-bit so it doesn't match
6286 // double since there is no vbroadcastsd xmm
6287 if (Subtarget->hasInt256() && Ld.getValueType().isInteger()) {
6288 if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)
6289 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6292 // Unsupported broadcast.
6296 /// \brief For an EXTRACT_VECTOR_ELT with a constant index return the real
6297 /// underlying vector and index.
6299 /// Modifies \p ExtractedFromVec to the real vector and returns the real
6301 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
6303 int Idx = cast<ConstantSDNode>(ExtIdx)->getZExtValue();
6304 if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
6307 // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
6309 // (extract_vector_elt (v8f32 %vreg1), Constant<6>)
6311 // (extract_vector_elt (vector_shuffle<2,u,u,u>
6312 // (extract_subvector (v8f32 %vreg0), Constant<4>),
6315 // In this case the vector is the extract_subvector expression and the index
6316 // is 2, as specified by the shuffle.
6317 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
6318 SDValue ShuffleVec = SVOp->getOperand(0);
6319 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
6320 assert(ShuffleVecVT.getVectorElementType() ==
6321 ExtractedFromVec.getSimpleValueType().getVectorElementType());
6323 int ShuffleIdx = SVOp->getMaskElt(Idx);
6324 if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
6325 ExtractedFromVec = ShuffleVec;
6331 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
6332 MVT VT = Op.getSimpleValueType();
6334 // Skip if insert_vec_elt is not supported.
6335 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6336 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
6340 unsigned NumElems = Op.getNumOperands();
6344 SmallVector<unsigned, 4> InsertIndices;
6345 SmallVector<int, 8> Mask(NumElems, -1);
6347 for (unsigned i = 0; i != NumElems; ++i) {
6348 unsigned Opc = Op.getOperand(i).getOpcode();
6350 if (Opc == ISD::UNDEF)
6353 if (Opc != ISD::EXTRACT_VECTOR_ELT) {
6354 // Quit if more than 1 elements need inserting.
6355 if (InsertIndices.size() > 1)
6358 InsertIndices.push_back(i);
6362 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
6363 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
6364 // Quit if non-constant index.
6365 if (!isa<ConstantSDNode>(ExtIdx))
6367 int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
6369 // Quit if extracted from vector of different type.
6370 if (ExtractedFromVec.getValueType() != VT)
6373 if (!VecIn1.getNode())
6374 VecIn1 = ExtractedFromVec;
6375 else if (VecIn1 != ExtractedFromVec) {
6376 if (!VecIn2.getNode())
6377 VecIn2 = ExtractedFromVec;
6378 else if (VecIn2 != ExtractedFromVec)
6379 // Quit if more than 2 vectors to shuffle
6383 if (ExtractedFromVec == VecIn1)
6385 else if (ExtractedFromVec == VecIn2)
6386 Mask[i] = Idx + NumElems;
6389 if (!VecIn1.getNode())
6392 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
6393 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]);
6394 for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) {
6395 unsigned Idx = InsertIndices[i];
6396 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
6397 DAG.getIntPtrConstant(Idx));
6403 // Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
6405 X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const {
6407 MVT VT = Op.getSimpleValueType();
6408 assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) &&
6409 "Unexpected type in LowerBUILD_VECTORvXi1!");
6412 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
6413 SDValue Cst = DAG.getTargetConstant(0, MVT::i1);
6414 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
6415 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
6418 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
6419 SDValue Cst = DAG.getTargetConstant(1, MVT::i1);
6420 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
6421 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
6424 bool AllContants = true;
6425 uint64_t Immediate = 0;
6426 int NonConstIdx = -1;
6427 bool IsSplat = true;
6428 unsigned NumNonConsts = 0;
6429 unsigned NumConsts = 0;
6430 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
6431 SDValue In = Op.getOperand(idx);
6432 if (In.getOpcode() == ISD::UNDEF)
6434 if (!isa<ConstantSDNode>(In)) {
6435 AllContants = false;
6440 if (cast<ConstantSDNode>(In)->getZExtValue())
6441 Immediate |= (1ULL << idx);
6443 if (In != Op.getOperand(0))
6448 SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1,
6449 DAG.getConstant(Immediate, MVT::i16));
6450 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, FullMask,
6451 DAG.getIntPtrConstant(0));
6454 if (NumNonConsts == 1 && NonConstIdx != 0) {
6457 SDValue VecAsImm = DAG.getConstant(Immediate,
6458 MVT::getIntegerVT(VT.getSizeInBits()));
6459 DstVec = DAG.getNode(ISD::BITCAST, dl, VT, VecAsImm);
6462 DstVec = DAG.getUNDEF(VT);
6463 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
6464 Op.getOperand(NonConstIdx),
6465 DAG.getIntPtrConstant(NonConstIdx));
6467 if (!IsSplat && (NonConstIdx != 0))
6468 llvm_unreachable("Unsupported BUILD_VECTOR operation");
6469 MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8;
6472 Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0),
6473 DAG.getConstant(-1, SelectVT),
6474 DAG.getConstant(0, SelectVT));
6476 Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0),
6477 DAG.getConstant((Immediate | 1), SelectVT),
6478 DAG.getConstant(Immediate, SelectVT));
6479 return DAG.getNode(ISD::BITCAST, dl, VT, Select);
6482 /// \brief Return true if \p N implements a horizontal binop and return the
6483 /// operands for the horizontal binop into V0 and V1.
6485 /// This is a helper function of PerformBUILD_VECTORCombine.
6486 /// This function checks that the build_vector \p N in input implements a
6487 /// horizontal operation. Parameter \p Opcode defines the kind of horizontal
6488 /// operation to match.
6489 /// For example, if \p Opcode is equal to ISD::ADD, then this function
6490 /// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
6491 /// is equal to ISD::SUB, then this function checks if this is a horizontal
6494 /// This function only analyzes elements of \p N whose indices are
6495 /// in range [BaseIdx, LastIdx).
6496 static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode,
6498 unsigned BaseIdx, unsigned LastIdx,
6499 SDValue &V0, SDValue &V1) {
6500 EVT VT = N->getValueType(0);
6502 assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
6503 assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
6504 "Invalid Vector in input!");
6506 bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
6507 bool CanFold = true;
6508 unsigned ExpectedVExtractIdx = BaseIdx;
6509 unsigned NumElts = LastIdx - BaseIdx;
6510 V0 = DAG.getUNDEF(VT);
6511 V1 = DAG.getUNDEF(VT);
6513 // Check if N implements a horizontal binop.
6514 for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
6515 SDValue Op = N->getOperand(i + BaseIdx);
6518 if (Op->getOpcode() == ISD::UNDEF) {
6519 // Update the expected vector extract index.
6520 if (i * 2 == NumElts)
6521 ExpectedVExtractIdx = BaseIdx;
6522 ExpectedVExtractIdx += 2;
6526 CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
6531 SDValue Op0 = Op.getOperand(0);
6532 SDValue Op1 = Op.getOperand(1);
6534 // Try to match the following pattern:
6535 // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
6536 CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6537 Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6538 Op0.getOperand(0) == Op1.getOperand(0) &&
6539 isa<ConstantSDNode>(Op0.getOperand(1)) &&
6540 isa<ConstantSDNode>(Op1.getOperand(1)));
6544 unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
6545 unsigned I1 = cast<ConstantSDNode>(Op1.getOperand(1))->getZExtValue();
6547 if (i * 2 < NumElts) {
6548 if (V0.getOpcode() == ISD::UNDEF)
6549 V0 = Op0.getOperand(0);
6551 if (V1.getOpcode() == ISD::UNDEF)
6552 V1 = Op0.getOperand(0);
6553 if (i * 2 == NumElts)
6554 ExpectedVExtractIdx = BaseIdx;
6557 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
6558 if (I0 == ExpectedVExtractIdx)
6559 CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
6560 else if (IsCommutable && I1 == ExpectedVExtractIdx) {
6561 // Try to match the following dag sequence:
6562 // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
6563 CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
6567 ExpectedVExtractIdx += 2;
6573 /// \brief Emit a sequence of two 128-bit horizontal add/sub followed by
6574 /// a concat_vector.
6576 /// This is a helper function of PerformBUILD_VECTORCombine.
6577 /// This function expects two 256-bit vectors called V0 and V1.
6578 /// At first, each vector is split into two separate 128-bit vectors.
6579 /// Then, the resulting 128-bit vectors are used to implement two
6580 /// horizontal binary operations.
6582 /// The kind of horizontal binary operation is defined by \p X86Opcode.
6584 /// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
6585 /// the two new horizontal binop.
6586 /// When Mode is set, the first horizontal binop dag node would take as input
6587 /// the lower 128-bit of V0 and the upper 128-bit of V0. The second
6588 /// horizontal binop dag node would take as input the lower 128-bit of V1
6589 /// and the upper 128-bit of V1.
6591 /// HADD V0_LO, V0_HI
6592 /// HADD V1_LO, V1_HI
6594 /// Otherwise, the first horizontal binop dag node takes as input the lower
6595 /// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
6596 /// dag node takes the the upper 128-bit of V0 and the upper 128-bit of V1.
6598 /// HADD V0_LO, V1_LO
6599 /// HADD V0_HI, V1_HI
6601 /// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
6602 /// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
6603 /// the upper 128-bits of the result.
6604 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
6605 SDLoc DL, SelectionDAG &DAG,
6606 unsigned X86Opcode, bool Mode,
6607 bool isUndefLO, bool isUndefHI) {
6608 EVT VT = V0.getValueType();
6609 assert(VT.is256BitVector() && VT == V1.getValueType() &&
6610 "Invalid nodes in input!");
6612 unsigned NumElts = VT.getVectorNumElements();
6613 SDValue V0_LO = Extract128BitVector(V0, 0, DAG, DL);
6614 SDValue V0_HI = Extract128BitVector(V0, NumElts/2, DAG, DL);
6615 SDValue V1_LO = Extract128BitVector(V1, 0, DAG, DL);
6616 SDValue V1_HI = Extract128BitVector(V1, NumElts/2, DAG, DL);
6617 EVT NewVT = V0_LO.getValueType();
6619 SDValue LO = DAG.getUNDEF(NewVT);
6620 SDValue HI = DAG.getUNDEF(NewVT);
6623 // Don't emit a horizontal binop if the result is expected to be UNDEF.
6624 if (!isUndefLO && V0->getOpcode() != ISD::UNDEF)
6625 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
6626 if (!isUndefHI && V1->getOpcode() != ISD::UNDEF)
6627 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
6629 // Don't emit a horizontal binop if the result is expected to be UNDEF.
6630 if (!isUndefLO && (V0_LO->getOpcode() != ISD::UNDEF ||
6631 V1_LO->getOpcode() != ISD::UNDEF))
6632 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
6634 if (!isUndefHI && (V0_HI->getOpcode() != ISD::UNDEF ||
6635 V1_HI->getOpcode() != ISD::UNDEF))
6636 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
6639 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
6642 /// \brief Try to fold a build_vector that performs an 'addsub' into the
6643 /// sequence of 'vadd + vsub + blendi'.
6644 static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG,
6645 const X86Subtarget *Subtarget) {
6647 EVT VT = BV->getValueType(0);
6648 unsigned NumElts = VT.getVectorNumElements();
6649 SDValue InVec0 = DAG.getUNDEF(VT);
6650 SDValue InVec1 = DAG.getUNDEF(VT);
6652 assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 ||
6653 VT == MVT::v2f64) && "build_vector with an invalid type found!");
6655 // Odd-numbered elements in the input build vector are obtained from
6656 // adding two integer/float elements.
6657 // Even-numbered elements in the input build vector are obtained from
6658 // subtracting two integer/float elements.
6659 unsigned ExpectedOpcode = ISD::FSUB;
6660 unsigned NextExpectedOpcode = ISD::FADD;
6661 bool AddFound = false;
6662 bool SubFound = false;
6664 for (unsigned i = 0, e = NumElts; i != e; ++i) {
6665 SDValue Op = BV->getOperand(i);
6667 // Skip 'undef' values.
6668 unsigned Opcode = Op.getOpcode();
6669 if (Opcode == ISD::UNDEF) {
6670 std::swap(ExpectedOpcode, NextExpectedOpcode);
6674 // Early exit if we found an unexpected opcode.
6675 if (Opcode != ExpectedOpcode)
6678 SDValue Op0 = Op.getOperand(0);
6679 SDValue Op1 = Op.getOperand(1);
6681 // Try to match the following pattern:
6682 // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
6683 // Early exit if we cannot match that sequence.
6684 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6685 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6686 !isa<ConstantSDNode>(Op0.getOperand(1)) ||
6687 !isa<ConstantSDNode>(Op1.getOperand(1)) ||
6688 Op0.getOperand(1) != Op1.getOperand(1))
6691 unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
6695 // We found a valid add/sub node. Update the information accordingly.
6701 // Update InVec0 and InVec1.
6702 if (InVec0.getOpcode() == ISD::UNDEF)
6703 InVec0 = Op0.getOperand(0);
6704 if (InVec1.getOpcode() == ISD::UNDEF)
6705 InVec1 = Op1.getOperand(0);
6707 // Make sure that operands in input to each add/sub node always
6708 // come from a same pair of vectors.
6709 if (InVec0 != Op0.getOperand(0)) {
6710 if (ExpectedOpcode == ISD::FSUB)
6713 // FADD is commutable. Try to commute the operands
6714 // and then test again.
6715 std::swap(Op0, Op1);
6716 if (InVec0 != Op0.getOperand(0))
6720 if (InVec1 != Op1.getOperand(0))
6723 // Update the pair of expected opcodes.
6724 std::swap(ExpectedOpcode, NextExpectedOpcode);
6727 // Don't try to fold this build_vector into an ADDSUB if the inputs are undef.
6728 if (AddFound && SubFound && InVec0.getOpcode() != ISD::UNDEF &&
6729 InVec1.getOpcode() != ISD::UNDEF)
6730 return DAG.getNode(X86ISD::ADDSUB, DL, VT, InVec0, InVec1);
6735 static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
6736 const X86Subtarget *Subtarget) {
6738 EVT VT = N->getValueType(0);
6739 unsigned NumElts = VT.getVectorNumElements();
6740 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(N);
6741 SDValue InVec0, InVec1;
6743 // Try to match an ADDSUB.
6744 if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
6745 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
6746 SDValue Value = matchAddSub(BV, DAG, Subtarget);
6747 if (Value.getNode())
6751 // Try to match horizontal ADD/SUB.
6752 unsigned NumUndefsLO = 0;
6753 unsigned NumUndefsHI = 0;
6754 unsigned Half = NumElts/2;
6756 // Count the number of UNDEF operands in the build_vector in input.
6757 for (unsigned i = 0, e = Half; i != e; ++i)
6758 if (BV->getOperand(i)->getOpcode() == ISD::UNDEF)
6761 for (unsigned i = Half, e = NumElts; i != e; ++i)
6762 if (BV->getOperand(i)->getOpcode() == ISD::UNDEF)
6765 // Early exit if this is either a build_vector of all UNDEFs or all the
6766 // operands but one are UNDEF.
6767 if (NumUndefsLO + NumUndefsHI + 1 >= NumElts)
6770 if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) {
6771 // Try to match an SSE3 float HADD/HSUB.
6772 if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
6773 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6775 if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
6776 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6777 } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) {
6778 // Try to match an SSSE3 integer HADD/HSUB.
6779 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
6780 return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
6782 if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
6783 return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
6786 if (!Subtarget->hasAVX())
6789 if ((VT == MVT::v8f32 || VT == MVT::v4f64)) {
6790 // Try to match an AVX horizontal add/sub of packed single/double
6791 // precision floating point values from 256-bit vectors.
6792 SDValue InVec2, InVec3;
6793 if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) &&
6794 isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) &&
6795 ((InVec0.getOpcode() == ISD::UNDEF ||
6796 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6797 ((InVec1.getOpcode() == ISD::UNDEF ||
6798 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6799 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6801 if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) &&
6802 isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) &&
6803 ((InVec0.getOpcode() == ISD::UNDEF ||
6804 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6805 ((InVec1.getOpcode() == ISD::UNDEF ||
6806 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6807 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6808 } else if (VT == MVT::v8i32 || VT == MVT::v16i16) {
6809 // Try to match an AVX2 horizontal add/sub of signed integers.
6810 SDValue InVec2, InVec3;
6812 bool CanFold = true;
6814 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
6815 isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) &&
6816 ((InVec0.getOpcode() == ISD::UNDEF ||
6817 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6818 ((InVec1.getOpcode() == ISD::UNDEF ||
6819 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6820 X86Opcode = X86ISD::HADD;
6821 else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) &&
6822 isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) &&
6823 ((InVec0.getOpcode() == ISD::UNDEF ||
6824 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6825 ((InVec1.getOpcode() == ISD::UNDEF ||
6826 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6827 X86Opcode = X86ISD::HSUB;
6832 // Fold this build_vector into a single horizontal add/sub.
6833 // Do this only if the target has AVX2.
6834 if (Subtarget->hasAVX2())
6835 return DAG.getNode(X86Opcode, DL, VT, InVec0, InVec1);
6837 // Do not try to expand this build_vector into a pair of horizontal
6838 // add/sub if we can emit a pair of scalar add/sub.
6839 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
6842 // Convert this build_vector into a pair of horizontal binop followed by
6844 bool isUndefLO = NumUndefsLO == Half;
6845 bool isUndefHI = NumUndefsHI == Half;
6846 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, false,
6847 isUndefLO, isUndefHI);
6851 if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
6852 VT == MVT::v16i16) && Subtarget->hasAVX()) {
6854 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
6855 X86Opcode = X86ISD::HADD;
6856 else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
6857 X86Opcode = X86ISD::HSUB;
6858 else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
6859 X86Opcode = X86ISD::FHADD;
6860 else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
6861 X86Opcode = X86ISD::FHSUB;
6865 // Don't try to expand this build_vector into a pair of horizontal add/sub
6866 // if we can simply emit a pair of scalar add/sub.
6867 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
6870 // Convert this build_vector into two horizontal add/sub followed by
6872 bool isUndefLO = NumUndefsLO == Half;
6873 bool isUndefHI = NumUndefsHI == Half;
6874 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
6875 isUndefLO, isUndefHI);
6882 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
6885 MVT VT = Op.getSimpleValueType();
6886 MVT ExtVT = VT.getVectorElementType();
6887 unsigned NumElems = Op.getNumOperands();
6889 // Generate vectors for predicate vectors.
6890 if (VT.getScalarType() == MVT::i1 && Subtarget->hasAVX512())
6891 return LowerBUILD_VECTORvXi1(Op, DAG);
6893 // Vectors containing all zeros can be matched by pxor and xorps later
6894 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
6895 // Canonicalize this to <4 x i32> to 1) ensure the zero vectors are CSE'd
6896 // and 2) ensure that i64 scalars are eliminated on x86-32 hosts.
6897 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)
6900 return getZeroVector(VT, Subtarget, DAG, dl);
6903 // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
6904 // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use
6905 // vpcmpeqd on 256-bit vectors.
6906 if (Subtarget->hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
6907 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256()))
6910 if (!VT.is512BitVector())
6911 return getOnesVector(VT, Subtarget->hasInt256(), DAG, dl);
6914 SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG);
6915 if (Broadcast.getNode())
6918 unsigned EVTBits = ExtVT.getSizeInBits();
6920 unsigned NumZero = 0;
6921 unsigned NumNonZero = 0;
6922 unsigned NonZeros = 0;
6923 bool IsAllConstants = true;
6924 SmallSet<SDValue, 8> Values;
6925 for (unsigned i = 0; i < NumElems; ++i) {
6926 SDValue Elt = Op.getOperand(i);
6927 if (Elt.getOpcode() == ISD::UNDEF)
6930 if (Elt.getOpcode() != ISD::Constant &&
6931 Elt.getOpcode() != ISD::ConstantFP)
6932 IsAllConstants = false;
6933 if (X86::isZeroNode(Elt))
6936 NonZeros |= (1 << i);
6941 // All undef vector. Return an UNDEF. All zero vectors were handled above.
6942 if (NumNonZero == 0)
6943 return DAG.getUNDEF(VT);
6945 // Special case for single non-zero, non-undef, element.
6946 if (NumNonZero == 1) {
6947 unsigned Idx = countTrailingZeros(NonZeros);
6948 SDValue Item = Op.getOperand(Idx);
6950 // If this is an insertion of an i64 value on x86-32, and if the top bits of
6951 // the value are obviously zero, truncate the value to i32 and do the
6952 // insertion that way. Only do this if the value is non-constant or if the
6953 // value is a constant being inserted into element 0. It is cheaper to do
6954 // a constant pool load than it is to do a movd + shuffle.
6955 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() &&
6956 (!IsAllConstants || Idx == 0)) {
6957 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) {
6959 assert(VT == MVT::v2i64 && "Expected an SSE value type!");
6960 EVT VecVT = MVT::v4i32;
6961 unsigned VecElts = 4;
6963 // Truncate the value (which may itself be a constant) to i32, and
6964 // convert it to a vector with movd (S2V+shuffle to zero extend).
6965 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item);
6966 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item);
6968 // If using the new shuffle lowering, just directly insert this.
6969 if (ExperimentalVectorShuffleLowering)
6971 ISD::BITCAST, dl, VT,
6972 getShuffleVectorZeroOrUndef(Item, Idx * 2, true, Subtarget, DAG));
6974 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
6976 // Now we have our 32-bit value zero extended in the low element of
6977 // a vector. If Idx != 0, swizzle it into place.
6979 SmallVector<int, 4> Mask;
6980 Mask.push_back(Idx);
6981 for (unsigned i = 1; i != VecElts; ++i)
6983 Item = DAG.getVectorShuffle(VecVT, dl, Item, DAG.getUNDEF(VecVT),
6986 return DAG.getNode(ISD::BITCAST, dl, VT, Item);
6990 // If we have a constant or non-constant insertion into the low element of
6991 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
6992 // the rest of the elements. This will be matched as movd/movq/movss/movsd
6993 // depending on what the source datatype is.
6996 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
6998 if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 ||
6999 (ExtVT == MVT::i64 && Subtarget->is64Bit())) {
7000 if (VT.is256BitVector() || VT.is512BitVector()) {
7001 SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl);
7002 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec,
7003 Item, DAG.getIntPtrConstant(0));
7005 assert(VT.is128BitVector() && "Expected an SSE value type!");
7006 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
7007 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
7008 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
7011 if (ExtVT == MVT::i16 || ExtVT == MVT::i8) {
7012 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
7013 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item);
7014 if (VT.is256BitVector()) {
7015 SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl);
7016 Item = Insert128BitVector(ZeroVec, Item, 0, DAG, dl);
7018 assert(VT.is128BitVector() && "Expected an SSE value type!");
7019 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
7021 return DAG.getNode(ISD::BITCAST, dl, VT, Item);
7025 // Is it a vector logical left shift?
7026 if (NumElems == 2 && Idx == 1 &&
7027 X86::isZeroNode(Op.getOperand(0)) &&
7028 !X86::isZeroNode(Op.getOperand(1))) {
7029 unsigned NumBits = VT.getSizeInBits();
7030 return getVShift(true, VT,
7031 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
7032 VT, Op.getOperand(1)),
7033 NumBits/2, DAG, *this, dl);
7036 if (IsAllConstants) // Otherwise, it's better to do a constpool load.
7039 // Otherwise, if this is a vector with i32 or f32 elements, and the element
7040 // is a non-constant being inserted into an element other than the low one,
7041 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka
7042 // movd/movss) to move this into the low element, then shuffle it into
7044 if (EVTBits == 32) {
7045 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
7047 // If using the new shuffle lowering, just directly insert this.
7048 if (ExperimentalVectorShuffleLowering)
7049 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
7051 // Turn it into a shuffle of zero and zero-extended scalar to vector.
7052 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, Subtarget, DAG);
7053 SmallVector<int, 8> MaskVec;
7054 for (unsigned i = 0; i != NumElems; ++i)
7055 MaskVec.push_back(i == Idx ? 0 : 1);
7056 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]);
7060 // Splat is obviously ok. Let legalizer expand it to a shuffle.
7061 if (Values.size() == 1) {
7062 if (EVTBits == 32) {
7063 // Instead of a shuffle like this:
7064 // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
7065 // Check if it's possible to issue this instead.
7066 // shuffle (vload ptr)), undef, <1, 1, 1, 1>
7067 unsigned Idx = countTrailingZeros(NonZeros);
7068 SDValue Item = Op.getOperand(Idx);
7069 if (Op.getNode()->isOnlyUserOf(Item.getNode()))
7070 return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
7075 // A vector full of immediates; various special cases are already
7076 // handled, so this is best done with a single constant-pool load.
7080 // For AVX-length vectors, see if we can use a vector load to get all of the
7081 // elements, otherwise build the individual 128-bit pieces and use
7082 // shuffles to put them in place.
7083 if (VT.is256BitVector() || VT.is512BitVector()) {
7084 SmallVector<SDValue, 64> V(Op->op_begin(), Op->op_begin() + NumElems);
7086 // Check for a build vector of consecutive loads.
7087 if (SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false))
7090 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2);
7092 // Build both the lower and upper subvector.
7093 SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT,
7094 makeArrayRef(&V[0], NumElems/2));
7095 SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT,
7096 makeArrayRef(&V[NumElems / 2], NumElems/2));
7098 // Recreate the wider vector with the lower and upper part.
7099 if (VT.is256BitVector())
7100 return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl);
7101 return Concat256BitVectors(Lower, Upper, VT, NumElems, DAG, dl);
7104 // Let legalizer expand 2-wide build_vectors.
7105 if (EVTBits == 64) {
7106 if (NumNonZero == 1) {
7107 // One half is zero or undef.
7108 unsigned Idx = countTrailingZeros(NonZeros);
7109 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
7110 Op.getOperand(Idx));
7111 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
7116 // If element VT is < 32 bits, convert it to inserts into a zero vector.
7117 if (EVTBits == 8 && NumElems == 16) {
7118 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
7120 if (V.getNode()) return V;
7123 if (EVTBits == 16 && NumElems == 8) {
7124 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
7126 if (V.getNode()) return V;
7129 // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS
7130 if (EVTBits == 32 && NumElems == 4) {
7131 SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget, *this);
7136 // If element VT is == 32 bits, turn it into a number of shuffles.
7137 SmallVector<SDValue, 8> V(NumElems);
7138 if (NumElems == 4 && NumZero > 0) {
7139 for (unsigned i = 0; i < 4; ++i) {
7140 bool isZero = !(NonZeros & (1 << i));
7142 V[i] = getZeroVector(VT, Subtarget, DAG, dl);
7144 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
7147 for (unsigned i = 0; i < 2; ++i) {
7148 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) {
7151 V[i] = V[i*2]; // Must be a zero vector.
7154 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]);
7157 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]);
7160 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]);
7165 bool Reverse1 = (NonZeros & 0x3) == 2;
7166 bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2;
7170 static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
7171 static_cast<int>(Reverse2 ? NumElems : NumElems+1)
7173 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]);
7176 if (Values.size() > 1 && VT.is128BitVector()) {
7177 // Check for a build vector of consecutive loads.
7178 for (unsigned i = 0; i < NumElems; ++i)
7179 V[i] = Op.getOperand(i);
7181 // Check for elements which are consecutive loads.
7182 SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false);
7186 // Check for a build vector from mostly shuffle plus few inserting.
7187 SDValue Sh = buildFromShuffleMostly(Op, DAG);
7191 // For SSE 4.1, use insertps to put the high elements into the low element.
7192 if (Subtarget->hasSSE41()) {
7194 if (Op.getOperand(0).getOpcode() != ISD::UNDEF)
7195 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
7197 Result = DAG.getUNDEF(VT);
7199 for (unsigned i = 1; i < NumElems; ++i) {
7200 if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue;
7201 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
7202 Op.getOperand(i), DAG.getIntPtrConstant(i));
7207 // Otherwise, expand into a number of unpckl*, start by extending each of
7208 // our (non-undef) elements to the full vector width with the element in the
7209 // bottom slot of the vector (which generates no code for SSE).
7210 for (unsigned i = 0; i < NumElems; ++i) {
7211 if (Op.getOperand(i).getOpcode() != ISD::UNDEF)
7212 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
7214 V[i] = DAG.getUNDEF(VT);
7217 // Next, we iteratively mix elements, e.g. for v4f32:
7218 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0>
7219 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1>
7220 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0>
7221 unsigned EltStride = NumElems >> 1;
7222 while (EltStride != 0) {
7223 for (unsigned i = 0; i < EltStride; ++i) {
7224 // If V[i+EltStride] is undef and this is the first round of mixing,
7225 // then it is safe to just drop this shuffle: V[i] is already in the
7226 // right place, the one element (since it's the first round) being
7227 // inserted as undef can be dropped. This isn't safe for successive
7228 // rounds because they will permute elements within both vectors.
7229 if (V[i+EltStride].getOpcode() == ISD::UNDEF &&
7230 EltStride == NumElems/2)
7233 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]);
7242 // LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction
7243 // to create 256-bit vectors from two other 128-bit ones.
7244 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
7246 MVT ResVT = Op.getSimpleValueType();
7248 assert((ResVT.is256BitVector() ||
7249 ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
7251 SDValue V1 = Op.getOperand(0);
7252 SDValue V2 = Op.getOperand(1);
7253 unsigned NumElems = ResVT.getVectorNumElements();
7254 if(ResVT.is256BitVector())
7255 return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
7257 if (Op.getNumOperands() == 4) {
7258 MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(),
7259 ResVT.getVectorNumElements()/2);
7260 SDValue V3 = Op.getOperand(2);
7261 SDValue V4 = Op.getOperand(3);
7262 return Concat256BitVectors(Concat128BitVectors(V1, V2, HalfVT, NumElems/2, DAG, dl),
7263 Concat128BitVectors(V3, V4, HalfVT, NumElems/2, DAG, dl), ResVT, NumElems, DAG, dl);
7265 return Concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
7268 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
7269 MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType();
7270 assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
7271 (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
7272 Op.getNumOperands() == 4)));
7274 // AVX can use the vinsertf128 instruction to create 256-bit vectors
7275 // from two other 128-bit ones.
7277 // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
7278 return LowerAVXCONCAT_VECTORS(Op, DAG);
7282 //===----------------------------------------------------------------------===//
7283 // Vector shuffle lowering
7285 // This is an experimental code path for lowering vector shuffles on x86. It is
7286 // designed to handle arbitrary vector shuffles and blends, gracefully
7287 // degrading performance as necessary. It works hard to recognize idiomatic
7288 // shuffles and lower them to optimal instruction patterns without leaving
7289 // a framework that allows reasonably efficient handling of all vector shuffle
7291 //===----------------------------------------------------------------------===//
7293 /// \brief Tiny helper function to identify a no-op mask.
7295 /// This is a somewhat boring predicate function. It checks whether the mask
7296 /// array input, which is assumed to be a single-input shuffle mask of the kind
7297 /// used by the X86 shuffle instructions (not a fully general
7298 /// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an
7299 /// in-place shuffle are 'no-op's.
7300 static bool isNoopShuffleMask(ArrayRef<int> Mask) {
7301 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7302 if (Mask[i] != -1 && Mask[i] != i)
7307 /// \brief Helper function to classify a mask as a single-input mask.
7309 /// This isn't a generic single-input test because in the vector shuffle
7310 /// lowering we canonicalize single inputs to be the first input operand. This
7311 /// means we can more quickly test for a single input by only checking whether
7312 /// an input from the second operand exists. We also assume that the size of
7313 /// mask corresponds to the size of the input vectors which isn't true in the
7314 /// fully general case.
7315 static bool isSingleInputShuffleMask(ArrayRef<int> Mask) {
7317 if (M >= (int)Mask.size())
7322 /// \brief Test whether there are elements crossing 128-bit lanes in this
7325 /// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
7326 /// and we routinely test for these.
7327 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {
7328 int LaneSize = 128 / VT.getScalarSizeInBits();
7329 int Size = Mask.size();
7330 for (int i = 0; i < Size; ++i)
7331 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
7336 /// \brief Test whether a shuffle mask is equivalent within each 128-bit lane.
7338 /// This checks a shuffle mask to see if it is performing the same
7339 /// 128-bit lane-relative shuffle in each 128-bit lane. This trivially implies
7340 /// that it is also not lane-crossing. It may however involve a blend from the
7341 /// same lane of a second vector.
7343 /// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
7344 /// non-trivial to compute in the face of undef lanes. The representation is
7345 /// *not* suitable for use with existing 128-bit shuffles as it will contain
7346 /// entries from both V1 and V2 inputs to the wider mask.
7348 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
7349 SmallVectorImpl<int> &RepeatedMask) {
7350 int LaneSize = 128 / VT.getScalarSizeInBits();
7351 RepeatedMask.resize(LaneSize, -1);
7352 int Size = Mask.size();
7353 for (int i = 0; i < Size; ++i) {
7356 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
7357 // This entry crosses lanes, so there is no way to model this shuffle.
7360 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
7361 if (RepeatedMask[i % LaneSize] == -1)
7362 // This is the first non-undef entry in this slot of a 128-bit lane.
7363 RepeatedMask[i % LaneSize] =
7364 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + Size;
7365 else if (RepeatedMask[i % LaneSize] + (i / LaneSize) * LaneSize != Mask[i])
7366 // Found a mismatch with the repeated mask.
7372 /// \brief Base case helper for testing a single mask element.
7373 static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2,
7374 BuildVectorSDNode *BV1,
7375 BuildVectorSDNode *BV2, ArrayRef<int> Mask,
7377 int Size = Mask.size();
7378 if (Mask[i] != -1 && Mask[i] != Arg) {
7379 auto *MaskBV = Mask[i] < Size ? BV1 : BV2;
7380 auto *ArgsBV = Arg < Size ? BV1 : BV2;
7381 if (!MaskBV || !ArgsBV ||
7382 MaskBV->getOperand(Mask[i] % Size) != ArgsBV->getOperand(Arg % Size))
7388 /// \brief Recursive helper to peel off and test each mask element.
7389 template <typename... Ts>
7390 static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2,
7391 BuildVectorSDNode *BV1,
7392 BuildVectorSDNode *BV2, ArrayRef<int> Mask,
7393 int i, int Arg, Ts... Args) {
7394 if (!isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i, Arg))
7397 return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i + 1, Args...);
7400 /// \brief Checks whether a shuffle mask is equivalent to an explicit list of
7403 /// This is a fast way to test a shuffle mask against a fixed pattern:
7405 /// if (isShuffleEquivalent(Mask, 3, 2, 1, 0)) { ... }
7407 /// It returns true if the mask is exactly as wide as the argument list, and
7408 /// each element of the mask is either -1 (signifying undef) or the value given
7409 /// in the argument.
7410 template <typename... Ts>
7411 static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
7413 if (Mask.size() != sizeof...(Args))
7416 // If the values are build vectors, we can look through them to find
7417 // equivalent inputs that make the shuffles equivalent.
7418 auto *BV1 = dyn_cast<BuildVectorSDNode>(V1);
7419 auto *BV2 = dyn_cast<BuildVectorSDNode>(V2);
7421 // Recursively peel off arguments and test them against the mask.
7422 return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, 0, Args...);
7425 /// \brief Get a 4-lane 8-bit shuffle immediate for a mask.
7427 /// This helper function produces an 8-bit shuffle immediate corresponding to
7428 /// the ubiquitous shuffle encoding scheme used in x86 instructions for
7429 /// shuffling 4 lanes. It can be used with most of the PSHUF instructions for
7432 /// NB: We rely heavily on "undef" masks preserving the input lane.
7433 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask,
7434 SelectionDAG &DAG) {
7435 assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
7436 assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
7437 assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!");
7438 assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
7439 assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
7442 Imm |= (Mask[0] == -1 ? 0 : Mask[0]) << 0;
7443 Imm |= (Mask[1] == -1 ? 1 : Mask[1]) << 2;
7444 Imm |= (Mask[2] == -1 ? 2 : Mask[2]) << 4;
7445 Imm |= (Mask[3] == -1 ? 3 : Mask[3]) << 6;
7446 return DAG.getConstant(Imm, MVT::i8);
7449 /// \brief Try to emit a blend instruction for a shuffle.
7451 /// This doesn't do any checks for the availability of instructions for blending
7452 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
7453 /// be matched in the backend with the type given. What it does check for is
7454 /// that the shuffle mask is in fact a blend.
7455 static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
7456 SDValue V2, ArrayRef<int> Mask,
7457 const X86Subtarget *Subtarget,
7458 SelectionDAG &DAG) {
7460 unsigned BlendMask = 0;
7461 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7462 if (Mask[i] >= Size) {
7463 if (Mask[i] != i + Size)
7464 return SDValue(); // Shuffled V2 input!
7465 BlendMask |= 1u << i;
7468 if (Mask[i] >= 0 && Mask[i] != i)
7469 return SDValue(); // Shuffled V1 input!
7471 switch (VT.SimpleTy) {
7476 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
7477 DAG.getConstant(BlendMask, MVT::i8));
7481 assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!");
7485 // If we have AVX2 it is faster to use VPBLENDD when the shuffle fits into
7486 // that instruction.
7487 if (Subtarget->hasAVX2()) {
7488 // Scale the blend by the number of 32-bit dwords per element.
7489 int Scale = VT.getScalarSizeInBits() / 32;
7491 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7492 if (Mask[i] >= Size)
7493 for (int j = 0; j < Scale; ++j)
7494 BlendMask |= 1u << (i * Scale + j);
7496 MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32;
7497 V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1);
7498 V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2);
7499 return DAG.getNode(ISD::BITCAST, DL, VT,
7500 DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
7501 DAG.getConstant(BlendMask, MVT::i8)));
7505 // For integer shuffles we need to expand the mask and cast the inputs to
7506 // v8i16s prior to blending.
7507 int Scale = 8 / VT.getVectorNumElements();
7509 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7510 if (Mask[i] >= Size)
7511 for (int j = 0; j < Scale; ++j)
7512 BlendMask |= 1u << (i * Scale + j);
7514 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1);
7515 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2);
7516 return DAG.getNode(ISD::BITCAST, DL, VT,
7517 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
7518 DAG.getConstant(BlendMask, MVT::i8)));
7522 assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!");
7523 SmallVector<int, 8> RepeatedMask;
7524 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
7525 // We can lower these with PBLENDW which is mirrored across 128-bit lanes.
7526 assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!");
7528 for (int i = 0; i < 8; ++i)
7529 if (RepeatedMask[i] >= 16)
7530 BlendMask |= 1u << i;
7531 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
7532 DAG.getConstant(BlendMask, MVT::i8));
7538 // Scale the blend by the number of bytes per element.
7539 int Scale = VT.getScalarSizeInBits() / 8;
7541 // This form of blend is always done on bytes. Compute the byte vector
7543 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
7545 // Compute the VSELECT mask. Note that VSELECT is really confusing in the
7546 // mix of LLVM's code generator and the x86 backend. We tell the code
7547 // generator that boolean values in the elements of an x86 vector register
7548 // are -1 for true and 0 for false. We then use the LLVM semantics of 'true'
7549 // mapping a select to operand #1, and 'false' mapping to operand #2. The
7550 // reality in x86 is that vector masks (pre-AVX-512) use only the high bit
7551 // of the element (the remaining are ignored) and 0 in that high bit would
7552 // mean operand #1 while 1 in the high bit would mean operand #2. So while
7553 // the LLVM model for boolean values in vector elements gets the relevant
7554 // bit set, it is set backwards and over constrained relative to x86's
7556 SmallVector<SDValue, 32> VSELECTMask;
7557 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7558 for (int j = 0; j < Scale; ++j)
7559 VSELECTMask.push_back(
7560 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
7561 : DAG.getConstant(Mask[i] < Size ? -1 : 0, MVT::i8));
7563 V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1);
7564 V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2);
7566 ISD::BITCAST, DL, VT,
7567 DAG.getNode(ISD::VSELECT, DL, BlendVT,
7568 DAG.getNode(ISD::BUILD_VECTOR, DL, BlendVT, VSELECTMask),
7573 llvm_unreachable("Not a supported integer vector type!");
7577 /// \brief Try to lower as a blend of elements from two inputs followed by
7578 /// a single-input permutation.
7580 /// This matches the pattern where we can blend elements from two inputs and
7581 /// then reduce the shuffle to a single-input permutation.
7582 static SDValue lowerVectorShuffleAsBlendAndPermute(SDLoc DL, MVT VT, SDValue V1,
7585 SelectionDAG &DAG) {
7586 // We build up the blend mask while checking whether a blend is a viable way
7587 // to reduce the shuffle.
7588 SmallVector<int, 32> BlendMask(Mask.size(), -1);
7589 SmallVector<int, 32> PermuteMask(Mask.size(), -1);
7591 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7595 assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds.");
7597 if (BlendMask[Mask[i] % Size] == -1)
7598 BlendMask[Mask[i] % Size] = Mask[i];
7599 else if (BlendMask[Mask[i] % Size] != Mask[i])
7600 return SDValue(); // Can't blend in the needed input!
7602 PermuteMask[i] = Mask[i] % Size;
7605 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
7606 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
7609 /// \brief Generic routine to decompose a shuffle and blend into indepndent
7610 /// blends and permutes.
7612 /// This matches the extremely common pattern for handling combined
7613 /// shuffle+blend operations on newer X86 ISAs where we have very fast blend
7614 /// operations. It will try to pick the best arrangement of shuffles and
7616 static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(SDLoc DL, MVT VT,
7620 SelectionDAG &DAG) {
7621 // Shuffle the input elements into the desired positions in V1 and V2 and
7622 // blend them together.
7623 SmallVector<int, 32> V1Mask(Mask.size(), -1);
7624 SmallVector<int, 32> V2Mask(Mask.size(), -1);
7625 SmallVector<int, 32> BlendMask(Mask.size(), -1);
7626 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7627 if (Mask[i] >= 0 && Mask[i] < Size) {
7628 V1Mask[i] = Mask[i];
7630 } else if (Mask[i] >= Size) {
7631 V2Mask[i] = Mask[i] - Size;
7632 BlendMask[i] = i + Size;
7635 // Try to lower with the simpler initial blend strategy unless one of the
7636 // input shuffles would be a no-op. We prefer to shuffle inputs as the
7637 // shuffle may be able to fold with a load or other benefit. However, when
7638 // we'll have to do 2x as many shuffles in order to achieve this, blending
7639 // first is a better strategy.
7640 if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask))
7641 if (SDValue BlendPerm =
7642 lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))
7645 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
7646 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
7647 return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
7650 /// \brief Try to lower a vector shuffle as a byte rotation.
7652 /// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary
7653 /// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
7654 /// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will
7655 /// try to generically lower a vector shuffle through such an pattern. It
7656 /// does not check for the profitability of lowering either as PALIGNR or
7657 /// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form.
7658 /// This matches shuffle vectors that look like:
7660 /// v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
7662 /// Essentially it concatenates V1 and V2, shifts right by some number of
7663 /// elements, and takes the low elements as the result. Note that while this is
7664 /// specified as a *right shift* because x86 is little-endian, it is a *left
7665 /// rotate* of the vector lanes.
7666 static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1,
7669 const X86Subtarget *Subtarget,
7670 SelectionDAG &DAG) {
7671 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
7673 int NumElts = Mask.size();
7674 int NumLanes = VT.getSizeInBits() / 128;
7675 int NumLaneElts = NumElts / NumLanes;
7677 // We need to detect various ways of spelling a rotation:
7678 // [11, 12, 13, 14, 15, 0, 1, 2]
7679 // [-1, 12, 13, 14, -1, -1, 1, -1]
7680 // [-1, -1, -1, -1, -1, -1, 1, 2]
7681 // [ 3, 4, 5, 6, 7, 8, 9, 10]
7682 // [-1, 4, 5, 6, -1, -1, 9, -1]
7683 // [-1, 4, 5, 6, -1, -1, -1, -1]
7686 for (int l = 0; l < NumElts; l += NumLaneElts) {
7687 for (int i = 0; i < NumLaneElts; ++i) {
7688 if (Mask[l + i] == -1)
7690 assert(Mask[l + i] >= 0 && "Only -1 is a valid negative mask element!");
7692 // Get the mod-Size index and lane correct it.
7693 int LaneIdx = (Mask[l + i] % NumElts) - l;
7694 // Make sure it was in this lane.
7695 if (LaneIdx < 0 || LaneIdx >= NumLaneElts)
7698 // Determine where a rotated vector would have started.
7699 int StartIdx = i - LaneIdx;
7701 // The identity rotation isn't interesting, stop.
7704 // If we found the tail of a vector the rotation must be the missing
7705 // front. If we found the head of a vector, it must be how much of the
7707 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumLaneElts - StartIdx;
7710 Rotation = CandidateRotation;
7711 else if (Rotation != CandidateRotation)
7712 // The rotations don't match, so we can't match this mask.
7715 // Compute which value this mask is pointing at.
7716 SDValue MaskV = Mask[l + i] < NumElts ? V1 : V2;
7718 // Compute which of the two target values this index should be assigned
7719 // to. This reflects whether the high elements are remaining or the low
7720 // elements are remaining.
7721 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
7723 // Either set up this value if we've not encountered it before, or check
7724 // that it remains consistent.
7727 else if (TargetV != MaskV)
7728 // This may be a rotation, but it pulls from the inputs in some
7729 // unsupported interleaving.
7734 // Check that we successfully analyzed the mask, and normalize the results.
7735 assert(Rotation != 0 && "Failed to locate a viable rotation!");
7736 assert((Lo || Hi) && "Failed to find a rotated input vector!");
7742 // The actual rotate instruction rotates bytes, so we need to scale the
7743 // rotation based on how many bytes are in the vector lane.
7744 int Scale = 16 / NumLaneElts;
7746 // SSSE3 targets can use the palignr instruction.
7747 if (Subtarget->hasSSSE3()) {
7748 // Cast the inputs to i8 vector of correct length to match PALIGNR.
7749 MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes);
7750 Lo = DAG.getNode(ISD::BITCAST, DL, AlignVT, Lo);
7751 Hi = DAG.getNode(ISD::BITCAST, DL, AlignVT, Hi);
7753 return DAG.getNode(ISD::BITCAST, DL, VT,
7754 DAG.getNode(X86ISD::PALIGNR, DL, AlignVT, Hi, Lo,
7755 DAG.getConstant(Rotation * Scale, MVT::i8)));
7758 assert(VT.getSizeInBits() == 128 &&
7759 "Rotate-based lowering only supports 128-bit lowering!");
7760 assert(Mask.size() <= 16 &&
7761 "Can shuffle at most 16 bytes in a 128-bit vector!");
7763 // Default SSE2 implementation
7764 int LoByteShift = 16 - Rotation * Scale;
7765 int HiByteShift = Rotation * Scale;
7767 // Cast the inputs to v2i64 to match PSLLDQ/PSRLDQ.
7768 Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo);
7769 Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi);
7771 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo,
7772 DAG.getConstant(LoByteShift, MVT::i8));
7773 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi,
7774 DAG.getConstant(HiByteShift, MVT::i8));
7775 return DAG.getNode(ISD::BITCAST, DL, VT,
7776 DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift));
7779 /// \brief Compute whether each element of a shuffle is zeroable.
7781 /// A "zeroable" vector shuffle element is one which can be lowered to zero.
7782 /// Either it is an undef element in the shuffle mask, the element of the input
7783 /// referenced is undef, or the element of the input referenced is known to be
7784 /// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
7785 /// as many lanes with this technique as possible to simplify the remaining
7787 static SmallBitVector computeZeroableShuffleElements(ArrayRef<int> Mask,
7788 SDValue V1, SDValue V2) {
7789 SmallBitVector Zeroable(Mask.size(), false);
7791 while (V1.getOpcode() == ISD::BITCAST)
7792 V1 = V1->getOperand(0);
7793 while (V2.getOpcode() == ISD::BITCAST)
7794 V2 = V2->getOperand(0);
7796 bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
7797 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
7799 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7801 // Handle the easy cases.
7802 if (M < 0 || (M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
7807 // If this is an index into a build_vector node (which has the same number
7808 // of elements), dig out the input value and use it.
7809 SDValue V = M < Size ? V1 : V2;
7810 if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands())
7813 SDValue Input = V.getOperand(M % Size);
7814 // The UNDEF opcode check really should be dead code here, but not quite
7815 // worth asserting on (it isn't invalid, just unexpected).
7816 if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input))
7823 /// \brief Try to emit a bitmask instruction for a shuffle.
7825 /// This handles cases where we can model a blend exactly as a bitmask due to
7826 /// one of the inputs being zeroable.
7827 static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1,
7828 SDValue V2, ArrayRef<int> Mask,
7829 SelectionDAG &DAG) {
7830 MVT EltVT = VT.getScalarType();
7831 int NumEltBits = EltVT.getSizeInBits();
7832 MVT IntEltVT = MVT::getIntegerVT(NumEltBits);
7833 SDValue Zero = DAG.getConstant(0, IntEltVT);
7834 SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), IntEltVT);
7835 if (EltVT.isFloatingPoint()) {
7836 Zero = DAG.getNode(ISD::BITCAST, DL, EltVT, Zero);
7837 AllOnes = DAG.getNode(ISD::BITCAST, DL, EltVT, AllOnes);
7839 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
7840 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7842 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7845 if (Mask[i] % Size != i)
7846 return SDValue(); // Not a blend.
7848 V = Mask[i] < Size ? V1 : V2;
7849 else if (V != (Mask[i] < Size ? V1 : V2))
7850 return SDValue(); // Can only let one input through the mask.
7852 VMaskOps[i] = AllOnes;
7855 return SDValue(); // No non-zeroable elements!
7857 SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps);
7858 V = DAG.getNode(VT.isFloatingPoint()
7859 ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND,
7864 /// \brief Try to lower a vector shuffle as a byte shift (shifts in zeros).
7866 /// Attempts to match a shuffle mask against the PSRLDQ and PSLLDQ
7867 /// byte-shift instructions. The mask must consist of a shifted sequential
7868 /// shuffle from one of the input vectors and zeroable elements for the
7869 /// remaining 'shifted in' elements.
7870 static SDValue lowerVectorShuffleAsByteShift(SDLoc DL, MVT VT, SDValue V1,
7871 SDValue V2, ArrayRef<int> Mask,
7872 SelectionDAG &DAG) {
7873 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
7875 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7877 int NumElts = VT.getVectorNumElements();
7878 int NumLanes = VT.getSizeInBits() / 128;
7879 int NumLaneElts = NumElts / NumLanes;
7880 int Scale = 16 / NumLaneElts;
7881 MVT ShiftVT = MVT::getVectorVT(MVT::i64, 2 * NumLanes);
7883 // PSLLDQ : (little-endian) left byte shift
7884 // [ zz, 0, 1, 2, 3, 4, 5, 6]
7885 // [ zz, zz, -1, -1, 2, 3, 4, -1]
7886 // [ zz, zz, zz, zz, zz, zz, -1, 1]
7887 // PSRLDQ : (little-endian) right byte shift
7888 // [ 5, 6, 7, zz, zz, zz, zz, zz]
7889 // [ -1, 5, 6, 7, zz, zz, zz, zz]
7890 // [ 1, 2, -1, -1, -1, -1, zz, zz]
7891 auto MatchByteShift = [&](int Shift) -> SDValue {
7892 bool MatchLeft = true, MatchRight = true;
7893 for (int l = 0; l < NumElts; l += NumLaneElts) {
7894 for (int i = 0; i < Shift; ++i)
7895 MatchLeft &= Zeroable[l + i];
7896 for (int i = NumLaneElts - Shift; i < NumLaneElts; ++i)
7897 MatchRight &= Zeroable[l + i];
7899 if (!(MatchLeft || MatchRight))
7902 bool MatchV1 = true, MatchV2 = true;
7903 for (int l = 0; l < NumElts; l += NumLaneElts) {
7904 unsigned Pos = MatchLeft ? Shift + l : l;
7905 unsigned Low = MatchLeft ? l : Shift + l;
7906 unsigned Len = NumLaneElts - Shift;
7907 MatchV1 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low);
7908 MatchV2 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low + NumElts);
7910 if (!(MatchV1 || MatchV2))
7913 int ByteShift = Shift * Scale;
7914 unsigned Op = MatchRight ? X86ISD::VSRLDQ : X86ISD::VSHLDQ;
7915 SDValue V = MatchV1 ? V1 : V2;
7916 V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V);
7917 V = DAG.getNode(Op, DL, ShiftVT, V,
7918 DAG.getConstant(ByteShift, MVT::i8));
7919 return DAG.getNode(ISD::BITCAST, DL, VT, V);
7922 for (int Shift = 1; Shift < NumLaneElts; ++Shift)
7923 if (SDValue S = MatchByteShift(Shift))
7930 /// \brief Try to lower a vector shuffle as a bit shift (shifts in zeros).
7932 /// Attempts to match a shuffle mask against the PSRL(W/D/Q) and PSLL(W/D/Q)
7933 /// SSE2 and AVX2 logical bit-shift instructions. The function matches
7934 /// elements from one of the input vectors shuffled to the left or right
7935 /// with zeroable elements 'shifted in'.
7936 static SDValue lowerVectorShuffleAsBitShift(SDLoc DL, MVT VT, SDValue V1,
7937 SDValue V2, ArrayRef<int> Mask,
7938 SelectionDAG &DAG) {
7939 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7941 int Size = Mask.size();
7942 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
7944 // PSRL : (little-endian) right bit shift.
7947 // PSHL : (little-endian) left bit shift.
7949 // [ -1, 4, zz, -1 ]
7950 auto MatchBitShift = [&](int Shift, int Scale) -> SDValue {
7951 MVT ShiftSVT = MVT::getIntegerVT(VT.getScalarSizeInBits() * Scale);
7952 MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale);
7953 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
7954 "Illegal integer vector type");
7956 bool MatchLeft = true, MatchRight = true;
7957 for (int i = 0; i != Size; i += Scale) {
7958 for (int j = 0; j != Shift; ++j) {
7959 MatchLeft &= Zeroable[i + j];
7961 for (int j = Scale - Shift; j != Scale; ++j) {
7962 MatchRight &= Zeroable[i + j];
7965 if (!(MatchLeft || MatchRight))
7968 bool MatchV1 = true, MatchV2 = true;
7969 for (int i = 0; i != Size; i += Scale) {
7970 unsigned Pos = MatchLeft ? i + Shift : i;
7971 unsigned Low = MatchLeft ? i : i + Shift;
7972 unsigned Len = Scale - Shift;
7973 MatchV1 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low);
7974 MatchV2 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low + Size);
7976 if (!(MatchV1 || MatchV2))
7979 // Cast the inputs to ShiftVT to match VSRLI/VSHLI and back again.
7980 unsigned OpCode = MatchLeft ? X86ISD::VSHLI : X86ISD::VSRLI;
7981 int ShiftAmt = Shift * VT.getScalarSizeInBits();
7982 SDValue V = MatchV1 ? V1 : V2;
7983 V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V);
7984 V = DAG.getNode(OpCode, DL, ShiftVT, V, DAG.getConstant(ShiftAmt, MVT::i8));
7985 return DAG.getNode(ISD::BITCAST, DL, VT, V);
7988 // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
7989 // keep doubling the size of the integer elements up to that. We can
7990 // then shift the elements of the integer vector by whole multiples of
7991 // their width within the elements of the larger integer vector. Test each
7992 // multiple to see if we can find a match with the moved element indices
7993 // and that the shifted in elements are all zeroable.
7994 for (int Scale = 2; Scale * VT.getScalarSizeInBits() <= 64; Scale *= 2)
7995 for (int Shift = 1; Shift != Scale; ++Shift)
7996 if (SDValue BitShift = MatchBitShift(Shift, Scale))
8003 /// \brief Lower a vector shuffle as a zero or any extension.
8005 /// Given a specific number of elements, element bit width, and extension
8006 /// stride, produce either a zero or any extension based on the available
8007 /// features of the subtarget.
8008 static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
8009 SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV,
8010 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
8011 assert(Scale > 1 && "Need a scale to extend.");
8012 int NumElements = VT.getVectorNumElements();
8013 int EltBits = VT.getScalarSizeInBits();
8014 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
8015 "Only 8, 16, and 32 bit elements can be extended.");
8016 assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits.");
8018 // Found a valid zext mask! Try various lowering strategies based on the
8019 // input type and available ISA extensions.
8020 if (Subtarget->hasSSE41()) {
8021 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
8022 NumElements / Scale);
8023 return DAG.getNode(ISD::BITCAST, DL, VT,
8024 DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV));
8027 // For any extends we can cheat for larger element sizes and use shuffle
8028 // instructions that can fold with a load and/or copy.
8029 if (AnyExt && EltBits == 32) {
8030 int PSHUFDMask[4] = {0, -1, 1, -1};
8032 ISD::BITCAST, DL, VT,
8033 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
8034 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV),
8035 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
8037 if (AnyExt && EltBits == 16 && Scale > 2) {
8038 int PSHUFDMask[4] = {0, -1, 0, -1};
8039 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
8040 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV),
8041 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG));
8042 int PSHUFHWMask[4] = {1, -1, -1, -1};
8044 ISD::BITCAST, DL, VT,
8045 DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16,
8046 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV),
8047 getV4X86ShuffleImm8ForMask(PSHUFHWMask, DAG)));
8050 // If this would require more than 2 unpack instructions to expand, use
8051 // pshufb when available. We can only use more than 2 unpack instructions
8052 // when zero extending i8 elements which also makes it easier to use pshufb.
8053 if (Scale > 4 && EltBits == 8 && Subtarget->hasSSSE3()) {
8054 assert(NumElements == 16 && "Unexpected byte vector width!");
8055 SDValue PSHUFBMask[16];
8056 for (int i = 0; i < 16; ++i)
8058 DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, MVT::i8);
8059 InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV);
8060 return DAG.getNode(ISD::BITCAST, DL, VT,
8061 DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
8062 DAG.getNode(ISD::BUILD_VECTOR, DL,
8063 MVT::v16i8, PSHUFBMask)));
8066 // Otherwise emit a sequence of unpacks.
8068 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
8069 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
8070 : getZeroVector(InputVT, Subtarget, DAG, DL);
8071 InputV = DAG.getNode(ISD::BITCAST, DL, InputVT, InputV);
8072 InputV = DAG.getNode(X86ISD::UNPCKL, DL, InputVT, InputV, Ext);
8076 } while (Scale > 1);
8077 return DAG.getNode(ISD::BITCAST, DL, VT, InputV);
8080 /// \brief Try to lower a vector shuffle as a zero extension on any microarch.
8082 /// This routine will try to do everything in its power to cleverly lower
8083 /// a shuffle which happens to match the pattern of a zero extend. It doesn't
8084 /// check for the profitability of this lowering, it tries to aggressively
8085 /// match this pattern. It will use all of the micro-architectural details it
8086 /// can to emit an efficient lowering. It handles both blends with all-zero
8087 /// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
8088 /// masking out later).
8090 /// The reason we have dedicated lowering for zext-style shuffles is that they
8091 /// are both incredibly common and often quite performance sensitive.
8092 static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
8093 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
8094 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
8095 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8097 int Bits = VT.getSizeInBits();
8098 int NumElements = VT.getVectorNumElements();
8099 assert(VT.getScalarSizeInBits() <= 32 &&
8100 "Exceeds 32-bit integer zero extension limit");
8101 assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size");
8103 // Define a helper function to check a particular ext-scale and lower to it if
8105 auto Lower = [&](int Scale) -> SDValue {
8108 for (int i = 0; i < NumElements; ++i) {
8110 continue; // Valid anywhere but doesn't tell us anything.
8111 if (i % Scale != 0) {
8112 // Each of the extended elements need to be zeroable.
8116 // We no longer are in the anyext case.
8121 // Each of the base elements needs to be consecutive indices into the
8122 // same input vector.
8123 SDValue V = Mask[i] < NumElements ? V1 : V2;
8126 else if (InputV != V)
8127 return SDValue(); // Flip-flopping inputs.
8129 if (Mask[i] % NumElements != i / Scale)
8130 return SDValue(); // Non-consecutive strided elements.
8133 // If we fail to find an input, we have a zero-shuffle which should always
8134 // have already been handled.
8135 // FIXME: Maybe handle this here in case during blending we end up with one?
8139 return lowerVectorShuffleAsSpecificZeroOrAnyExtend(
8140 DL, VT, Scale, AnyExt, InputV, Subtarget, DAG);
8143 // The widest scale possible for extending is to a 64-bit integer.
8144 assert(Bits % 64 == 0 &&
8145 "The number of bits in a vector must be divisible by 64 on x86!");
8146 int NumExtElements = Bits / 64;
8148 // Each iteration, try extending the elements half as much, but into twice as
8150 for (; NumExtElements < NumElements; NumExtElements *= 2) {
8151 assert(NumElements % NumExtElements == 0 &&
8152 "The input vector size must be divisible by the extended size.");
8153 if (SDValue V = Lower(NumElements / NumExtElements))
8157 // General extends failed, but 128-bit vectors may be able to use MOVQ.
8161 // Returns one of the source operands if the shuffle can be reduced to a
8162 // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits.
8163 auto CanZExtLowHalf = [&]() {
8164 for (int i = NumElements / 2; i != NumElements; ++i)
8167 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0))
8169 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements))
8174 if (SDValue V = CanZExtLowHalf()) {
8175 V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V);
8176 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
8177 return DAG.getNode(ISD::BITCAST, DL, VT, V);
8180 // No viable ext lowering found.
8184 /// \brief Try to get a scalar value for a specific element of a vector.
8186 /// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
8187 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
8188 SelectionDAG &DAG) {
8189 MVT VT = V.getSimpleValueType();
8190 MVT EltVT = VT.getVectorElementType();
8191 while (V.getOpcode() == ISD::BITCAST)
8192 V = V.getOperand(0);
8193 // If the bitcasts shift the element size, we can't extract an equivalent
8195 MVT NewVT = V.getSimpleValueType();
8196 if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
8199 if (V.getOpcode() == ISD::BUILD_VECTOR ||
8200 (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR))
8201 return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, V.getOperand(Idx));
8206 /// \brief Helper to test for a load that can be folded with x86 shuffles.
8208 /// This is particularly important because the set of instructions varies
8209 /// significantly based on whether the operand is a load or not.
8210 static bool isShuffleFoldableLoad(SDValue V) {
8211 while (V.getOpcode() == ISD::BITCAST)
8212 V = V.getOperand(0);
8214 return ISD::isNON_EXTLoad(V.getNode());
8217 /// \brief Try to lower insertion of a single element into a zero vector.
8219 /// This is a common pattern that we have especially efficient patterns to lower
8220 /// across all subtarget feature sets.
8221 static SDValue lowerVectorShuffleAsElementInsertion(
8222 MVT VT, SDLoc DL, SDValue V1, SDValue V2, ArrayRef<int> Mask,
8223 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
8224 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8226 MVT EltVT = VT.getVectorElementType();
8228 int V2Index = std::find_if(Mask.begin(), Mask.end(),
8229 [&Mask](int M) { return M >= (int)Mask.size(); }) -
8231 bool IsV1Zeroable = true;
8232 for (int i = 0, Size = Mask.size(); i < Size; ++i)
8233 if (i != V2Index && !Zeroable[i]) {
8234 IsV1Zeroable = false;
8238 // Check for a single input from a SCALAR_TO_VECTOR node.
8239 // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and
8240 // all the smarts here sunk into that routine. However, the current
8241 // lowering of BUILD_VECTOR makes that nearly impossible until the old
8242 // vector shuffle lowering is dead.
8243 if (SDValue V2S = getScalarValueForVectorElement(
8244 V2, Mask[V2Index] - Mask.size(), DAG)) {
8245 // We need to zext the scalar if it is smaller than an i32.
8246 V2S = DAG.getNode(ISD::BITCAST, DL, EltVT, V2S);
8247 if (EltVT == MVT::i8 || EltVT == MVT::i16) {
8248 // Using zext to expand a narrow element won't work for non-zero
8253 // Zero-extend directly to i32.
8255 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
8257 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
8258 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||
8259 EltVT == MVT::i16) {
8260 // Either not inserting from the low element of the input or the input
8261 // element size is too small to use VZEXT_MOVL to clear the high bits.
8265 if (!IsV1Zeroable) {
8266 // If V1 can't be treated as a zero vector we have fewer options to lower
8267 // this. We can't support integer vectors or non-zero targets cheaply, and
8268 // the V1 elements can't be permuted in any way.
8269 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");
8270 if (!VT.isFloatingPoint() || V2Index != 0)
8272 SmallVector<int, 8> V1Mask(Mask.begin(), Mask.end());
8273 V1Mask[V2Index] = -1;
8274 if (!isNoopShuffleMask(V1Mask))
8276 // This is essentially a special case blend operation, but if we have
8277 // general purpose blend operations, they are always faster. Bail and let
8278 // the rest of the lowering handle these as blends.
8279 if (Subtarget->hasSSE41())
8282 // Otherwise, use MOVSD or MOVSS.
8283 assert((EltVT == MVT::f32 || EltVT == MVT::f64) &&
8284 "Only two types of floating point element types to handle!");
8285 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL,
8289 // This lowering only works for the low element with floating point vectors.
8290 if (VT.isFloatingPoint() && V2Index != 0)
8293 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
8295 V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2);
8298 // If we have 4 or fewer lanes we can cheaply shuffle the element into
8299 // the desired position. Otherwise it is more efficient to do a vector
8300 // shift left. We know that we can do a vector shift left because all
8301 // the inputs are zero.
8302 if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) {
8303 SmallVector<int, 4> V2Shuffle(Mask.size(), 1);
8304 V2Shuffle[V2Index] = 0;
8305 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
8307 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2);
8309 X86ISD::VSHLDQ, DL, MVT::v2i64, V2,
8311 V2Index * EltVT.getSizeInBits()/8,
8312 DAG.getTargetLoweringInfo().getScalarShiftAmountTy(MVT::v2i64)));
8313 V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2);
8319 /// \brief Try to lower broadcast of a single element.
8321 /// For convenience, this code also bundles all of the subtarget feature set
8322 /// filtering. While a little annoying to re-dispatch on type here, there isn't
8323 /// a convenient way to factor it out.
8324 static SDValue lowerVectorShuffleAsBroadcast(MVT VT, SDLoc DL, SDValue V,
8326 const X86Subtarget *Subtarget,
8327 SelectionDAG &DAG) {
8328 if (!Subtarget->hasAVX())
8330 if (VT.isInteger() && !Subtarget->hasAVX2())
8333 // Check that the mask is a broadcast.
8334 int BroadcastIdx = -1;
8336 if (M >= 0 && BroadcastIdx == -1)
8338 else if (M >= 0 && M != BroadcastIdx)
8341 assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
8342 "a sorted mask where the broadcast "
8345 // Go up the chain of (vector) values to try and find a scalar load that
8346 // we can combine with the broadcast.
8348 switch (V.getOpcode()) {
8349 case ISD::CONCAT_VECTORS: {
8350 int OperandSize = Mask.size() / V.getNumOperands();
8351 V = V.getOperand(BroadcastIdx / OperandSize);
8352 BroadcastIdx %= OperandSize;
8356 case ISD::INSERT_SUBVECTOR: {
8357 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
8358 auto ConstantIdx = dyn_cast<ConstantSDNode>(V.getOperand(2));
8362 int BeginIdx = (int)ConstantIdx->getZExtValue();
8364 BeginIdx + (int)VInner.getValueType().getVectorNumElements();
8365 if (BroadcastIdx >= BeginIdx && BroadcastIdx < EndIdx) {
8366 BroadcastIdx -= BeginIdx;
8377 // Check if this is a broadcast of a scalar. We special case lowering
8378 // for scalars so that we can more effectively fold with loads.
8379 if (V.getOpcode() == ISD::BUILD_VECTOR ||
8380 (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0)) {
8381 V = V.getOperand(BroadcastIdx);
8383 // If the scalar isn't a load we can't broadcast from it in AVX1, only with
8385 if (!Subtarget->hasAVX2() && !isShuffleFoldableLoad(V))
8387 } else if (BroadcastIdx != 0 || !Subtarget->hasAVX2()) {
8388 // We can't broadcast from a vector register w/o AVX2, and we can only
8389 // broadcast from the zero-element of a vector register.
8393 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, V);
8396 // Check for whether we can use INSERTPS to perform the shuffle. We only use
8397 // INSERTPS when the V1 elements are already in the correct locations
8398 // because otherwise we can just always use two SHUFPS instructions which
8399 // are much smaller to encode than a SHUFPS and an INSERTPS. We can also
8400 // perform INSERTPS if a single V1 element is out of place and all V2
8401 // elements are zeroable.
8402 static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2,
8404 SelectionDAG &DAG) {
8405 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!");
8406 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8407 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8408 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
8410 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8413 int V1DstIndex = -1;
8414 int V2DstIndex = -1;
8415 bool V1UsedInPlace = false;
8417 for (int i = 0; i < 4; ++i) {
8418 // Synthesize a zero mask from the zeroable elements (includes undefs).
8424 // Flag if we use any V1 inputs in place.
8426 V1UsedInPlace = true;
8430 // We can only insert a single non-zeroable element.
8431 if (V1DstIndex != -1 || V2DstIndex != -1)
8435 // V1 input out of place for insertion.
8438 // V2 input for insertion.
8443 // Don't bother if we have no (non-zeroable) element for insertion.
8444 if (V1DstIndex == -1 && V2DstIndex == -1)
8447 // Determine element insertion src/dst indices. The src index is from the
8448 // start of the inserted vector, not the start of the concatenated vector.
8449 unsigned V2SrcIndex = 0;
8450 if (V1DstIndex != -1) {
8451 // If we have a V1 input out of place, we use V1 as the V2 element insertion
8452 // and don't use the original V2 at all.
8453 V2SrcIndex = Mask[V1DstIndex];
8454 V2DstIndex = V1DstIndex;
8457 V2SrcIndex = Mask[V2DstIndex] - 4;
8460 // If no V1 inputs are used in place, then the result is created only from
8461 // the zero mask and the V2 insertion - so remove V1 dependency.
8463 V1 = DAG.getUNDEF(MVT::v4f32);
8465 unsigned InsertPSMask = V2SrcIndex << 6 | V2DstIndex << 4 | ZMask;
8466 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
8468 // Insert the V2 element into the desired position.
8470 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
8471 DAG.getConstant(InsertPSMask, MVT::i8));
8474 /// \brief Try to lower a shuffle as a permute of the inputs followed by an
8475 /// UNPCK instruction.
8477 /// This specifically targets cases where we end up with alternating between
8478 /// the two inputs, and so can permute them into something that feeds a single
8479 /// UNPCK instruction. Note that this routine only targets integer vectors
8480 /// because for floating point vectors we have a generalized SHUFPS lowering
8481 /// strategy that handles everything that doesn't *exactly* match an unpack,
8482 /// making this clever lowering unnecessary.
8483 static SDValue lowerVectorShuffleAsUnpack(MVT VT, SDLoc DL, SDValue V1,
8484 SDValue V2, ArrayRef<int> Mask,
8485 SelectionDAG &DAG) {
8486 assert(!VT.isFloatingPoint() &&
8487 "This routine only supports integer vectors.");
8488 assert(!isSingleInputShuffleMask(Mask) &&
8489 "This routine should only be used when blending two inputs.");
8490 assert(Mask.size() >= 2 && "Single element masks are invalid.");
8492 int Size = Mask.size();
8494 int NumLoInputs = std::count_if(Mask.begin(), Mask.end(), [Size](int M) {
8495 return M >= 0 && M % Size < Size / 2;
8497 int NumHiInputs = std::count_if(
8498 Mask.begin(), Mask.end(), [Size](int M) { return M % Size > Size / 2; });
8500 bool UnpackLo = NumLoInputs >= NumHiInputs;
8502 auto TryUnpack = [&](MVT UnpackVT, int Scale) {
8503 SmallVector<int, 32> V1Mask(Mask.size(), -1);
8504 SmallVector<int, 32> V2Mask(Mask.size(), -1);
8506 for (int i = 0; i < Size; ++i) {
8510 // Each element of the unpack contains Scale elements from this mask.
8511 int UnpackIdx = i / Scale;
8513 // We only handle the case where V1 feeds the first slots of the unpack.
8514 // We rely on canonicalization to ensure this is the case.
8515 if ((UnpackIdx % 2 == 0) != (Mask[i] < Size))
8518 // Setup the mask for this input. The indexing is tricky as we have to
8519 // handle the unpack stride.
8520 SmallVectorImpl<int> &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask;
8521 VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] =
8525 // Shuffle the inputs into place.
8526 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
8527 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
8529 // Cast the inputs to the type we will use to unpack them.
8530 V1 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V1);
8531 V2 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V2);
8533 // Unpack the inputs and cast the result back to the desired type.
8534 return DAG.getNode(ISD::BITCAST, DL, VT,
8535 DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
8536 DL, UnpackVT, V1, V2));
8539 // We try each unpack from the largest to the smallest to try and find one
8540 // that fits this mask.
8541 int OrigNumElements = VT.getVectorNumElements();
8542 int OrigScalarSize = VT.getScalarSizeInBits();
8543 for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2) {
8544 int Scale = ScalarSize / OrigScalarSize;
8545 int NumElements = OrigNumElements / Scale;
8546 MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), NumElements);
8547 if (SDValue Unpack = TryUnpack(UnpackVT, Scale))
8554 /// \brief Handle lowering of 2-lane 64-bit floating point shuffles.
8556 /// This is the basis function for the 2-lane 64-bit shuffles as we have full
8557 /// support for floating point shuffles but not integer shuffles. These
8558 /// instructions will incur a domain crossing penalty on some chips though so
8559 /// it is better to avoid lowering through this for integer vectors where
8561 static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8562 const X86Subtarget *Subtarget,
8563 SelectionDAG &DAG) {
8565 assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!");
8566 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
8567 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
8568 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8569 ArrayRef<int> Mask = SVOp->getMask();
8570 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
8572 if (isSingleInputShuffleMask(Mask)) {
8573 // Use low duplicate instructions for masks that match their pattern.
8574 if (Subtarget->hasSSE3())
8575 if (isShuffleEquivalent(V1, V2, Mask, 0, 0))
8576 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, V1);
8578 // Straight shuffle of a single input vector. Simulate this by using the
8579 // single input as both of the "inputs" to this instruction..
8580 unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
8582 if (Subtarget->hasAVX()) {
8583 // If we have AVX, we can use VPERMILPS which will allow folding a load
8584 // into the shuffle.
8585 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
8586 DAG.getConstant(SHUFPDMask, MVT::i8));
8589 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1,
8590 DAG.getConstant(SHUFPDMask, MVT::i8));
8592 assert(Mask[0] >= 0 && Mask[0] < 2 && "Non-canonicalized blend!");
8593 assert(Mask[1] >= 2 && "Non-canonicalized blend!");
8595 // If we have a single input, insert that into V1 if we can do so cheaply.
8596 if ((Mask[0] >= 2) + (Mask[1] >= 2) == 1) {
8597 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8598 MVT::v2f64, DL, V1, V2, Mask, Subtarget, DAG))
8600 // Try inverting the insertion since for v2 masks it is easy to do and we
8601 // can't reliably sort the mask one way or the other.
8602 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
8603 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
8604 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8605 MVT::v2f64, DL, V2, V1, InverseMask, Subtarget, DAG))
8609 // Try to use one of the special instruction patterns to handle two common
8610 // blend patterns if a zero-blend above didn't work.
8611 if (isShuffleEquivalent(V1, V2, Mask, 0, 3) || isShuffleEquivalent(V1, V2, Mask, 1, 3))
8612 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
8613 // We can either use a special instruction to load over the low double or
8614 // to move just the low double.
8616 isShuffleFoldableLoad(V1S) ? X86ISD::MOVLPD : X86ISD::MOVSD,
8618 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
8620 if (Subtarget->hasSSE41())
8621 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
8625 // Use dedicated unpack instructions for masks that match their pattern.
8626 if (isShuffleEquivalent(V1, V2, Mask, 0, 2))
8627 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2f64, V1, V2);
8628 if (isShuffleEquivalent(V1, V2, Mask, 1, 3))
8629 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2f64, V1, V2);
8631 unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
8632 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2,
8633 DAG.getConstant(SHUFPDMask, MVT::i8));
8636 /// \brief Handle lowering of 2-lane 64-bit integer shuffles.
8638 /// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
8639 /// the integer unit to minimize domain crossing penalties. However, for blends
8640 /// it falls back to the floating point shuffle operation with appropriate bit
8642 static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8643 const X86Subtarget *Subtarget,
8644 SelectionDAG &DAG) {
8646 assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!");
8647 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
8648 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
8649 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8650 ArrayRef<int> Mask = SVOp->getMask();
8651 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
8653 if (isSingleInputShuffleMask(Mask)) {
8654 // Check for being able to broadcast a single element.
8655 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v2i64, DL, V1,
8656 Mask, Subtarget, DAG))
8659 // Straight shuffle of a single input vector. For everything from SSE2
8660 // onward this has a single fast instruction with no scary immediates.
8661 // We have to map the mask as it is actually a v4i32 shuffle instruction.
8662 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1);
8663 int WidenedMask[4] = {
8664 std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1,
8665 std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1};
8667 ISD::BITCAST, DL, MVT::v2i64,
8668 DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1,
8669 getV4X86ShuffleImm8ForMask(WidenedMask, DAG)));
8672 // Try to use byte shift instructions.
8673 if (SDValue Shift = lowerVectorShuffleAsByteShift(
8674 DL, MVT::v2i64, V1, V2, Mask, DAG))
8677 // If we have a single input from V2 insert that into V1 if we can do so
8679 if ((Mask[0] >= 2) + (Mask[1] >= 2) == 1) {
8680 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8681 MVT::v2i64, DL, V1, V2, Mask, Subtarget, DAG))
8683 // Try inverting the insertion since for v2 masks it is easy to do and we
8684 // can't reliably sort the mask one way or the other.
8685 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
8686 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
8687 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8688 MVT::v2i64, DL, V2, V1, InverseMask, Subtarget, DAG))
8692 // We have different paths for blend lowering, but they all must use the
8693 // *exact* same predicate.
8694 bool IsBlendSupported = Subtarget->hasSSE41();
8695 if (IsBlendSupported)
8696 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
8700 // Use dedicated unpack instructions for masks that match their pattern.
8701 if (isShuffleEquivalent(V1, V2, Mask, 0, 2))
8702 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, V1, V2);
8703 if (isShuffleEquivalent(V1, V2, Mask, 1, 3))
8704 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2i64, V1, V2);
8706 // Try to use byte rotation instructions.
8707 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
8708 if (Subtarget->hasSSSE3())
8709 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
8710 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
8713 // If we have direct support for blends, we should lower by decomposing into
8714 // a permute. That will be faster than the domain cross.
8715 if (IsBlendSupported)
8716 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2,
8719 // We implement this with SHUFPD which is pretty lame because it will likely
8720 // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
8721 // However, all the alternatives are still more cycles and newer chips don't
8722 // have this problem. It would be really nice if x86 had better shuffles here.
8723 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1);
8724 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2);
8725 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
8726 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
8729 /// \brief Test whether this can be lowered with a single SHUFPS instruction.
8731 /// This is used to disable more specialized lowerings when the shufps lowering
8732 /// will happen to be efficient.
8733 static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
8734 // This routine only handles 128-bit shufps.
8735 assert(Mask.size() == 4 && "Unsupported mask size!");
8737 // To lower with a single SHUFPS we need to have the low half and high half
8738 // each requiring a single input.
8739 if (Mask[0] != -1 && Mask[1] != -1 && (Mask[0] < 4) != (Mask[1] < 4))
8741 if (Mask[2] != -1 && Mask[3] != -1 && (Mask[2] < 4) != (Mask[3] < 4))
8747 /// \brief Lower a vector shuffle using the SHUFPS instruction.
8749 /// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
8750 /// It makes no assumptions about whether this is the *best* lowering, it simply
8752 static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT,
8753 ArrayRef<int> Mask, SDValue V1,
8754 SDValue V2, SelectionDAG &DAG) {
8755 SDValue LowV = V1, HighV = V2;
8756 int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]};
8759 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
8761 if (NumV2Elements == 1) {
8763 std::find_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }) -
8766 // Compute the index adjacent to V2Index and in the same half by toggling
8768 int V2AdjIndex = V2Index ^ 1;
8770 if (Mask[V2AdjIndex] == -1) {
8771 // Handles all the cases where we have a single V2 element and an undef.
8772 // This will only ever happen in the high lanes because we commute the
8773 // vector otherwise.
8775 std::swap(LowV, HighV);
8776 NewMask[V2Index] -= 4;
8778 // Handle the case where the V2 element ends up adjacent to a V1 element.
8779 // To make this work, blend them together as the first step.
8780 int V1Index = V2AdjIndex;
8781 int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
8782 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
8783 getV4X86ShuffleImm8ForMask(BlendMask, DAG));
8785 // Now proceed to reconstruct the final blend as we have the necessary
8786 // high or low half formed.
8793 NewMask[V1Index] = 2; // We put the V1 element in V2[2].
8794 NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
8796 } else if (NumV2Elements == 2) {
8797 if (Mask[0] < 4 && Mask[1] < 4) {
8798 // Handle the easy case where we have V1 in the low lanes and V2 in the
8802 } else if (Mask[2] < 4 && Mask[3] < 4) {
8803 // We also handle the reversed case because this utility may get called
8804 // when we detect a SHUFPS pattern but can't easily commute the shuffle to
8805 // arrange things in the right direction.
8811 // We have a mixture of V1 and V2 in both low and high lanes. Rather than
8812 // trying to place elements directly, just blend them and set up the final
8813 // shuffle to place them.
8815 // The first two blend mask elements are for V1, the second two are for
8817 int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
8818 Mask[2] < 4 ? Mask[2] : Mask[3],
8819 (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
8820 (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
8821 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
8822 getV4X86ShuffleImm8ForMask(BlendMask, DAG));
8824 // Now we do a normal shuffle of V1 by giving V1 as both operands to
8827 NewMask[0] = Mask[0] < 4 ? 0 : 2;
8828 NewMask[1] = Mask[0] < 4 ? 2 : 0;
8829 NewMask[2] = Mask[2] < 4 ? 1 : 3;
8830 NewMask[3] = Mask[2] < 4 ? 3 : 1;
8833 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
8834 getV4X86ShuffleImm8ForMask(NewMask, DAG));
8837 /// \brief Lower 4-lane 32-bit floating point shuffles.
8839 /// Uses instructions exclusively from the floating point unit to minimize
8840 /// domain crossing penalties, as these are sufficient to implement all v4f32
8842 static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8843 const X86Subtarget *Subtarget,
8844 SelectionDAG &DAG) {
8846 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!");
8847 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8848 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8849 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8850 ArrayRef<int> Mask = SVOp->getMask();
8851 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
8854 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
8856 if (NumV2Elements == 0) {
8857 // Check for being able to broadcast a single element.
8858 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f32, DL, V1,
8859 Mask, Subtarget, DAG))
8862 // Use even/odd duplicate instructions for masks that match their pattern.
8863 if (Subtarget->hasSSE3()) {
8864 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2))
8865 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
8866 if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3))
8867 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
8870 if (Subtarget->hasAVX()) {
8871 // If we have AVX, we can use VPERMILPS which will allow folding a load
8872 // into the shuffle.
8873 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
8874 getV4X86ShuffleImm8ForMask(Mask, DAG));
8877 // Otherwise, use a straight shuffle of a single input vector. We pass the
8878 // input vector to both operands to simulate this with a SHUFPS.
8879 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
8880 getV4X86ShuffleImm8ForMask(Mask, DAG));
8883 // There are special ways we can lower some single-element blends. However, we
8884 // have custom ways we can lower more complex single-element blends below that
8885 // we defer to if both this and BLENDPS fail to match, so restrict this to
8886 // when the V2 input is targeting element 0 of the mask -- that is the fast
8888 if (NumV2Elements == 1 && Mask[0] >= 4)
8889 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4f32, DL, V1, V2,
8890 Mask, Subtarget, DAG))
8893 if (Subtarget->hasSSE41()) {
8894 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
8898 // Use INSERTPS if we can complete the shuffle efficiently.
8899 if (SDValue V = lowerVectorShuffleAsInsertPS(Op, V1, V2, Mask, DAG))
8902 if (!isSingleSHUFPSMask(Mask))
8903 if (SDValue BlendPerm = lowerVectorShuffleAsBlendAndPermute(
8904 DL, MVT::v4f32, V1, V2, Mask, DAG))
8908 // Use dedicated unpack instructions for masks that match their pattern.
8909 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5))
8910 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V1, V2);
8911 if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7))
8912 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V1, V2);
8914 // Otherwise fall back to a SHUFPS lowering strategy.
8915 return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
8918 /// \brief Lower 4-lane i32 vector shuffles.
8920 /// We try to handle these with integer-domain shuffles where we can, but for
8921 /// blends we use the floating point domain blend instructions.
8922 static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8923 const X86Subtarget *Subtarget,
8924 SelectionDAG &DAG) {
8926 assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!");
8927 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
8928 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
8929 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8930 ArrayRef<int> Mask = SVOp->getMask();
8931 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
8933 // Whenever we can lower this as a zext, that instruction is strictly faster
8934 // than any alternative. It also allows us to fold memory operands into the
8935 // shuffle in many cases.
8936 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2,
8937 Mask, Subtarget, DAG))
8941 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
8943 if (NumV2Elements == 0) {
8944 // Check for being able to broadcast a single element.
8945 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i32, DL, V1,
8946 Mask, Subtarget, DAG))
8949 // Straight shuffle of a single input vector. For everything from SSE2
8950 // onward this has a single fast instruction with no scary immediates.
8951 // We coerce the shuffle pattern to be compatible with UNPCK instructions
8952 // but we aren't actually going to use the UNPCK instruction because doing
8953 // so prevents folding a load into this instruction or making a copy.
8954 const int UnpackLoMask[] = {0, 0, 1, 1};
8955 const int UnpackHiMask[] = {2, 2, 3, 3};
8956 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 1, 1))
8957 Mask = UnpackLoMask;
8958 else if (isShuffleEquivalent(V1, V2, Mask, 2, 2, 3, 3))
8959 Mask = UnpackHiMask;
8961 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
8962 getV4X86ShuffleImm8ForMask(Mask, DAG));
8965 // Try to use bit shift instructions.
8966 if (SDValue Shift = lowerVectorShuffleAsBitShift(
8967 DL, MVT::v4i32, V1, V2, Mask, DAG))
8970 // Try to use byte shift instructions.
8971 if (SDValue Shift = lowerVectorShuffleAsByteShift(
8972 DL, MVT::v4i32, V1, V2, Mask, DAG))
8975 // There are special ways we can lower some single-element blends.
8976 if (NumV2Elements == 1)
8977 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4i32, DL, V1, V2,
8978 Mask, Subtarget, DAG))
8981 // We have different paths for blend lowering, but they all must use the
8982 // *exact* same predicate.
8983 bool IsBlendSupported = Subtarget->hasSSE41();
8984 if (IsBlendSupported)
8985 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
8989 if (SDValue Masked =
8990 lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG))
8993 // Use dedicated unpack instructions for masks that match their pattern.
8994 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5))
8995 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V1, V2);
8996 if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7))
8997 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V1, V2);
8999 // Try to use byte rotation instructions.
9000 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
9001 if (Subtarget->hasSSSE3())
9002 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9003 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG))
9006 // If we have direct support for blends, we should lower by decomposing into
9007 // a permute. That will be faster than the domain cross.
9008 if (IsBlendSupported)
9009 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2,
9012 // Try to lower by permuting the inputs into an unpack instruction.
9013 if (SDValue Unpack =
9014 lowerVectorShuffleAsUnpack(MVT::v4i32, DL, V1, V2, Mask, DAG))
9017 // We implement this with SHUFPS because it can blend from two vectors.
9018 // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
9019 // up the inputs, bypassing domain shift penalties that we would encur if we
9020 // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
9022 return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32,
9023 DAG.getVectorShuffle(
9025 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1),
9026 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask));
9029 /// \brief Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
9030 /// shuffle lowering, and the most complex part.
9032 /// The lowering strategy is to try to form pairs of input lanes which are
9033 /// targeted at the same half of the final vector, and then use a dword shuffle
9034 /// to place them onto the right half, and finally unpack the paired lanes into
9035 /// their final position.
9037 /// The exact breakdown of how to form these dword pairs and align them on the
9038 /// correct sides is really tricky. See the comments within the function for
9039 /// more of the details.
9040 static SDValue lowerV8I16SingleInputVectorShuffle(
9041 SDLoc DL, SDValue V, MutableArrayRef<int> Mask,
9042 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
9043 assert(V.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
9044 MutableArrayRef<int> LoMask = Mask.slice(0, 4);
9045 MutableArrayRef<int> HiMask = Mask.slice(4, 4);
9047 SmallVector<int, 4> LoInputs;
9048 std::copy_if(LoMask.begin(), LoMask.end(), std::back_inserter(LoInputs),
9049 [](int M) { return M >= 0; });
9050 std::sort(LoInputs.begin(), LoInputs.end());
9051 LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end());
9052 SmallVector<int, 4> HiInputs;
9053 std::copy_if(HiMask.begin(), HiMask.end(), std::back_inserter(HiInputs),
9054 [](int M) { return M >= 0; });
9055 std::sort(HiInputs.begin(), HiInputs.end());
9056 HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end());
9058 std::lower_bound(LoInputs.begin(), LoInputs.end(), 4) - LoInputs.begin();
9059 int NumHToL = LoInputs.size() - NumLToL;
9061 std::lower_bound(HiInputs.begin(), HiInputs.end(), 4) - HiInputs.begin();
9062 int NumHToH = HiInputs.size() - NumLToH;
9063 MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
9064 MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
9065 MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
9066 MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
9068 // Check for being able to broadcast a single element.
9069 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i16, DL, V,
9070 Mask, Subtarget, DAG))
9073 // Try to use bit shift instructions.
9074 if (SDValue Shift = lowerVectorShuffleAsBitShift(
9075 DL, MVT::v8i16, V, V, Mask, DAG))
9078 // Try to use byte shift instructions.
9079 if (SDValue Shift = lowerVectorShuffleAsByteShift(
9080 DL, MVT::v8i16, V, V, Mask, DAG))
9083 // Use dedicated unpack instructions for masks that match their pattern.
9084 if (isShuffleEquivalent(V, V, Mask, 0, 0, 1, 1, 2, 2, 3, 3))
9085 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V, V);
9086 if (isShuffleEquivalent(V, V, Mask, 4, 4, 5, 5, 6, 6, 7, 7))
9087 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V, V);
9089 // Try to use byte rotation instructions.
9090 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9091 DL, MVT::v8i16, V, V, Mask, Subtarget, DAG))
9094 // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
9095 // such inputs we can swap two of the dwords across the half mark and end up
9096 // with <=2 inputs to each half in each half. Once there, we can fall through
9097 // to the generic code below. For example:
9099 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
9100 // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
9102 // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half
9103 // and an existing 2-into-2 on the other half. In this case we may have to
9104 // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or
9105 // 1-into-3 which could cause us to cycle endlessly fixing each side in turn.
9106 // Fortunately, we don't have to handle anything but a 2-into-2 pattern
9107 // because any other situation (including a 3-into-1 or 1-into-3 in the other
9108 // half than the one we target for fixing) will be fixed when we re-enter this
9109 // path. We will also combine away any sequence of PSHUFD instructions that
9110 // result into a single instruction. Here is an example of the tricky case:
9112 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
9113 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3]
9115 // This now has a 1-into-3 in the high half! Instead, we do two shuffles:
9117 // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h]
9118 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6]
9120 // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h]
9121 // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6]
9123 // The result is fine to be handled by the generic logic.
9124 auto balanceSides = [&](ArrayRef<int> AToAInputs, ArrayRef<int> BToAInputs,
9125 ArrayRef<int> BToBInputs, ArrayRef<int> AToBInputs,
9126 int AOffset, int BOffset) {
9127 assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
9128 "Must call this with A having 3 or 1 inputs from the A half.");
9129 assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
9130 "Must call this with B having 1 or 3 inputs from the B half.");
9131 assert(AToAInputs.size() + BToAInputs.size() == 4 &&
9132 "Must call this with either 3:1 or 1:3 inputs (summing to 4).");
9134 // Compute the index of dword with only one word among the three inputs in
9135 // a half by taking the sum of the half with three inputs and subtracting
9136 // the sum of the actual three inputs. The difference is the remaining
9139 int &TripleDWord = AToAInputs.size() == 3 ? ADWord : BDWord;
9140 int &OneInputDWord = AToAInputs.size() == 3 ? BDWord : ADWord;
9141 int TripleInputOffset = AToAInputs.size() == 3 ? AOffset : BOffset;
9142 ArrayRef<int> TripleInputs = AToAInputs.size() == 3 ? AToAInputs : BToAInputs;
9143 int OneInput = AToAInputs.size() == 3 ? BToAInputs[0] : AToAInputs[0];
9144 int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
9145 int TripleNonInputIdx =
9146 TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
9147 TripleDWord = TripleNonInputIdx / 2;
9149 // We use xor with one to compute the adjacent DWord to whichever one the
9151 OneInputDWord = (OneInput / 2) ^ 1;
9153 // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA
9154 // and BToA inputs. If there is also such a problem with the BToB and AToB
9155 // inputs, we don't try to fix it necessarily -- we'll recurse and see it in
9156 // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it
9157 // is essential that we don't *create* a 3<-1 as then we might oscillate.
9158 if (BToBInputs.size() == 2 && AToBInputs.size() == 2) {
9159 // Compute how many inputs will be flipped by swapping these DWords. We
9161 // to balance this to ensure we don't form a 3-1 shuffle in the other
9163 int NumFlippedAToBInputs =
9164 std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) +
9165 std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1);
9166 int NumFlippedBToBInputs =
9167 std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) +
9168 std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1);
9169 if ((NumFlippedAToBInputs == 1 &&
9170 (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
9171 (NumFlippedBToBInputs == 1 &&
9172 (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) {
9173 // We choose whether to fix the A half or B half based on whether that
9174 // half has zero flipped inputs. At zero, we may not be able to fix it
9175 // with that half. We also bias towards fixing the B half because that
9176 // will more commonly be the high half, and we have to bias one way.
9177 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
9178 ArrayRef<int> Inputs) {
9179 int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot.
9180 bool IsFixIdxInput = std::find(Inputs.begin(), Inputs.end(),
9181 PinnedIdx ^ 1) != Inputs.end();
9182 // Determine whether the free index is in the flipped dword or the
9183 // unflipped dword based on where the pinned index is. We use this bit
9184 // in an xor to conditionally select the adjacent dword.
9185 int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord));
9186 bool IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(),
9187 FixFreeIdx) != Inputs.end();
9188 if (IsFixIdxInput == IsFixFreeIdxInput)
9190 IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(),
9191 FixFreeIdx) != Inputs.end();
9192 assert(IsFixIdxInput != IsFixFreeIdxInput &&
9193 "We need to be changing the number of flipped inputs!");
9194 int PSHUFHalfMask[] = {0, 1, 2, 3};
9195 std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
9196 V = DAG.getNode(FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
9198 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DAG));
9201 if (M != -1 && M == FixIdx)
9203 else if (M != -1 && M == FixFreeIdx)
9206 if (NumFlippedBToBInputs != 0) {
9208 BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
9209 FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs);
9211 assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!");
9213 AToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
9214 FixFlippedInputs(APinnedIdx, ADWord, AToBInputs);
9219 int PSHUFDMask[] = {0, 1, 2, 3};
9220 PSHUFDMask[ADWord] = BDWord;
9221 PSHUFDMask[BDWord] = ADWord;
9222 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
9223 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
9224 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V),
9225 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
9227 // Adjust the mask to match the new locations of A and B.
9229 if (M != -1 && M/2 == ADWord)
9230 M = 2 * BDWord + M % 2;
9231 else if (M != -1 && M/2 == BDWord)
9232 M = 2 * ADWord + M % 2;
9234 // Recurse back into this routine to re-compute state now that this isn't
9235 // a 3 and 1 problem.
9236 return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16),
9239 if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
9240 return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
9241 else if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3))
9242 return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0);
9244 // At this point there are at most two inputs to the low and high halves from
9245 // each half. That means the inputs can always be grouped into dwords and
9246 // those dwords can then be moved to the correct half with a dword shuffle.
9247 // We use at most one low and one high word shuffle to collect these paired
9248 // inputs into dwords, and finally a dword shuffle to place them.
9249 int PSHUFLMask[4] = {-1, -1, -1, -1};
9250 int PSHUFHMask[4] = {-1, -1, -1, -1};
9251 int PSHUFDMask[4] = {-1, -1, -1, -1};
9253 // First fix the masks for all the inputs that are staying in their
9254 // original halves. This will then dictate the targets of the cross-half
9256 auto fixInPlaceInputs =
9257 [&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
9258 MutableArrayRef<int> SourceHalfMask,
9259 MutableArrayRef<int> HalfMask, int HalfOffset) {
9260 if (InPlaceInputs.empty())
9262 if (InPlaceInputs.size() == 1) {
9263 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
9264 InPlaceInputs[0] - HalfOffset;
9265 PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
9268 if (IncomingInputs.empty()) {
9269 // Just fix all of the in place inputs.
9270 for (int Input : InPlaceInputs) {
9271 SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
9272 PSHUFDMask[Input / 2] = Input / 2;
9277 assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
9278 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
9279 InPlaceInputs[0] - HalfOffset;
9280 // Put the second input next to the first so that they are packed into
9281 // a dword. We find the adjacent index by toggling the low bit.
9282 int AdjIndex = InPlaceInputs[0] ^ 1;
9283 SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
9284 std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
9285 PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
9287 fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
9288 fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
9290 // Now gather the cross-half inputs and place them into a free dword of
9291 // their target half.
9292 // FIXME: This operation could almost certainly be simplified dramatically to
9293 // look more like the 3-1 fixing operation.
9294 auto moveInputsToRightHalf = [&PSHUFDMask](
9295 MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
9296 MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
9297 MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
9299 auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
9300 return SourceHalfMask[Word] != -1 && SourceHalfMask[Word] != Word;
9302 auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
9304 int LowWord = Word & ~1;
9305 int HighWord = Word | 1;
9306 return isWordClobbered(SourceHalfMask, LowWord) ||
9307 isWordClobbered(SourceHalfMask, HighWord);
9310 if (IncomingInputs.empty())
9313 if (ExistingInputs.empty()) {
9314 // Map any dwords with inputs from them into the right half.
9315 for (int Input : IncomingInputs) {
9316 // If the source half mask maps over the inputs, turn those into
9317 // swaps and use the swapped lane.
9318 if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
9319 if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == -1) {
9320 SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
9321 Input - SourceOffset;
9322 // We have to swap the uses in our half mask in one sweep.
9323 for (int &M : HalfMask)
9324 if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset)
9326 else if (M == Input)
9327 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
9329 assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==
9330 Input - SourceOffset &&
9331 "Previous placement doesn't match!");
9333 // Note that this correctly re-maps both when we do a swap and when
9334 // we observe the other side of the swap above. We rely on that to
9335 // avoid swapping the members of the input list directly.
9336 Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
9339 // Map the input's dword into the correct half.
9340 if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == -1)
9341 PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
9343 assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==
9345 "Previous placement doesn't match!");
9348 // And just directly shift any other-half mask elements to be same-half
9349 // as we will have mirrored the dword containing the element into the
9350 // same position within that half.
9351 for (int &M : HalfMask)
9352 if (M >= SourceOffset && M < SourceOffset + 4) {
9353 M = M - SourceOffset + DestOffset;
9354 assert(M >= 0 && "This should never wrap below zero!");
9359 // Ensure we have the input in a viable dword of its current half. This
9360 // is particularly tricky because the original position may be clobbered
9361 // by inputs being moved and *staying* in that half.
9362 if (IncomingInputs.size() == 1) {
9363 if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
9364 int InputFixed = std::find(std::begin(SourceHalfMask),
9365 std::end(SourceHalfMask), -1) -
9366 std::begin(SourceHalfMask) + SourceOffset;
9367 SourceHalfMask[InputFixed - SourceOffset] =
9368 IncomingInputs[0] - SourceOffset;
9369 std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
9371 IncomingInputs[0] = InputFixed;
9373 } else if (IncomingInputs.size() == 2) {
9374 if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
9375 isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
9376 // We have two non-adjacent or clobbered inputs we need to extract from
9377 // the source half. To do this, we need to map them into some adjacent
9378 // dword slot in the source mask.
9379 int InputsFixed[2] = {IncomingInputs[0] - SourceOffset,
9380 IncomingInputs[1] - SourceOffset};
9382 // If there is a free slot in the source half mask adjacent to one of
9383 // the inputs, place the other input in it. We use (Index XOR 1) to
9384 // compute an adjacent index.
9385 if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) &&
9386 SourceHalfMask[InputsFixed[0] ^ 1] == -1) {
9387 SourceHalfMask[InputsFixed[0]] = InputsFixed[0];
9388 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
9389 InputsFixed[1] = InputsFixed[0] ^ 1;
9390 } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) &&
9391 SourceHalfMask[InputsFixed[1] ^ 1] == -1) {
9392 SourceHalfMask[InputsFixed[1]] = InputsFixed[1];
9393 SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0];
9394 InputsFixed[0] = InputsFixed[1] ^ 1;
9395 } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] == -1 &&
9396 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] == -1) {
9397 // The two inputs are in the same DWord but it is clobbered and the
9398 // adjacent DWord isn't used at all. Move both inputs to the free
9400 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0];
9401 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1];
9402 InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1);
9403 InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1;
9405 // The only way we hit this point is if there is no clobbering
9406 // (because there are no off-half inputs to this half) and there is no
9407 // free slot adjacent to one of the inputs. In this case, we have to
9408 // swap an input with a non-input.
9409 for (int i = 0; i < 4; ++i)
9410 assert((SourceHalfMask[i] == -1 || SourceHalfMask[i] == i) &&
9411 "We can't handle any clobbers here!");
9412 assert(InputsFixed[1] != (InputsFixed[0] ^ 1) &&
9413 "Cannot have adjacent inputs here!");
9415 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
9416 SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1;
9418 // We also have to update the final source mask in this case because
9419 // it may need to undo the above swap.
9420 for (int &M : FinalSourceHalfMask)
9421 if (M == (InputsFixed[0] ^ 1) + SourceOffset)
9422 M = InputsFixed[1] + SourceOffset;
9423 else if (M == InputsFixed[1] + SourceOffset)
9424 M = (InputsFixed[0] ^ 1) + SourceOffset;
9426 InputsFixed[1] = InputsFixed[0] ^ 1;
9429 // Point everything at the fixed inputs.
9430 for (int &M : HalfMask)
9431 if (M == IncomingInputs[0])
9432 M = InputsFixed[0] + SourceOffset;
9433 else if (M == IncomingInputs[1])
9434 M = InputsFixed[1] + SourceOffset;
9436 IncomingInputs[0] = InputsFixed[0] + SourceOffset;
9437 IncomingInputs[1] = InputsFixed[1] + SourceOffset;
9440 llvm_unreachable("Unhandled input size!");
9443 // Now hoist the DWord down to the right half.
9444 int FreeDWord = (PSHUFDMask[DestOffset / 2] == -1 ? 0 : 1) + DestOffset / 2;
9445 assert(PSHUFDMask[FreeDWord] == -1 && "DWord not free");
9446 PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
9447 for (int &M : HalfMask)
9448 for (int Input : IncomingInputs)
9450 M = FreeDWord * 2 + Input % 2;
9452 moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask,
9453 /*SourceOffset*/ 4, /*DestOffset*/ 0);
9454 moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask,
9455 /*SourceOffset*/ 0, /*DestOffset*/ 4);
9457 // Now enact all the shuffles we've computed to move the inputs into their
9459 if (!isNoopShuffleMask(PSHUFLMask))
9460 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
9461 getV4X86ShuffleImm8ForMask(PSHUFLMask, DAG));
9462 if (!isNoopShuffleMask(PSHUFHMask))
9463 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
9464 getV4X86ShuffleImm8ForMask(PSHUFHMask, DAG));
9465 if (!isNoopShuffleMask(PSHUFDMask))
9466 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
9467 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
9468 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V),
9469 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
9471 // At this point, each half should contain all its inputs, and we can then
9472 // just shuffle them into their final position.
9473 assert(std::count_if(LoMask.begin(), LoMask.end(),
9474 [](int M) { return M >= 4; }) == 0 &&
9475 "Failed to lift all the high half inputs to the low mask!");
9476 assert(std::count_if(HiMask.begin(), HiMask.end(),
9477 [](int M) { return M >= 0 && M < 4; }) == 0 &&
9478 "Failed to lift all the low half inputs to the high mask!");
9480 // Do a half shuffle for the low mask.
9481 if (!isNoopShuffleMask(LoMask))
9482 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
9483 getV4X86ShuffleImm8ForMask(LoMask, DAG));
9485 // Do a half shuffle with the high mask after shifting its values down.
9486 for (int &M : HiMask)
9489 if (!isNoopShuffleMask(HiMask))
9490 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
9491 getV4X86ShuffleImm8ForMask(HiMask, DAG));
9496 /// \brief Detect whether the mask pattern should be lowered through
9499 /// This essentially tests whether viewing the mask as an interleaving of two
9500 /// sub-sequences reduces the cross-input traffic of a blend operation. If so,
9501 /// lowering it through interleaving is a significantly better strategy.
9502 static bool shouldLowerAsInterleaving(ArrayRef<int> Mask) {
9503 int NumEvenInputs[2] = {0, 0};
9504 int NumOddInputs[2] = {0, 0};
9505 int NumLoInputs[2] = {0, 0};
9506 int NumHiInputs[2] = {0, 0};
9507 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
9511 int InputIdx = Mask[i] >= Size;
9514 ++NumLoInputs[InputIdx];
9516 ++NumHiInputs[InputIdx];
9519 ++NumEvenInputs[InputIdx];
9521 ++NumOddInputs[InputIdx];
9524 // The minimum number of cross-input results for both the interleaved and
9525 // split cases. If interleaving results in fewer cross-input results, return
9527 int InterleavedCrosses = std::min(NumEvenInputs[1] + NumOddInputs[0],
9528 NumEvenInputs[0] + NumOddInputs[1]);
9529 int SplitCrosses = std::min(NumLoInputs[1] + NumHiInputs[0],
9530 NumLoInputs[0] + NumHiInputs[1]);
9531 return InterleavedCrosses < SplitCrosses;
9534 /// \brief Blend two v8i16 vectors using a naive unpack strategy.
9536 /// This strategy only works when the inputs from each vector fit into a single
9537 /// half of that vector, and generally there are not so many inputs as to leave
9538 /// the in-place shuffles required highly constrained (and thus expensive). It
9539 /// shifts all the inputs into a single side of both input vectors and then
9540 /// uses an unpack to interleave these inputs in a single vector. At that
9541 /// point, we will fall back on the generic single input shuffle lowering.
9542 static SDValue lowerV8I16BasicBlendVectorShuffle(SDLoc DL, SDValue V1,
9544 MutableArrayRef<int> Mask,
9545 const X86Subtarget *Subtarget,
9546 SelectionDAG &DAG) {
9547 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
9548 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
9549 SmallVector<int, 3> LoV1Inputs, HiV1Inputs, LoV2Inputs, HiV2Inputs;
9550 for (int i = 0; i < 8; ++i)
9551 if (Mask[i] >= 0 && Mask[i] < 4)
9552 LoV1Inputs.push_back(i);
9553 else if (Mask[i] >= 4 && Mask[i] < 8)
9554 HiV1Inputs.push_back(i);
9555 else if (Mask[i] >= 8 && Mask[i] < 12)
9556 LoV2Inputs.push_back(i);
9557 else if (Mask[i] >= 12)
9558 HiV2Inputs.push_back(i);
9560 int NumV1Inputs = LoV1Inputs.size() + HiV1Inputs.size();
9561 int NumV2Inputs = LoV2Inputs.size() + HiV2Inputs.size();
9564 assert(NumV1Inputs > 0 && NumV1Inputs <= 3 && "At most 3 inputs supported");
9565 assert(NumV2Inputs > 0 && NumV2Inputs <= 3 && "At most 3 inputs supported");
9566 assert(NumV1Inputs + NumV2Inputs <= 4 && "At most 4 combined inputs");
9568 bool MergeFromLo = LoV1Inputs.size() + LoV2Inputs.size() >=
9569 HiV1Inputs.size() + HiV2Inputs.size();
9571 auto moveInputsToHalf = [&](SDValue V, ArrayRef<int> LoInputs,
9572 ArrayRef<int> HiInputs, bool MoveToLo,
9574 ArrayRef<int> GoodInputs = MoveToLo ? LoInputs : HiInputs;
9575 ArrayRef<int> BadInputs = MoveToLo ? HiInputs : LoInputs;
9576 if (BadInputs.empty())
9579 int MoveMask[] = {-1, -1, -1, -1, -1, -1, -1, -1};
9580 int MoveOffset = MoveToLo ? 0 : 4;
9582 if (GoodInputs.empty()) {
9583 for (int BadInput : BadInputs) {
9584 MoveMask[Mask[BadInput] % 4 + MoveOffset] = Mask[BadInput] - MaskOffset;
9585 Mask[BadInput] = Mask[BadInput] % 4 + MoveOffset + MaskOffset;
9588 if (GoodInputs.size() == 2) {
9589 // If the low inputs are spread across two dwords, pack them into
9591 MoveMask[MoveOffset] = Mask[GoodInputs[0]] - MaskOffset;
9592 MoveMask[MoveOffset + 1] = Mask[GoodInputs[1]] - MaskOffset;
9593 Mask[GoodInputs[0]] = MoveOffset + MaskOffset;
9594 Mask[GoodInputs[1]] = MoveOffset + 1 + MaskOffset;
9596 // Otherwise pin the good inputs.
9597 for (int GoodInput : GoodInputs)
9598 MoveMask[Mask[GoodInput] - MaskOffset] = Mask[GoodInput] - MaskOffset;
9601 if (BadInputs.size() == 2) {
9602 // If we have two bad inputs then there may be either one or two good
9603 // inputs fixed in place. Find a fixed input, and then find the *other*
9604 // two adjacent indices by using modular arithmetic.
9606 std::find_if(std::begin(MoveMask) + MoveOffset, std::end(MoveMask),
9607 [](int M) { return M >= 0; }) -
9608 std::begin(MoveMask);
9610 ((((GoodMaskIdx - MoveOffset) & ~1) + 2) % 4) + MoveOffset;
9611 assert(MoveMask[MoveMaskIdx] == -1 && "Expected empty slot");
9612 assert(MoveMask[MoveMaskIdx + 1] == -1 && "Expected empty slot");
9613 MoveMask[MoveMaskIdx] = Mask[BadInputs[0]] - MaskOffset;
9614 MoveMask[MoveMaskIdx + 1] = Mask[BadInputs[1]] - MaskOffset;
9615 Mask[BadInputs[0]] = MoveMaskIdx + MaskOffset;
9616 Mask[BadInputs[1]] = MoveMaskIdx + 1 + MaskOffset;
9618 assert(BadInputs.size() == 1 && "All sizes handled");
9619 int MoveMaskIdx = std::find(std::begin(MoveMask) + MoveOffset,
9620 std::end(MoveMask), -1) -
9621 std::begin(MoveMask);
9622 MoveMask[MoveMaskIdx] = Mask[BadInputs[0]] - MaskOffset;
9623 Mask[BadInputs[0]] = MoveMaskIdx + MaskOffset;
9627 return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16),
9630 V1 = moveInputsToHalf(V1, LoV1Inputs, HiV1Inputs, MergeFromLo,
9632 V2 = moveInputsToHalf(V2, LoV2Inputs, HiV2Inputs, MergeFromLo,
9635 // FIXME: Select an interleaving of the merge of V1 and V2 that minimizes
9636 // cross-half traffic in the final shuffle.
9638 // Munge the mask to be a single-input mask after the unpack merges the
9642 M = 2 * (M % 4) + (M / 8);
9644 return DAG.getVectorShuffle(
9645 MVT::v8i16, DL, DAG.getNode(MergeFromLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
9646 DL, MVT::v8i16, V1, V2),
9647 DAG.getUNDEF(MVT::v8i16), Mask);
9650 /// \brief Generic lowering of 8-lane i16 shuffles.
9652 /// This handles both single-input shuffles and combined shuffle/blends with
9653 /// two inputs. The single input shuffles are immediately delegated to
9654 /// a dedicated lowering routine.
9656 /// The blends are lowered in one of three fundamental ways. If there are few
9657 /// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
9658 /// of the input is significantly cheaper when lowered as an interleaving of
9659 /// the two inputs, try to interleave them. Otherwise, blend the low and high
9660 /// halves of the inputs separately (making them have relatively few inputs)
9661 /// and then concatenate them.
9662 static SDValue lowerV8I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
9663 const X86Subtarget *Subtarget,
9664 SelectionDAG &DAG) {
9666 assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!");
9667 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
9668 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
9669 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
9670 ArrayRef<int> OrigMask = SVOp->getMask();
9671 int MaskStorage[8] = {OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3],
9672 OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7]};
9673 MutableArrayRef<int> Mask(MaskStorage);
9675 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
9677 // Whenever we can lower this as a zext, that instruction is strictly faster
9678 // than any alternative.
9679 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
9680 DL, MVT::v8i16, V1, V2, OrigMask, Subtarget, DAG))
9683 auto isV1 = [](int M) { return M >= 0 && M < 8; };
9684 auto isV2 = [](int M) { return M >= 8; };
9686 int NumV1Inputs = std::count_if(Mask.begin(), Mask.end(), isV1);
9687 int NumV2Inputs = std::count_if(Mask.begin(), Mask.end(), isV2);
9689 if (NumV2Inputs == 0)
9690 return lowerV8I16SingleInputVectorShuffle(DL, V1, Mask, Subtarget, DAG);
9692 assert(NumV1Inputs > 0 && "All single-input shuffles should be canonicalized "
9693 "to be V1-input shuffles.");
9695 // Try to use bit shift instructions.
9696 if (SDValue Shift = lowerVectorShuffleAsBitShift(
9697 DL, MVT::v8i16, V1, V2, Mask, DAG))
9700 // Try to use byte shift instructions.
9701 if (SDValue Shift = lowerVectorShuffleAsByteShift(
9702 DL, MVT::v8i16, V1, V2, Mask, DAG))
9705 // There are special ways we can lower some single-element blends.
9706 if (NumV2Inputs == 1)
9707 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v8i16, DL, V1, V2,
9708 Mask, Subtarget, DAG))
9711 // We have different paths for blend lowering, but they all must use the
9712 // *exact* same predicate.
9713 bool IsBlendSupported = Subtarget->hasSSE41();
9714 if (IsBlendSupported)
9715 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
9719 if (SDValue Masked =
9720 lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG))
9723 // Use dedicated unpack instructions for masks that match their pattern.
9724 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 2, 10, 3, 11))
9725 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V1, V2);
9726 if (isShuffleEquivalent(V1, V2, Mask, 4, 12, 5, 13, 6, 14, 7, 15))
9727 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V1, V2);
9729 // Try to use byte rotation instructions.
9730 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9731 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
9734 if (NumV1Inputs + NumV2Inputs <= 4)
9735 return lowerV8I16BasicBlendVectorShuffle(DL, V1, V2, Mask, Subtarget, DAG);
9737 // Check whether an interleaving lowering is likely to be more efficient.
9738 // This isn't perfect but it is a strong heuristic that tends to work well on
9739 // the kinds of shuffles that show up in practice.
9741 // FIXME: Handle 1x, 2x, and 4x interleaving.
9742 if (shouldLowerAsInterleaving(Mask)) {
9743 // FIXME: Figure out whether we should pack these into the low or high
9746 int EMask[8], OMask[8];
9747 for (int i = 0; i < 4; ++i) {
9748 EMask[i] = Mask[2*i];
9749 OMask[i] = Mask[2*i + 1];
9754 SDValue Evens = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, EMask);
9755 SDValue Odds = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, OMask);
9757 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, Evens, Odds);
9760 // If we have direct support for blends, we should lower by decomposing into
9762 if (IsBlendSupported)
9763 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
9766 // Try to lower by permuting the inputs into an unpack instruction.
9767 if (SDValue Unpack =
9768 lowerVectorShuffleAsUnpack(MVT::v8i16, DL, V1, V2, Mask, DAG))
9771 int LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
9772 int HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
9774 for (int i = 0; i < 4; ++i) {
9775 LoBlendMask[i] = Mask[i];
9776 HiBlendMask[i] = Mask[i + 4];
9779 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask);
9780 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask);
9781 LoV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, LoV);
9782 HiV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, HiV);
9784 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
9785 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, LoV, HiV));
9788 /// \brief Check whether a compaction lowering can be done by dropping even
9789 /// elements and compute how many times even elements must be dropped.
9791 /// This handles shuffles which take every Nth element where N is a power of
9792 /// two. Example shuffle masks:
9794 /// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14
9795 /// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
9796 /// N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12
9797 /// N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28
9798 /// N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8
9799 /// N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
9801 /// Any of these lanes can of course be undef.
9803 /// This routine only supports N <= 3.
9804 /// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
9807 /// \returns N above, or the number of times even elements must be dropped if
9808 /// there is such a number. Otherwise returns zero.
9809 static int canLowerByDroppingEvenElements(ArrayRef<int> Mask) {
9810 // Figure out whether we're looping over two inputs or just one.
9811 bool IsSingleInput = isSingleInputShuffleMask(Mask);
9813 // The modulus for the shuffle vector entries is based on whether this is
9814 // a single input or not.
9815 int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
9816 assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
9817 "We should only be called with masks with a power-of-2 size!");
9819 uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
9821 // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
9822 // and 2^3 simultaneously. This is because we may have ambiguity with
9823 // partially undef inputs.
9824 bool ViableForN[3] = {true, true, true};
9826 for (int i = 0, e = Mask.size(); i < e; ++i) {
9827 // Ignore undef lanes, we'll optimistically collapse them to the pattern we
9832 bool IsAnyViable = false;
9833 for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
9834 if (ViableForN[j]) {
9837 // The shuffle mask must be equal to (i * 2^N) % M.
9838 if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask))
9841 ViableForN[j] = false;
9843 // Early exit if we exhaust the possible powers of two.
9848 for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
9852 // Return 0 as there is no viable power of two.
9856 /// \brief Generic lowering of v16i8 shuffles.
9858 /// This is a hybrid strategy to lower v16i8 vectors. It first attempts to
9859 /// detect any complexity reducing interleaving. If that doesn't help, it uses
9860 /// UNPCK to spread the i8 elements across two i16-element vectors, and uses
9861 /// the existing lowering for v8i16 blends on each half, finally PACK-ing them
9863 static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
9864 const X86Subtarget *Subtarget,
9865 SelectionDAG &DAG) {
9867 assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!");
9868 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
9869 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
9870 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
9871 ArrayRef<int> OrigMask = SVOp->getMask();
9872 assert(OrigMask.size() == 16 && "Unexpected mask size for v16 shuffle!");
9874 // Try to use bit shift instructions.
9875 if (SDValue Shift = lowerVectorShuffleAsBitShift(
9876 DL, MVT::v16i8, V1, V2, OrigMask, DAG))
9879 // Try to use byte shift instructions.
9880 if (SDValue Shift = lowerVectorShuffleAsByteShift(
9881 DL, MVT::v16i8, V1, V2, OrigMask, DAG))
9884 // Try to use byte rotation instructions.
9885 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9886 DL, MVT::v16i8, V1, V2, OrigMask, Subtarget, DAG))
9889 // Try to use a zext lowering.
9890 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
9891 DL, MVT::v16i8, V1, V2, OrigMask, Subtarget, DAG))
9894 int MaskStorage[16] = {
9895 OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3],
9896 OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7],
9897 OrigMask[8], OrigMask[9], OrigMask[10], OrigMask[11],
9898 OrigMask[12], OrigMask[13], OrigMask[14], OrigMask[15]};
9899 MutableArrayRef<int> Mask(MaskStorage);
9900 MutableArrayRef<int> LoMask = Mask.slice(0, 8);
9901 MutableArrayRef<int> HiMask = Mask.slice(8, 8);
9904 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 16; });
9906 // For single-input shuffles, there are some nicer lowering tricks we can use.
9907 if (NumV2Elements == 0) {
9908 // Check for being able to broadcast a single element.
9909 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i8, DL, V1,
9910 Mask, Subtarget, DAG))
9913 // Check whether we can widen this to an i16 shuffle by duplicating bytes.
9914 // Notably, this handles splat and partial-splat shuffles more efficiently.
9915 // However, it only makes sense if the pre-duplication shuffle simplifies
9916 // things significantly. Currently, this means we need to be able to
9917 // express the pre-duplication shuffle as an i16 shuffle.
9919 // FIXME: We should check for other patterns which can be widened into an
9920 // i16 shuffle as well.
9921 auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
9922 for (int i = 0; i < 16; i += 2)
9923 if (Mask[i] != -1 && Mask[i + 1] != -1 && Mask[i] != Mask[i + 1])
9928 auto tryToWidenViaDuplication = [&]() -> SDValue {
9929 if (!canWidenViaDuplication(Mask))
9931 SmallVector<int, 4> LoInputs;
9932 std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(LoInputs),
9933 [](int M) { return M >= 0 && M < 8; });
9934 std::sort(LoInputs.begin(), LoInputs.end());
9935 LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()),
9937 SmallVector<int, 4> HiInputs;
9938 std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(HiInputs),
9939 [](int M) { return M >= 8; });
9940 std::sort(HiInputs.begin(), HiInputs.end());
9941 HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()),
9944 bool TargetLo = LoInputs.size() >= HiInputs.size();
9945 ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
9946 ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
9948 int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
9949 SmallDenseMap<int, int, 8> LaneMap;
9950 for (int I : InPlaceInputs) {
9951 PreDupI16Shuffle[I/2] = I/2;
9954 int j = TargetLo ? 0 : 4, je = j + 4;
9955 for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
9956 // Check if j is already a shuffle of this input. This happens when
9957 // there are two adjacent bytes after we move the low one.
9958 if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
9959 // If we haven't yet mapped the input, search for a slot into which
9961 while (j < je && PreDupI16Shuffle[j] != -1)
9965 // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
9968 // Map this input with the i16 shuffle.
9969 PreDupI16Shuffle[j] = MovingInputs[i] / 2;
9972 // Update the lane map based on the mapping we ended up with.
9973 LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
9976 ISD::BITCAST, DL, MVT::v16i8,
9977 DAG.getVectorShuffle(MVT::v8i16, DL,
9978 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
9979 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
9981 // Unpack the bytes to form the i16s that will be shuffled into place.
9982 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
9983 MVT::v16i8, V1, V1);
9985 int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
9986 for (int i = 0; i < 16; ++i)
9987 if (Mask[i] != -1) {
9988 int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
9989 assert(MappedMask < 8 && "Invalid v8 shuffle mask!");
9990 if (PostDupI16Shuffle[i / 2] == -1)
9991 PostDupI16Shuffle[i / 2] = MappedMask;
9993 assert(PostDupI16Shuffle[i / 2] == MappedMask &&
9994 "Conflicting entrties in the original shuffle!");
9997 ISD::BITCAST, DL, MVT::v16i8,
9998 DAG.getVectorShuffle(MVT::v8i16, DL,
9999 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
10000 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
10002 if (SDValue V = tryToWidenViaDuplication())
10006 // Check whether an interleaving lowering is likely to be more efficient.
10007 // This isn't perfect but it is a strong heuristic that tends to work well on
10008 // the kinds of shuffles that show up in practice.
10010 // FIXME: We need to handle other interleaving widths (i16, i32, ...).
10011 if (shouldLowerAsInterleaving(Mask)) {
10012 int NumLoHalf = std::count_if(Mask.begin(), Mask.end(), [](int M) {
10013 return (M >= 0 && M < 8) || (M >= 16 && M < 24);
10015 int NumHiHalf = std::count_if(Mask.begin(), Mask.end(), [](int M) {
10016 return (M >= 8 && M < 16) || M >= 24;
10018 int EMask[16] = {-1, -1, -1, -1, -1, -1, -1, -1,
10019 -1, -1, -1, -1, -1, -1, -1, -1};
10020 int OMask[16] = {-1, -1, -1, -1, -1, -1, -1, -1,
10021 -1, -1, -1, -1, -1, -1, -1, -1};
10022 bool UnpackLo = NumLoHalf >= NumHiHalf;
10023 MutableArrayRef<int> TargetEMask(UnpackLo ? EMask : EMask + 8, 8);
10024 MutableArrayRef<int> TargetOMask(UnpackLo ? OMask : OMask + 8, 8);
10025 for (int i = 0; i < 8; ++i) {
10026 TargetEMask[i] = Mask[2 * i];
10027 TargetOMask[i] = Mask[2 * i + 1];
10030 SDValue Evens = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, EMask);
10031 SDValue Odds = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, OMask);
10033 return DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
10034 MVT::v16i8, Evens, Odds);
10037 // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
10038 // with PSHUFB. It is important to do this before we attempt to generate any
10039 // blends but after all of the single-input lowerings. If the single input
10040 // lowerings can find an instruction sequence that is faster than a PSHUFB, we
10041 // want to preserve that and we can DAG combine any longer sequences into
10042 // a PSHUFB in the end. But once we start blending from multiple inputs,
10043 // the complexity of DAG combining bad patterns back into PSHUFB is too high,
10044 // and there are *very* few patterns that would actually be faster than the
10045 // PSHUFB approach because of its ability to zero lanes.
10047 // FIXME: The only exceptions to the above are blends which are exact
10048 // interleavings with direct instructions supporting them. We currently don't
10049 // handle those well here.
10050 if (Subtarget->hasSSSE3()) {
10051 SDValue V1Mask[16];
10052 SDValue V2Mask[16];
10053 bool V1InUse = false;
10054 bool V2InUse = false;
10055 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
10057 for (int i = 0; i < 16; ++i) {
10058 if (Mask[i] == -1) {
10059 V1Mask[i] = V2Mask[i] = DAG.getUNDEF(MVT::i8);
10061 const int ZeroMask = 0x80;
10062 int V1Idx = (Mask[i] < 16 ? Mask[i] : ZeroMask);
10063 int V2Idx = (Mask[i] < 16 ? ZeroMask : Mask[i] - 16);
10065 V1Idx = V2Idx = ZeroMask;
10066 V1Mask[i] = DAG.getConstant(V1Idx, MVT::i8);
10067 V2Mask[i] = DAG.getConstant(V2Idx, MVT::i8);
10068 V1InUse |= (ZeroMask != V1Idx);
10069 V2InUse |= (ZeroMask != V2Idx);
10073 // If both V1 and V2 are in use and we can use a direct blend, do so. This
10074 // avoids using blends to handle blends-with-zero which is important as
10075 // a single pshufb is significantly faster for that.
10076 if (V1InUse && V2InUse && Subtarget->hasSSE41())
10077 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
10083 V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, V1,
10084 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask));
10086 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, V2,
10087 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask));
10089 // If we need shuffled inputs from both, blend the two.
10090 if (V1InUse && V2InUse)
10091 return DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2);
10093 return V1; // Single inputs are easy.
10095 return V2; // Single inputs are easy.
10096 // Shuffling to a zeroable vector.
10097 return getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
10100 // There are special ways we can lower some single-element blends.
10101 if (NumV2Elements == 1)
10102 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v16i8, DL, V1, V2,
10103 Mask, Subtarget, DAG))
10106 // Check whether a compaction lowering can be done. This handles shuffles
10107 // which take every Nth element for some even N. See the helper function for
10110 // We special case these as they can be particularly efficiently handled with
10111 // the PACKUSB instruction on x86 and they show up in common patterns of
10112 // rearranging bytes to truncate wide elements.
10113 if (int NumEvenDrops = canLowerByDroppingEvenElements(Mask)) {
10114 // NumEvenDrops is the power of two stride of the elements. Another way of
10115 // thinking about it is that we need to drop the even elements this many
10116 // times to get the original input.
10117 bool IsSingleInput = isSingleInputShuffleMask(Mask);
10119 // First we need to zero all the dropped bytes.
10120 assert(NumEvenDrops <= 3 &&
10121 "No support for dropping even elements more than 3 times.");
10122 // We use the mask type to pick which bytes are preserved based on how many
10123 // elements are dropped.
10124 MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 };
10125 SDValue ByteClearMask =
10126 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8,
10127 DAG.getConstant(0xFF, MaskVTs[NumEvenDrops - 1]));
10128 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask);
10129 if (!IsSingleInput)
10130 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask);
10132 // Now pack things back together.
10133 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1);
10134 V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2);
10135 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
10136 for (int i = 1; i < NumEvenDrops; ++i) {
10137 Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result);
10138 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
10144 int V1LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10145 int V1HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10146 int V2LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10147 int V2HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10149 auto buildBlendMasks = [](MutableArrayRef<int> HalfMask,
10150 MutableArrayRef<int> V1HalfBlendMask,
10151 MutableArrayRef<int> V2HalfBlendMask) {
10152 for (int i = 0; i < 8; ++i)
10153 if (HalfMask[i] >= 0 && HalfMask[i] < 16) {
10154 V1HalfBlendMask[i] = HalfMask[i];
10156 } else if (HalfMask[i] >= 16) {
10157 V2HalfBlendMask[i] = HalfMask[i] - 16;
10158 HalfMask[i] = i + 8;
10161 buildBlendMasks(LoMask, V1LoBlendMask, V2LoBlendMask);
10162 buildBlendMasks(HiMask, V1HiBlendMask, V2HiBlendMask);
10164 SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL);
10166 auto buildLoAndHiV8s = [&](SDValue V, MutableArrayRef<int> LoBlendMask,
10167 MutableArrayRef<int> HiBlendMask) {
10169 // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
10170 // them out and avoid using UNPCK{L,H} to extract the elements of V as
10172 if (std::none_of(LoBlendMask.begin(), LoBlendMask.end(),
10173 [](int M) { return M >= 0 && M % 2 == 1; }) &&
10174 std::none_of(HiBlendMask.begin(), HiBlendMask.end(),
10175 [](int M) { return M >= 0 && M % 2 == 1; })) {
10176 // Use a mask to drop the high bytes.
10177 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
10178 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, V1,
10179 DAG.getConstant(0x00FF, MVT::v8i16));
10181 // This will be a single vector shuffle instead of a blend so nuke V2.
10182 V2 = DAG.getUNDEF(MVT::v8i16);
10184 // Squash the masks to point directly into V1.
10185 for (int &M : LoBlendMask)
10188 for (int &M : HiBlendMask)
10192 // Otherwise just unpack the low half of V into V1 and the high half into
10193 // V2 so that we can blend them as i16s.
10194 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
10195 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
10196 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
10197 DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
10200 SDValue BlendedLo = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask);
10201 SDValue BlendedHi = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask);
10202 return std::make_pair(BlendedLo, BlendedHi);
10204 SDValue V1Lo, V1Hi, V2Lo, V2Hi;
10205 std::tie(V1Lo, V1Hi) = buildLoAndHiV8s(V1, V1LoBlendMask, V1HiBlendMask);
10206 std::tie(V2Lo, V2Hi) = buildLoAndHiV8s(V2, V2LoBlendMask, V2HiBlendMask);
10208 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Lo, V2Lo, LoMask);
10209 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Hi, V2Hi, HiMask);
10211 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
10214 /// \brief Dispatching routine to lower various 128-bit x86 vector shuffles.
10216 /// This routine breaks down the specific type of 128-bit shuffle and
10217 /// dispatches to the lowering routines accordingly.
10218 static SDValue lower128BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10219 MVT VT, const X86Subtarget *Subtarget,
10220 SelectionDAG &DAG) {
10221 switch (VT.SimpleTy) {
10223 return lowerV2I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
10225 return lowerV2F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
10227 return lowerV4I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
10229 return lowerV4F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
10231 return lowerV8I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
10233 return lowerV16I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
10236 llvm_unreachable("Unimplemented!");
10240 /// \brief Helper function to test whether a shuffle mask could be
10241 /// simplified by widening the elements being shuffled.
10243 /// Appends the mask for wider elements in WidenedMask if valid. Otherwise
10244 /// leaves it in an unspecified state.
10246 /// NOTE: This must handle normal vector shuffle masks and *target* vector
10247 /// shuffle masks. The latter have the special property of a '-2' representing
10248 /// a zero-ed lane of a vector.
10249 static bool canWidenShuffleElements(ArrayRef<int> Mask,
10250 SmallVectorImpl<int> &WidenedMask) {
10251 for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
10252 // If both elements are undef, its trivial.
10253 if (Mask[i] == SM_SentinelUndef && Mask[i + 1] == SM_SentinelUndef) {
10254 WidenedMask.push_back(SM_SentinelUndef);
10258 // Check for an undef mask and a mask value properly aligned to fit with
10259 // a pair of values. If we find such a case, use the non-undef mask's value.
10260 if (Mask[i] == SM_SentinelUndef && Mask[i + 1] >= 0 && Mask[i + 1] % 2 == 1) {
10261 WidenedMask.push_back(Mask[i + 1] / 2);
10264 if (Mask[i + 1] == SM_SentinelUndef && Mask[i] >= 0 && Mask[i] % 2 == 0) {
10265 WidenedMask.push_back(Mask[i] / 2);
10269 // When zeroing, we need to spread the zeroing across both lanes to widen.
10270 if (Mask[i] == SM_SentinelZero || Mask[i + 1] == SM_SentinelZero) {
10271 if ((Mask[i] == SM_SentinelZero || Mask[i] == SM_SentinelUndef) &&
10272 (Mask[i + 1] == SM_SentinelZero || Mask[i + 1] == SM_SentinelUndef)) {
10273 WidenedMask.push_back(SM_SentinelZero);
10279 // Finally check if the two mask values are adjacent and aligned with
10281 if (Mask[i] != SM_SentinelUndef && Mask[i] % 2 == 0 && Mask[i] + 1 == Mask[i + 1]) {
10282 WidenedMask.push_back(Mask[i] / 2);
10286 // Otherwise we can't safely widen the elements used in this shuffle.
10289 assert(WidenedMask.size() == Mask.size() / 2 &&
10290 "Incorrect size of mask after widening the elements!");
10295 /// \brief Generic routine to split vector shuffle into half-sized shuffles.
10297 /// This routine just extracts two subvectors, shuffles them independently, and
10298 /// then concatenates them back together. This should work effectively with all
10299 /// AVX vector shuffle types.
10300 static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1,
10301 SDValue V2, ArrayRef<int> Mask,
10302 SelectionDAG &DAG) {
10303 assert(VT.getSizeInBits() >= 256 &&
10304 "Only for 256-bit or wider vector shuffles!");
10305 assert(V1.getSimpleValueType() == VT && "Bad operand type!");
10306 assert(V2.getSimpleValueType() == VT && "Bad operand type!");
10308 ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
10309 ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
10311 int NumElements = VT.getVectorNumElements();
10312 int SplitNumElements = NumElements / 2;
10313 MVT ScalarVT = VT.getScalarType();
10314 MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2);
10316 // Rather than splitting build-vectors, just build two narrower build
10317 // vectors. This helps shuffling with splats and zeros.
10318 auto SplitVector = [&](SDValue V) {
10319 while (V.getOpcode() == ISD::BITCAST)
10320 V = V->getOperand(0);
10322 MVT OrigVT = V.getSimpleValueType();
10323 int OrigNumElements = OrigVT.getVectorNumElements();
10324 int OrigSplitNumElements = OrigNumElements / 2;
10325 MVT OrigScalarVT = OrigVT.getScalarType();
10326 MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2);
10330 auto *BV = dyn_cast<BuildVectorSDNode>(V);
10332 LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
10333 DAG.getIntPtrConstant(0));
10334 HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
10335 DAG.getIntPtrConstant(OrigSplitNumElements));
10338 SmallVector<SDValue, 16> LoOps, HiOps;
10339 for (int i = 0; i < OrigSplitNumElements; ++i) {
10340 LoOps.push_back(BV->getOperand(i));
10341 HiOps.push_back(BV->getOperand(i + OrigSplitNumElements));
10343 LoV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, LoOps);
10344 HiV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, HiOps);
10346 return std::make_pair(DAG.getNode(ISD::BITCAST, DL, SplitVT, LoV),
10347 DAG.getNode(ISD::BITCAST, DL, SplitVT, HiV));
10350 SDValue LoV1, HiV1, LoV2, HiV2;
10351 std::tie(LoV1, HiV1) = SplitVector(V1);
10352 std::tie(LoV2, HiV2) = SplitVector(V2);
10354 // Now create two 4-way blends of these half-width vectors.
10355 auto HalfBlend = [&](ArrayRef<int> HalfMask) {
10356 bool UseLoV1 = false, UseHiV1 = false, UseLoV2 = false, UseHiV2 = false;
10357 SmallVector<int, 32> V1BlendMask, V2BlendMask, BlendMask;
10358 for (int i = 0; i < SplitNumElements; ++i) {
10359 int M = HalfMask[i];
10360 if (M >= NumElements) {
10361 if (M >= NumElements + SplitNumElements)
10365 V2BlendMask.push_back(M - NumElements);
10366 V1BlendMask.push_back(-1);
10367 BlendMask.push_back(SplitNumElements + i);
10368 } else if (M >= 0) {
10369 if (M >= SplitNumElements)
10373 V2BlendMask.push_back(-1);
10374 V1BlendMask.push_back(M);
10375 BlendMask.push_back(i);
10377 V2BlendMask.push_back(-1);
10378 V1BlendMask.push_back(-1);
10379 BlendMask.push_back(-1);
10383 // Because the lowering happens after all combining takes place, we need to
10384 // manually combine these blend masks as much as possible so that we create
10385 // a minimal number of high-level vector shuffle nodes.
10387 // First try just blending the halves of V1 or V2.
10388 if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2)
10389 return DAG.getUNDEF(SplitVT);
10390 if (!UseLoV2 && !UseHiV2)
10391 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
10392 if (!UseLoV1 && !UseHiV1)
10393 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
10395 SDValue V1Blend, V2Blend;
10396 if (UseLoV1 && UseHiV1) {
10398 DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
10400 // We only use half of V1 so map the usage down into the final blend mask.
10401 V1Blend = UseLoV1 ? LoV1 : HiV1;
10402 for (int i = 0; i < SplitNumElements; ++i)
10403 if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements)
10404 BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements);
10406 if (UseLoV2 && UseHiV2) {
10408 DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
10410 // We only use half of V2 so map the usage down into the final blend mask.
10411 V2Blend = UseLoV2 ? LoV2 : HiV2;
10412 for (int i = 0; i < SplitNumElements; ++i)
10413 if (BlendMask[i] >= SplitNumElements)
10414 BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0);
10416 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
10418 SDValue Lo = HalfBlend(LoMask);
10419 SDValue Hi = HalfBlend(HiMask);
10420 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
10423 /// \brief Either split a vector in halves or decompose the shuffles and the
10426 /// This is provided as a good fallback for many lowerings of non-single-input
10427 /// shuffles with more than one 128-bit lane. In those cases, we want to select
10428 /// between splitting the shuffle into 128-bit components and stitching those
10429 /// back together vs. extracting the single-input shuffles and blending those
10431 static SDValue lowerVectorShuffleAsSplitOrBlend(SDLoc DL, MVT VT, SDValue V1,
10432 SDValue V2, ArrayRef<int> Mask,
10433 SelectionDAG &DAG) {
10434 assert(!isSingleInputShuffleMask(Mask) && "This routine must not be used to "
10435 "lower single-input shuffles as it "
10436 "could then recurse on itself.");
10437 int Size = Mask.size();
10439 // If this can be modeled as a broadcast of two elements followed by a blend,
10440 // prefer that lowering. This is especially important because broadcasts can
10441 // often fold with memory operands.
10442 auto DoBothBroadcast = [&] {
10443 int V1BroadcastIdx = -1, V2BroadcastIdx = -1;
10446 if (V2BroadcastIdx == -1)
10447 V2BroadcastIdx = M - Size;
10448 else if (M - Size != V2BroadcastIdx)
10450 } else if (M >= 0) {
10451 if (V1BroadcastIdx == -1)
10452 V1BroadcastIdx = M;
10453 else if (M != V1BroadcastIdx)
10458 if (DoBothBroadcast())
10459 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
10462 // If the inputs all stem from a single 128-bit lane of each input, then we
10463 // split them rather than blending because the split will decompose to
10464 // unusually few instructions.
10465 int LaneCount = VT.getSizeInBits() / 128;
10466 int LaneSize = Size / LaneCount;
10467 SmallBitVector LaneInputs[2];
10468 LaneInputs[0].resize(LaneCount, false);
10469 LaneInputs[1].resize(LaneCount, false);
10470 for (int i = 0; i < Size; ++i)
10472 LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
10473 if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
10474 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
10476 // Otherwise, just fall back to decomposed shuffles and a blend. This requires
10477 // that the decomposed single-input shuffles don't end up here.
10478 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG);
10481 /// \brief Lower a vector shuffle crossing multiple 128-bit lanes as
10482 /// a permutation and blend of those lanes.
10484 /// This essentially blends the out-of-lane inputs to each lane into the lane
10485 /// from a permuted copy of the vector. This lowering strategy results in four
10486 /// instructions in the worst case for a single-input cross lane shuffle which
10487 /// is lower than any other fully general cross-lane shuffle strategy I'm aware
10488 /// of. Special cases for each particular shuffle pattern should be handled
10489 /// prior to trying this lowering.
10490 static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT,
10491 SDValue V1, SDValue V2,
10492 ArrayRef<int> Mask,
10493 SelectionDAG &DAG) {
10494 // FIXME: This should probably be generalized for 512-bit vectors as well.
10495 assert(VT.getSizeInBits() == 256 && "Only for 256-bit vector shuffles!");
10496 int LaneSize = Mask.size() / 2;
10498 // If there are only inputs from one 128-bit lane, splitting will in fact be
10499 // less expensive. The flags track wether the given lane contains an element
10500 // that crosses to another lane.
10501 bool LaneCrossing[2] = {false, false};
10502 for (int i = 0, Size = Mask.size(); i < Size; ++i)
10503 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
10504 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
10505 if (!LaneCrossing[0] || !LaneCrossing[1])
10506 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
10508 if (isSingleInputShuffleMask(Mask)) {
10509 SmallVector<int, 32> FlippedBlendMask;
10510 for (int i = 0, Size = Mask.size(); i < Size; ++i)
10511 FlippedBlendMask.push_back(
10512 Mask[i] < 0 ? -1 : (((Mask[i] % Size) / LaneSize == i / LaneSize)
10514 : Mask[i] % LaneSize +
10515 (i / LaneSize) * LaneSize + Size));
10517 // Flip the vector, and blend the results which should now be in-lane. The
10518 // VPERM2X128 mask uses the low 2 bits for the low source and bits 4 and
10519 // 5 for the high source. The value 3 selects the high half of source 2 and
10520 // the value 2 selects the low half of source 2. We only use source 2 to
10521 // allow folding it into a memory operand.
10522 unsigned PERMMask = 3 | 2 << 4;
10523 SDValue Flipped = DAG.getNode(X86ISD::VPERM2X128, DL, VT, DAG.getUNDEF(VT),
10524 V1, DAG.getConstant(PERMMask, MVT::i8));
10525 return DAG.getVectorShuffle(VT, DL, V1, Flipped, FlippedBlendMask);
10528 // This now reduces to two single-input shuffles of V1 and V2 which at worst
10529 // will be handled by the above logic and a blend of the results, much like
10530 // other patterns in AVX.
10531 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG);
10534 /// \brief Handle lowering 2-lane 128-bit shuffles.
10535 static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1,
10536 SDValue V2, ArrayRef<int> Mask,
10537 const X86Subtarget *Subtarget,
10538 SelectionDAG &DAG) {
10539 // Blends are faster and handle all the non-lane-crossing cases.
10540 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask,
10544 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),
10545 VT.getVectorNumElements() / 2);
10546 // Check for patterns which can be matched with a single insert of a 128-bit
10548 if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 0, 1) ||
10549 isShuffleEquivalent(V1, V2, Mask, 0, 1, 4, 5)) {
10550 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
10551 DAG.getIntPtrConstant(0));
10552 SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
10553 Mask[2] < 4 ? V1 : V2, DAG.getIntPtrConstant(0));
10554 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV);
10556 if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 6, 7)) {
10557 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
10558 DAG.getIntPtrConstant(0));
10559 SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
10560 DAG.getIntPtrConstant(2));
10561 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV);
10564 // Otherwise form a 128-bit permutation.
10565 // FIXME: Detect zero-vector inputs and use the VPERM2X128 to zero that half.
10566 unsigned PermMask = Mask[0] / 2 | (Mask[2] / 2) << 4;
10567 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
10568 DAG.getConstant(PermMask, MVT::i8));
10571 /// \brief Lower a vector shuffle by first fixing the 128-bit lanes and then
10572 /// shuffling each lane.
10574 /// This will only succeed when the result of fixing the 128-bit lanes results
10575 /// in a single-input non-lane-crossing shuffle with a repeating shuffle mask in
10576 /// each 128-bit lanes. This handles many cases where we can quickly blend away
10577 /// the lane crosses early and then use simpler shuffles within each lane.
10579 /// FIXME: It might be worthwhile at some point to support this without
10580 /// requiring the 128-bit lane-relative shuffles to be repeating, but currently
10581 /// in x86 only floating point has interesting non-repeating shuffles, and even
10582 /// those are still *marginally* more expensive.
10583 static SDValue lowerVectorShuffleByMerging128BitLanes(
10584 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
10585 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
10586 assert(!isSingleInputShuffleMask(Mask) &&
10587 "This is only useful with multiple inputs.");
10589 int Size = Mask.size();
10590 int LaneSize = 128 / VT.getScalarSizeInBits();
10591 int NumLanes = Size / LaneSize;
10592 assert(NumLanes > 1 && "Only handles 256-bit and wider shuffles.");
10594 // See if we can build a hypothetical 128-bit lane-fixing shuffle mask. Also
10595 // check whether the in-128-bit lane shuffles share a repeating pattern.
10596 SmallVector<int, 4> Lanes;
10597 Lanes.resize(NumLanes, -1);
10598 SmallVector<int, 4> InLaneMask;
10599 InLaneMask.resize(LaneSize, -1);
10600 for (int i = 0; i < Size; ++i) {
10604 int j = i / LaneSize;
10606 if (Lanes[j] < 0) {
10607 // First entry we've seen for this lane.
10608 Lanes[j] = Mask[i] / LaneSize;
10609 } else if (Lanes[j] != Mask[i] / LaneSize) {
10610 // This doesn't match the lane selected previously!
10614 // Check that within each lane we have a consistent shuffle mask.
10615 int k = i % LaneSize;
10616 if (InLaneMask[k] < 0) {
10617 InLaneMask[k] = Mask[i] % LaneSize;
10618 } else if (InLaneMask[k] != Mask[i] % LaneSize) {
10619 // This doesn't fit a repeating in-lane mask.
10624 // First shuffle the lanes into place.
10625 MVT LaneVT = MVT::getVectorVT(VT.isFloatingPoint() ? MVT::f64 : MVT::i64,
10626 VT.getSizeInBits() / 64);
10627 SmallVector<int, 8> LaneMask;
10628 LaneMask.resize(NumLanes * 2, -1);
10629 for (int i = 0; i < NumLanes; ++i)
10630 if (Lanes[i] >= 0) {
10631 LaneMask[2 * i + 0] = 2*Lanes[i] + 0;
10632 LaneMask[2 * i + 1] = 2*Lanes[i] + 1;
10635 V1 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V1);
10636 V2 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V2);
10637 SDValue LaneShuffle = DAG.getVectorShuffle(LaneVT, DL, V1, V2, LaneMask);
10639 // Cast it back to the type we actually want.
10640 LaneShuffle = DAG.getNode(ISD::BITCAST, DL, VT, LaneShuffle);
10642 // Now do a simple shuffle that isn't lane crossing.
10643 SmallVector<int, 8> NewMask;
10644 NewMask.resize(Size, -1);
10645 for (int i = 0; i < Size; ++i)
10647 NewMask[i] = (i / LaneSize) * LaneSize + Mask[i] % LaneSize;
10648 assert(!is128BitLaneCrossingShuffleMask(VT, NewMask) &&
10649 "Must not introduce lane crosses at this point!");
10651 return DAG.getVectorShuffle(VT, DL, LaneShuffle, DAG.getUNDEF(VT), NewMask);
10654 /// \brief Test whether the specified input (0 or 1) is in-place blended by the
10657 /// This returns true if the elements from a particular input are already in the
10658 /// slot required by the given mask and require no permutation.
10659 static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
10660 assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.");
10661 int Size = Mask.size();
10662 for (int i = 0; i < Size; ++i)
10663 if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i)
10669 /// \brief Handle lowering of 4-lane 64-bit floating point shuffles.
10671 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
10672 /// isn't available.
10673 static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10674 const X86Subtarget *Subtarget,
10675 SelectionDAG &DAG) {
10677 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
10678 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
10679 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10680 ArrayRef<int> Mask = SVOp->getMask();
10681 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
10683 SmallVector<int, 4> WidenedMask;
10684 if (canWidenShuffleElements(Mask, WidenedMask))
10685 return lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, Subtarget,
10688 if (isSingleInputShuffleMask(Mask)) {
10689 // Check for being able to broadcast a single element.
10690 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f64, DL, V1,
10691 Mask, Subtarget, DAG))
10694 // Use low duplicate instructions for masks that match their pattern.
10695 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2))
10696 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
10698 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {
10699 // Non-half-crossing single input shuffles can be lowerid with an
10700 // interleaved permutation.
10701 unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
10702 ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
10703 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
10704 DAG.getConstant(VPERMILPMask, MVT::i8));
10707 // With AVX2 we have direct support for this permutation.
10708 if (Subtarget->hasAVX2())
10709 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
10710 getV4X86ShuffleImm8ForMask(Mask, DAG));
10712 // Otherwise, fall back.
10713 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask,
10717 // X86 has dedicated unpack instructions that can handle specific blend
10718 // operations: UNPCKH and UNPCKL.
10719 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6))
10720 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V1, V2);
10721 if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7))
10722 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V1, V2);
10724 // If we have a single input to the zero element, insert that into V1 if we
10725 // can do so cheaply.
10726 int NumV2Elements =
10727 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
10728 if (NumV2Elements == 1 && Mask[0] >= 4)
10729 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
10730 MVT::v4f64, DL, V1, V2, Mask, Subtarget, DAG))
10733 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
10737 // Check if the blend happens to exactly fit that of SHUFPD.
10738 if ((Mask[0] == -1 || Mask[0] < 2) &&
10739 (Mask[1] == -1 || (Mask[1] >= 4 && Mask[1] < 6)) &&
10740 (Mask[2] == -1 || (Mask[2] >= 2 && Mask[2] < 4)) &&
10741 (Mask[3] == -1 || Mask[3] >= 6)) {
10742 unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 5) << 1) |
10743 ((Mask[2] == 3) << 2) | ((Mask[3] == 7) << 3);
10744 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V1, V2,
10745 DAG.getConstant(SHUFPDMask, MVT::i8));
10747 if ((Mask[0] == -1 || (Mask[0] >= 4 && Mask[0] < 6)) &&
10748 (Mask[1] == -1 || Mask[1] < 2) &&
10749 (Mask[2] == -1 || Mask[2] >= 6) &&
10750 (Mask[3] == -1 || (Mask[3] >= 2 && Mask[3] < 4))) {
10751 unsigned SHUFPDMask = (Mask[0] == 5) | ((Mask[1] == 1) << 1) |
10752 ((Mask[2] == 7) << 2) | ((Mask[3] == 3) << 3);
10753 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V2, V1,
10754 DAG.getConstant(SHUFPDMask, MVT::i8));
10757 // Try to simplify this by merging 128-bit lanes to enable a lane-based
10758 // shuffle. However, if we have AVX2 and either inputs are already in place,
10759 // we will be able to shuffle even across lanes the other input in a single
10760 // instruction so skip this pattern.
10761 if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) ||
10762 isShuffleMaskInputInPlace(1, Mask))))
10763 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
10764 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
10767 // If we have AVX2 then we always want to lower with a blend because an v4 we
10768 // can fully permute the elements.
10769 if (Subtarget->hasAVX2())
10770 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2,
10773 // Otherwise fall back on generic lowering.
10774 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG);
10777 /// \brief Handle lowering of 4-lane 64-bit integer shuffles.
10779 /// This routine is only called when we have AVX2 and thus a reasonable
10780 /// instruction set for v4i64 shuffling..
10781 static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10782 const X86Subtarget *Subtarget,
10783 SelectionDAG &DAG) {
10785 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
10786 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
10787 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10788 ArrayRef<int> Mask = SVOp->getMask();
10789 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
10790 assert(Subtarget->hasAVX2() && "We can only lower v4i64 with AVX2!");
10792 SmallVector<int, 4> WidenedMask;
10793 if (canWidenShuffleElements(Mask, WidenedMask))
10794 return lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, Subtarget,
10797 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
10801 // Check for being able to broadcast a single element.
10802 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i64, DL, V1,
10803 Mask, Subtarget, DAG))
10806 // When the shuffle is mirrored between the 128-bit lanes of the unit, we can
10807 // use lower latency instructions that will operate on both 128-bit lanes.
10808 SmallVector<int, 2> RepeatedMask;
10809 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
10810 if (isSingleInputShuffleMask(Mask)) {
10811 int PSHUFDMask[] = {-1, -1, -1, -1};
10812 for (int i = 0; i < 2; ++i)
10813 if (RepeatedMask[i] >= 0) {
10814 PSHUFDMask[2 * i] = 2 * RepeatedMask[i];
10815 PSHUFDMask[2 * i + 1] = 2 * RepeatedMask[i] + 1;
10817 return DAG.getNode(
10818 ISD::BITCAST, DL, MVT::v4i64,
10819 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
10820 DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1),
10821 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
10825 // AVX2 provides a direct instruction for permuting a single input across
10827 if (isSingleInputShuffleMask(Mask))
10828 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
10829 getV4X86ShuffleImm8ForMask(Mask, DAG));
10831 // Try to use byte shift instructions.
10832 if (SDValue Shift = lowerVectorShuffleAsByteShift(
10833 DL, MVT::v4i64, V1, V2, Mask, DAG))
10836 // Use dedicated unpack instructions for masks that match their pattern.
10837 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6))
10838 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V1, V2);
10839 if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7))
10840 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V1, V2);
10842 // Try to simplify this by merging 128-bit lanes to enable a lane-based
10843 // shuffle. However, if we have AVX2 and either inputs are already in place,
10844 // we will be able to shuffle even across lanes the other input in a single
10845 // instruction so skip this pattern.
10846 if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) ||
10847 isShuffleMaskInputInPlace(1, Mask))))
10848 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
10849 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
10852 // Otherwise fall back on generic blend lowering.
10853 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2,
10857 /// \brief Handle lowering of 8-lane 32-bit floating point shuffles.
10859 /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
10860 /// isn't available.
10861 static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10862 const X86Subtarget *Subtarget,
10863 SelectionDAG &DAG) {
10865 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
10866 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
10867 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10868 ArrayRef<int> Mask = SVOp->getMask();
10869 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
10871 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
10875 // Check for being able to broadcast a single element.
10876 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8f32, DL, V1,
10877 Mask, Subtarget, DAG))
10880 // If the shuffle mask is repeated in each 128-bit lane, we have many more
10881 // options to efficiently lower the shuffle.
10882 SmallVector<int, 4> RepeatedMask;
10883 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {
10884 assert(RepeatedMask.size() == 4 &&
10885 "Repeated masks must be half the mask width!");
10887 // Use even/odd duplicate instructions for masks that match their pattern.
10888 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2, 4, 4, 6, 6))
10889 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
10890 if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3, 5, 5, 7, 7))
10891 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
10893 if (isSingleInputShuffleMask(Mask))
10894 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
10895 getV4X86ShuffleImm8ForMask(RepeatedMask, DAG));
10897 // Use dedicated unpack instructions for masks that match their pattern.
10898 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13))
10899 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V1, V2);
10900 if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15))
10901 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V1, V2);
10903 // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
10904 // have already handled any direct blends. We also need to squash the
10905 // repeated mask into a simulated v4f32 mask.
10906 for (int i = 0; i < 4; ++i)
10907 if (RepeatedMask[i] >= 8)
10908 RepeatedMask[i] -= 4;
10909 return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
10912 // If we have a single input shuffle with different shuffle patterns in the
10913 // two 128-bit lanes use the variable mask to VPERMILPS.
10914 if (isSingleInputShuffleMask(Mask)) {
10915 SDValue VPermMask[8];
10916 for (int i = 0; i < 8; ++i)
10917 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
10918 : DAG.getConstant(Mask[i], MVT::i32);
10919 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask))
10920 return DAG.getNode(
10921 X86ISD::VPERMILPV, DL, MVT::v8f32, V1,
10922 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask));
10924 if (Subtarget->hasAVX2())
10925 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32,
10926 DAG.getNode(ISD::BITCAST, DL, MVT::v8f32,
10927 DAG.getNode(ISD::BUILD_VECTOR, DL,
10928 MVT::v8i32, VPermMask)),
10931 // Otherwise, fall back.
10932 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask,
10936 // Try to simplify this by merging 128-bit lanes to enable a lane-based
10938 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
10939 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
10942 // If we have AVX2 then we always want to lower with a blend because at v8 we
10943 // can fully permute the elements.
10944 if (Subtarget->hasAVX2())
10945 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2,
10948 // Otherwise fall back on generic lowering.
10949 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG);
10952 /// \brief Handle lowering of 8-lane 32-bit integer shuffles.
10954 /// This routine is only called when we have AVX2 and thus a reasonable
10955 /// instruction set for v8i32 shuffling..
10956 static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10957 const X86Subtarget *Subtarget,
10958 SelectionDAG &DAG) {
10960 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
10961 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
10962 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10963 ArrayRef<int> Mask = SVOp->getMask();
10964 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
10965 assert(Subtarget->hasAVX2() && "We can only lower v8i32 with AVX2!");
10967 // Whenever we can lower this as a zext, that instruction is strictly faster
10968 // than any alternative. It also allows us to fold memory operands into the
10969 // shuffle in many cases.
10970 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2,
10971 Mask, Subtarget, DAG))
10974 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
10978 // Check for being able to broadcast a single element.
10979 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i32, DL, V1,
10980 Mask, Subtarget, DAG))
10983 // If the shuffle mask is repeated in each 128-bit lane we can use more
10984 // efficient instructions that mirror the shuffles across the two 128-bit
10986 SmallVector<int, 4> RepeatedMask;
10987 if (is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask)) {
10988 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
10989 if (isSingleInputShuffleMask(Mask))
10990 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
10991 getV4X86ShuffleImm8ForMask(RepeatedMask, DAG));
10993 // Use dedicated unpack instructions for masks that match their pattern.
10994 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13))
10995 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V1, V2);
10996 if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15))
10997 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V1, V2);
11000 // Try to use bit shift instructions.
11001 if (SDValue Shift = lowerVectorShuffleAsBitShift(
11002 DL, MVT::v8i32, V1, V2, Mask, DAG))
11005 // Try to use byte shift instructions.
11006 if (SDValue Shift = lowerVectorShuffleAsByteShift(
11007 DL, MVT::v8i32, V1, V2, Mask, DAG))
11010 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
11011 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11014 // If the shuffle patterns aren't repeated but it is a single input, directly
11015 // generate a cross-lane VPERMD instruction.
11016 if (isSingleInputShuffleMask(Mask)) {
11017 SDValue VPermMask[8];
11018 for (int i = 0; i < 8; ++i)
11019 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
11020 : DAG.getConstant(Mask[i], MVT::i32);
11021 return DAG.getNode(
11022 X86ISD::VPERMV, DL, MVT::v8i32,
11023 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1);
11026 // Try to simplify this by merging 128-bit lanes to enable a lane-based
11028 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
11029 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11032 // Otherwise fall back on generic blend lowering.
11033 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2,
11037 /// \brief Handle lowering of 16-lane 16-bit integer shuffles.
11039 /// This routine is only called when we have AVX2 and thus a reasonable
11040 /// instruction set for v16i16 shuffling..
11041 static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11042 const X86Subtarget *Subtarget,
11043 SelectionDAG &DAG) {
11045 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
11046 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
11047 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11048 ArrayRef<int> Mask = SVOp->getMask();
11049 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
11050 assert(Subtarget->hasAVX2() && "We can only lower v16i16 with AVX2!");
11052 // Whenever we can lower this as a zext, that instruction is strictly faster
11053 // than any alternative. It also allows us to fold memory operands into the
11054 // shuffle in many cases.
11055 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2,
11056 Mask, Subtarget, DAG))
11059 // Check for being able to broadcast a single element.
11060 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i16, DL, V1,
11061 Mask, Subtarget, DAG))
11064 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
11068 // Use dedicated unpack instructions for masks that match their pattern.
11069 if (isShuffleEquivalent(V1, V2, Mask,
11070 // First 128-bit lane:
11071 0, 16, 1, 17, 2, 18, 3, 19,
11072 // Second 128-bit lane:
11073 8, 24, 9, 25, 10, 26, 11, 27))
11074 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i16, V1, V2);
11075 if (isShuffleEquivalent(V1, V2, Mask,
11076 // First 128-bit lane:
11077 4, 20, 5, 21, 6, 22, 7, 23,
11078 // Second 128-bit lane:
11079 12, 28, 13, 29, 14, 30, 15, 31))
11080 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i16, V1, V2);
11082 // Try to use bit shift instructions.
11083 if (SDValue Shift = lowerVectorShuffleAsBitShift(
11084 DL, MVT::v16i16, V1, V2, Mask, DAG))
11087 // Try to use byte shift instructions.
11088 if (SDValue Shift = lowerVectorShuffleAsByteShift(
11089 DL, MVT::v16i16, V1, V2, Mask, DAG))
11092 // Try to use byte rotation instructions.
11093 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
11094 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11097 if (isSingleInputShuffleMask(Mask)) {
11098 // There are no generalized cross-lane shuffle operations available on i16
11100 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask))
11101 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2,
11104 SDValue PSHUFBMask[32];
11105 for (int i = 0; i < 16; ++i) {
11106 if (Mask[i] == -1) {
11107 PSHUFBMask[2 * i] = PSHUFBMask[2 * i + 1] = DAG.getUNDEF(MVT::i8);
11111 int M = i < 8 ? Mask[i] : Mask[i] - 8;
11112 assert(M >= 0 && M < 8 && "Invalid single-input mask!");
11113 PSHUFBMask[2 * i] = DAG.getConstant(2 * M, MVT::i8);
11114 PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, MVT::i8);
11116 return DAG.getNode(
11117 ISD::BITCAST, DL, MVT::v16i16,
11119 X86ISD::PSHUFB, DL, MVT::v32i8,
11120 DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1),
11121 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)));
11124 // Try to simplify this by merging 128-bit lanes to enable a lane-based
11126 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
11127 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11130 // Otherwise fall back on generic lowering.
11131 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG);
11134 /// \brief Handle lowering of 32-lane 8-bit integer shuffles.
11136 /// This routine is only called when we have AVX2 and thus a reasonable
11137 /// instruction set for v32i8 shuffling..
11138 static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11139 const X86Subtarget *Subtarget,
11140 SelectionDAG &DAG) {
11142 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
11143 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
11144 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11145 ArrayRef<int> Mask = SVOp->getMask();
11146 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
11147 assert(Subtarget->hasAVX2() && "We can only lower v32i8 with AVX2!");
11149 // Whenever we can lower this as a zext, that instruction is strictly faster
11150 // than any alternative. It also allows us to fold memory operands into the
11151 // shuffle in many cases.
11152 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2,
11153 Mask, Subtarget, DAG))
11156 // Check for being able to broadcast a single element.
11157 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v32i8, DL, V1,
11158 Mask, Subtarget, DAG))
11161 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
11165 // Use dedicated unpack instructions for masks that match their pattern.
11166 // Note that these are repeated 128-bit lane unpacks, not unpacks across all
11168 if (isShuffleEquivalent(
11170 // First 128-bit lane:
11171 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
11172 // Second 128-bit lane:
11173 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55))
11174 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v32i8, V1, V2);
11175 if (isShuffleEquivalent(
11177 // First 128-bit lane:
11178 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
11179 // Second 128-bit lane:
11180 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63))
11181 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v32i8, V1, V2);
11183 // Try to use bit shift instructions.
11184 if (SDValue Shift = lowerVectorShuffleAsBitShift(
11185 DL, MVT::v32i8, V1, V2, Mask, DAG))
11188 // Try to use byte shift instructions.
11189 if (SDValue Shift = lowerVectorShuffleAsByteShift(
11190 DL, MVT::v32i8, V1, V2, Mask, DAG))
11193 // Try to use byte rotation instructions.
11194 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
11195 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11198 if (isSingleInputShuffleMask(Mask)) {
11199 // There are no generalized cross-lane shuffle operations available on i8
11201 if (is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask))
11202 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2,
11205 SDValue PSHUFBMask[32];
11206 for (int i = 0; i < 32; ++i)
11209 ? DAG.getUNDEF(MVT::i8)
11210 : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, MVT::i8);
11212 return DAG.getNode(
11213 X86ISD::PSHUFB, DL, MVT::v32i8, V1,
11214 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask));
11217 // Try to simplify this by merging 128-bit lanes to enable a lane-based
11219 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
11220 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11223 // Otherwise fall back on generic lowering.
11224 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG);
11227 /// \brief High-level routine to lower various 256-bit x86 vector shuffles.
11229 /// This routine either breaks down the specific type of a 256-bit x86 vector
11230 /// shuffle or splits it into two 128-bit shuffles and fuses the results back
11231 /// together based on the available instructions.
11232 static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11233 MVT VT, const X86Subtarget *Subtarget,
11234 SelectionDAG &DAG) {
11236 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11237 ArrayRef<int> Mask = SVOp->getMask();
11239 // There is a really nice hard cut-over between AVX1 and AVX2 that means we can
11240 // check for those subtargets here and avoid much of the subtarget querying in
11241 // the per-vector-type lowering routines. With AVX1 we have essentially *zero*
11242 // ability to manipulate a 256-bit vector with integer types. Since we'll use
11243 // floating point types there eventually, just immediately cast everything to
11244 // a float and operate entirely in that domain.
11245 if (VT.isInteger() && !Subtarget->hasAVX2()) {
11246 int ElementBits = VT.getScalarSizeInBits();
11247 if (ElementBits < 32)
11248 // No floating point type available, decompose into 128-bit vectors.
11249 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
11251 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
11252 VT.getVectorNumElements());
11253 V1 = DAG.getNode(ISD::BITCAST, DL, FpVT, V1);
11254 V2 = DAG.getNode(ISD::BITCAST, DL, FpVT, V2);
11255 return DAG.getNode(ISD::BITCAST, DL, VT,
11256 DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
11259 switch (VT.SimpleTy) {
11261 return lowerV4F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11263 return lowerV4I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11265 return lowerV8F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11267 return lowerV8I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11269 return lowerV16I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
11271 return lowerV32I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
11274 llvm_unreachable("Not a valid 256-bit x86 vector type!");
11278 /// \brief Handle lowering of 8-lane 64-bit floating point shuffles.
11279 static SDValue lowerV8F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11280 const X86Subtarget *Subtarget,
11281 SelectionDAG &DAG) {
11283 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
11284 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
11285 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11286 ArrayRef<int> Mask = SVOp->getMask();
11287 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
11289 // X86 has dedicated unpack instructions that can handle specific blend
11290 // operations: UNPCKH and UNPCKL.
11291 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14))
11292 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f64, V1, V2);
11293 if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15))
11294 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f64, V1, V2);
11296 // FIXME: Implement direct support for this type!
11297 return splitAndLowerVectorShuffle(DL, MVT::v8f64, V1, V2, Mask, DAG);
11300 /// \brief Handle lowering of 16-lane 32-bit floating point shuffles.
11301 static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11302 const X86Subtarget *Subtarget,
11303 SelectionDAG &DAG) {
11305 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
11306 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
11307 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11308 ArrayRef<int> Mask = SVOp->getMask();
11309 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
11311 // Use dedicated unpack instructions for masks that match their pattern.
11312 if (isShuffleEquivalent(V1, V2, Mask,
11313 0, 16, 1, 17, 4, 20, 5, 21,
11314 8, 24, 9, 25, 12, 28, 13, 29))
11315 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16f32, V1, V2);
11316 if (isShuffleEquivalent(V1, V2, Mask,
11317 2, 18, 3, 19, 6, 22, 7, 23,
11318 10, 26, 11, 27, 14, 30, 15, 31))
11319 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16f32, V1, V2);
11321 // FIXME: Implement direct support for this type!
11322 return splitAndLowerVectorShuffle(DL, MVT::v16f32, V1, V2, Mask, DAG);
11325 /// \brief Handle lowering of 8-lane 64-bit integer shuffles.
11326 static SDValue lowerV8I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11327 const X86Subtarget *Subtarget,
11328 SelectionDAG &DAG) {
11330 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
11331 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
11332 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11333 ArrayRef<int> Mask = SVOp->getMask();
11334 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
11336 // X86 has dedicated unpack instructions that can handle specific blend
11337 // operations: UNPCKH and UNPCKL.
11338 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14))
11339 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i64, V1, V2);
11340 if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15))
11341 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i64, V1, V2);
11343 // FIXME: Implement direct support for this type!
11344 return splitAndLowerVectorShuffle(DL, MVT::v8i64, V1, V2, Mask, DAG);
11347 /// \brief Handle lowering of 16-lane 32-bit integer shuffles.
11348 static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11349 const X86Subtarget *Subtarget,
11350 SelectionDAG &DAG) {
11352 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
11353 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
11354 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11355 ArrayRef<int> Mask = SVOp->getMask();
11356 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
11358 // Use dedicated unpack instructions for masks that match their pattern.
11359 if (isShuffleEquivalent(V1, V2, Mask,
11360 0, 16, 1, 17, 4, 20, 5, 21,
11361 8, 24, 9, 25, 12, 28, 13, 29))
11362 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i32, V1, V2);
11363 if (isShuffleEquivalent(V1, V2, Mask,
11364 2, 18, 3, 19, 6, 22, 7, 23,
11365 10, 26, 11, 27, 14, 30, 15, 31))
11366 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i32, V1, V2);
11368 // FIXME: Implement direct support for this type!
11369 return splitAndLowerVectorShuffle(DL, MVT::v16i32, V1, V2, Mask, DAG);
11372 /// \brief Handle lowering of 32-lane 16-bit integer shuffles.
11373 static SDValue lowerV32I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11374 const X86Subtarget *Subtarget,
11375 SelectionDAG &DAG) {
11377 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
11378 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
11379 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11380 ArrayRef<int> Mask = SVOp->getMask();
11381 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
11382 assert(Subtarget->hasBWI() && "We can only lower v32i16 with AVX-512-BWI!");
11384 // FIXME: Implement direct support for this type!
11385 return splitAndLowerVectorShuffle(DL, MVT::v32i16, V1, V2, Mask, DAG);
11388 /// \brief Handle lowering of 64-lane 8-bit integer shuffles.
11389 static SDValue lowerV64I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11390 const X86Subtarget *Subtarget,
11391 SelectionDAG &DAG) {
11393 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
11394 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
11395 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11396 ArrayRef<int> Mask = SVOp->getMask();
11397 assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!");
11398 assert(Subtarget->hasBWI() && "We can only lower v64i8 with AVX-512-BWI!");
11400 // FIXME: Implement direct support for this type!
11401 return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
11404 /// \brief High-level routine to lower various 512-bit x86 vector shuffles.
11406 /// This routine either breaks down the specific type of a 512-bit x86 vector
11407 /// shuffle or splits it into two 256-bit shuffles and fuses the results back
11408 /// together based on the available instructions.
11409 static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11410 MVT VT, const X86Subtarget *Subtarget,
11411 SelectionDAG &DAG) {
11413 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11414 ArrayRef<int> Mask = SVOp->getMask();
11415 assert(Subtarget->hasAVX512() &&
11416 "Cannot lower 512-bit vectors w/ basic ISA!");
11418 // Check for being able to broadcast a single element.
11419 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(VT.SimpleTy, DL, V1,
11420 Mask, Subtarget, DAG))
11423 // Dispatch to each element type for lowering. If we don't have supprot for
11424 // specific element type shuffles at 512 bits, immediately split them and
11425 // lower them. Each lowering routine of a given type is allowed to assume that
11426 // the requisite ISA extensions for that element type are available.
11427 switch (VT.SimpleTy) {
11429 return lowerV8F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11431 return lowerV16F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11433 return lowerV8I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11435 return lowerV16I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11437 if (Subtarget->hasBWI())
11438 return lowerV32I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
11441 if (Subtarget->hasBWI())
11442 return lowerV64I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
11446 llvm_unreachable("Not a valid 512-bit x86 vector type!");
11449 // Otherwise fall back on splitting.
11450 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
11453 /// \brief Top-level lowering for x86 vector shuffles.
11455 /// This handles decomposition, canonicalization, and lowering of all x86
11456 /// vector shuffles. Most of the specific lowering strategies are encapsulated
11457 /// above in helper routines. The canonicalization attempts to widen shuffles
11458 /// to involve fewer lanes of wider elements, consolidate symmetric patterns
11459 /// s.t. only one of the two inputs needs to be tested, etc.
11460 static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget *Subtarget,
11461 SelectionDAG &DAG) {
11462 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11463 ArrayRef<int> Mask = SVOp->getMask();
11464 SDValue V1 = Op.getOperand(0);
11465 SDValue V2 = Op.getOperand(1);
11466 MVT VT = Op.getSimpleValueType();
11467 int NumElements = VT.getVectorNumElements();
11470 assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
11472 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
11473 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
11474 if (V1IsUndef && V2IsUndef)
11475 return DAG.getUNDEF(VT);
11477 // When we create a shuffle node we put the UNDEF node to second operand,
11478 // but in some cases the first operand may be transformed to UNDEF.
11479 // In this case we should just commute the node.
11481 return DAG.getCommutedVectorShuffle(*SVOp);
11483 // Check for non-undef masks pointing at an undef vector and make the masks
11484 // undef as well. This makes it easier to match the shuffle based solely on
11488 if (M >= NumElements) {
11489 SmallVector<int, 8> NewMask(Mask.begin(), Mask.end());
11490 for (int &M : NewMask)
11491 if (M >= NumElements)
11493 return DAG.getVectorShuffle(VT, dl, V1, V2, NewMask);
11496 // We actually see shuffles that are entirely re-arrangements of a set of
11497 // zero inputs. This mostly happens while decomposing complex shuffles into
11498 // simple ones. Directly lower these as a buildvector of zeros.
11499 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
11500 if (Zeroable.all())
11501 return getZeroVector(VT, Subtarget, DAG, dl);
11503 // Try to collapse shuffles into using a vector type with fewer elements but
11504 // wider element types. We cap this to not form integers or floating point
11505 // elements wider than 64 bits, but it might be interesting to form i128
11506 // integers to handle flipping the low and high halves of AVX 256-bit vectors.
11507 SmallVector<int, 16> WidenedMask;
11508 if (VT.getScalarSizeInBits() < 64 &&
11509 canWidenShuffleElements(Mask, WidenedMask)) {
11510 MVT NewEltVT = VT.isFloatingPoint()
11511 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
11512 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
11513 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
11514 // Make sure that the new vector type is legal. For example, v2f64 isn't
11516 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
11517 V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1);
11518 V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2);
11519 return DAG.getNode(ISD::BITCAST, dl, VT,
11520 DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask));
11524 int NumV1Elements = 0, NumUndefElements = 0, NumV2Elements = 0;
11525 for (int M : SVOp->getMask())
11527 ++NumUndefElements;
11528 else if (M < NumElements)
11533 // Commute the shuffle as needed such that more elements come from V1 than
11534 // V2. This allows us to match the shuffle pattern strictly on how many
11535 // elements come from V1 without handling the symmetric cases.
11536 if (NumV2Elements > NumV1Elements)
11537 return DAG.getCommutedVectorShuffle(*SVOp);
11539 // When the number of V1 and V2 elements are the same, try to minimize the
11540 // number of uses of V2 in the low half of the vector. When that is tied,
11541 // ensure that the sum of indices for V1 is equal to or lower than the sum
11542 // indices for V2. When those are equal, try to ensure that the number of odd
11543 // indices for V1 is lower than the number of odd indices for V2.
11544 if (NumV1Elements == NumV2Elements) {
11545 int LowV1Elements = 0, LowV2Elements = 0;
11546 for (int M : SVOp->getMask().slice(0, NumElements / 2))
11547 if (M >= NumElements)
11551 if (LowV2Elements > LowV1Elements) {
11552 return DAG.getCommutedVectorShuffle(*SVOp);
11553 } else if (LowV2Elements == LowV1Elements) {
11554 int SumV1Indices = 0, SumV2Indices = 0;
11555 for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i)
11556 if (SVOp->getMask()[i] >= NumElements)
11558 else if (SVOp->getMask()[i] >= 0)
11560 if (SumV2Indices < SumV1Indices) {
11561 return DAG.getCommutedVectorShuffle(*SVOp);
11562 } else if (SumV2Indices == SumV1Indices) {
11563 int NumV1OddIndices = 0, NumV2OddIndices = 0;
11564 for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i)
11565 if (SVOp->getMask()[i] >= NumElements)
11566 NumV2OddIndices += i % 2;
11567 else if (SVOp->getMask()[i] >= 0)
11568 NumV1OddIndices += i % 2;
11569 if (NumV2OddIndices < NumV1OddIndices)
11570 return DAG.getCommutedVectorShuffle(*SVOp);
11575 // For each vector width, delegate to a specialized lowering routine.
11576 if (VT.getSizeInBits() == 128)
11577 return lower128BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
11579 if (VT.getSizeInBits() == 256)
11580 return lower256BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
11582 // Force AVX-512 vectors to be scalarized for now.
11583 // FIXME: Implement AVX-512 support!
11584 if (VT.getSizeInBits() == 512)
11585 return lower512BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
11587 llvm_unreachable("Unimplemented!");
11591 //===----------------------------------------------------------------------===//
11592 // Legacy vector shuffle lowering
11594 // This code is the legacy code handling vector shuffles until the above
11595 // replaces its functionality and performance.
11596 //===----------------------------------------------------------------------===//
11598 static bool isBlendMask(ArrayRef<int> MaskVals, MVT VT, bool hasSSE41,
11599 bool hasInt256, unsigned *MaskOut = nullptr) {
11600 MVT EltVT = VT.getVectorElementType();
11602 // There is no blend with immediate in AVX-512.
11603 if (VT.is512BitVector())
11606 if (!hasSSE41 || EltVT == MVT::i8)
11608 if (!hasInt256 && VT == MVT::v16i16)
11611 unsigned MaskValue = 0;
11612 unsigned NumElems = VT.getVectorNumElements();
11613 // There are 2 lanes if (NumElems > 8), and 1 lane otherwise.
11614 unsigned NumLanes = (NumElems - 1) / 8 + 1;
11615 unsigned NumElemsInLane = NumElems / NumLanes;
11617 // Blend for v16i16 should be symmetric for both lanes.
11618 for (unsigned i = 0; i < NumElemsInLane; ++i) {
11620 int SndLaneEltIdx = (NumLanes == 2) ? MaskVals[i + NumElemsInLane] : -1;
11621 int EltIdx = MaskVals[i];
11623 if ((EltIdx < 0 || EltIdx == (int)i) &&
11624 (SndLaneEltIdx < 0 || SndLaneEltIdx == (int)(i + NumElemsInLane)))
11627 if (((unsigned)EltIdx == (i + NumElems)) &&
11628 (SndLaneEltIdx < 0 ||
11629 (unsigned)SndLaneEltIdx == i + NumElems + NumElemsInLane))
11630 MaskValue |= (1 << i);
11636 *MaskOut = MaskValue;
11640 // Try to lower a shuffle node into a simple blend instruction.
11641 // This function assumes isBlendMask returns true for this
11642 // SuffleVectorSDNode
11643 static SDValue LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp,
11644 unsigned MaskValue,
11645 const X86Subtarget *Subtarget,
11646 SelectionDAG &DAG) {
11647 MVT VT = SVOp->getSimpleValueType(0);
11648 MVT EltVT = VT.getVectorElementType();
11649 assert(isBlendMask(SVOp->getMask(), VT, Subtarget->hasSSE41(),
11650 Subtarget->hasInt256() && "Trying to lower a "
11651 "VECTOR_SHUFFLE to a Blend but "
11652 "with the wrong mask"));
11653 SDValue V1 = SVOp->getOperand(0);
11654 SDValue V2 = SVOp->getOperand(1);
11656 unsigned NumElems = VT.getVectorNumElements();
11658 // Convert i32 vectors to floating point if it is not AVX2.
11659 // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors.
11661 if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) {
11662 BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()),
11664 V1 = DAG.getNode(ISD::BITCAST, dl, VT, V1);
11665 V2 = DAG.getNode(ISD::BITCAST, dl, VT, V2);
11668 SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, V1, V2,
11669 DAG.getConstant(MaskValue, MVT::i32));
11670 return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
11673 /// In vector type \p VT, return true if the element at index \p InputIdx
11674 /// falls on a different 128-bit lane than \p OutputIdx.
11675 static bool ShuffleCrosses128bitLane(MVT VT, unsigned InputIdx,
11676 unsigned OutputIdx) {
11677 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
11678 return InputIdx * EltSize / 128 != OutputIdx * EltSize / 128;
11681 /// Generate a PSHUFB if possible. Selects elements from \p V1 according to
11682 /// \p MaskVals. MaskVals[OutputIdx] = InputIdx specifies that we want to
11683 /// shuffle the element at InputIdx in V1 to OutputIdx in the result. If \p
11684 /// MaskVals refers to elements outside of \p V1 or is undef (-1), insert a
11686 static SDValue getPSHUFB(ArrayRef<int> MaskVals, SDValue V1, SDLoc &dl,
11687 SelectionDAG &DAG) {
11688 MVT VT = V1.getSimpleValueType();
11689 assert(VT.is128BitVector() || VT.is256BitVector());
11691 MVT EltVT = VT.getVectorElementType();
11692 unsigned EltSizeInBytes = EltVT.getSizeInBits() / 8;
11693 unsigned NumElts = VT.getVectorNumElements();
11695 SmallVector<SDValue, 32> PshufbMask;
11696 for (unsigned OutputIdx = 0; OutputIdx < NumElts; ++OutputIdx) {
11697 int InputIdx = MaskVals[OutputIdx];
11698 unsigned InputByteIdx;
11700 if (InputIdx < 0 || NumElts <= (unsigned)InputIdx)
11701 InputByteIdx = 0x80;
11703 // Cross lane is not allowed.
11704 if (ShuffleCrosses128bitLane(VT, InputIdx, OutputIdx))
11706 InputByteIdx = InputIdx * EltSizeInBytes;
11707 // Index is an byte offset within the 128-bit lane.
11708 InputByteIdx &= 0xf;
11711 for (unsigned j = 0; j < EltSizeInBytes; ++j) {
11712 PshufbMask.push_back(DAG.getConstant(InputByteIdx, MVT::i8));
11713 if (InputByteIdx != 0x80)
11718 MVT ShufVT = MVT::getVectorVT(MVT::i8, PshufbMask.size());
11720 V1 = DAG.getNode(ISD::BITCAST, dl, ShufVT, V1);
11721 return DAG.getNode(X86ISD::PSHUFB, dl, ShufVT, V1,
11722 DAG.getNode(ISD::BUILD_VECTOR, dl, ShufVT, PshufbMask));
11725 // v8i16 shuffles - Prefer shuffles in the following order:
11726 // 1. [all] pshuflw, pshufhw, optional move
11727 // 2. [ssse3] 1 x pshufb
11728 // 3. [ssse3] 2 x pshufb + 1 x por
11729 // 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw)
11731 LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget,
11732 SelectionDAG &DAG) {
11733 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11734 SDValue V1 = SVOp->getOperand(0);
11735 SDValue V2 = SVOp->getOperand(1);
11737 SmallVector<int, 8> MaskVals;
11739 // Determine if more than 1 of the words in each of the low and high quadwords
11740 // of the result come from the same quadword of one of the two inputs. Undef
11741 // mask values count as coming from any quadword, for better codegen.
11743 // Lo/HiQuad[i] = j indicates how many words from the ith quad of the input
11744 // feeds this quad. For i, 0 and 1 refer to V1, 2 and 3 refer to V2.
11745 unsigned LoQuad[] = { 0, 0, 0, 0 };
11746 unsigned HiQuad[] = { 0, 0, 0, 0 };
11747 // Indices of quads used.
11748 std::bitset<4> InputQuads;
11749 for (unsigned i = 0; i < 8; ++i) {
11750 unsigned *Quad = i < 4 ? LoQuad : HiQuad;
11751 int EltIdx = SVOp->getMaskElt(i);
11752 MaskVals.push_back(EltIdx);
11760 ++Quad[EltIdx / 4];
11761 InputQuads.set(EltIdx / 4);
11764 int BestLoQuad = -1;
11765 unsigned MaxQuad = 1;
11766 for (unsigned i = 0; i < 4; ++i) {
11767 if (LoQuad[i] > MaxQuad) {
11769 MaxQuad = LoQuad[i];
11773 int BestHiQuad = -1;
11775 for (unsigned i = 0; i < 4; ++i) {
11776 if (HiQuad[i] > MaxQuad) {
11778 MaxQuad = HiQuad[i];
11782 // For SSSE3, If all 8 words of the result come from only 1 quadword of each
11783 // of the two input vectors, shuffle them into one input vector so only a
11784 // single pshufb instruction is necessary. If there are more than 2 input
11785 // quads, disable the next transformation since it does not help SSSE3.
11786 bool V1Used = InputQuads[0] || InputQuads[1];
11787 bool V2Used = InputQuads[2] || InputQuads[3];
11788 if (Subtarget->hasSSSE3()) {
11789 if (InputQuads.count() == 2 && V1Used && V2Used) {
11790 BestLoQuad = InputQuads[0] ? 0 : 1;
11791 BestHiQuad = InputQuads[2] ? 2 : 3;
11793 if (InputQuads.count() > 2) {
11799 // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update
11800 // the shuffle mask. If a quad is scored as -1, that means that it contains
11801 // words from all 4 input quadwords.
11803 if (BestLoQuad >= 0 || BestHiQuad >= 0) {
11805 BestLoQuad < 0 ? 0 : BestLoQuad,
11806 BestHiQuad < 0 ? 1 : BestHiQuad
11808 NewV = DAG.getVectorShuffle(MVT::v2i64, dl,
11809 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1),
11810 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]);
11811 NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV);
11813 // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the
11814 // source words for the shuffle, to aid later transformations.
11815 bool AllWordsInNewV = true;
11816 bool InOrder[2] = { true, true };
11817 for (unsigned i = 0; i != 8; ++i) {
11818 int idx = MaskVals[i];
11820 InOrder[i/4] = false;
11821 if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad)
11823 AllWordsInNewV = false;
11827 bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV;
11828 if (AllWordsInNewV) {
11829 for (int i = 0; i != 8; ++i) {
11830 int idx = MaskVals[i];
11833 idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4;
11834 if ((idx != i) && idx < 4)
11836 if ((idx != i) && idx > 3)
11845 // If we've eliminated the use of V2, and the new mask is a pshuflw or
11846 // pshufhw, that's as cheap as it gets. Return the new shuffle.
11847 if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) {
11848 unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW;
11849 unsigned TargetMask = 0;
11850 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV,
11851 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]);
11852 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
11853 TargetMask = pshufhw ? getShufflePSHUFHWImmediate(SVOp):
11854 getShufflePSHUFLWImmediate(SVOp);
11855 V1 = NewV.getOperand(0);
11856 return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG);
11860 // Promote splats to a larger type which usually leads to more efficient code.
11861 // FIXME: Is this true if pshufb is available?
11862 if (SVOp->isSplat())
11863 return PromoteSplat(SVOp, DAG);
11865 // If we have SSSE3, and all words of the result are from 1 input vector,
11866 // case 2 is generated, otherwise case 3 is generated. If no SSSE3
11867 // is present, fall back to case 4.
11868 if (Subtarget->hasSSSE3()) {
11869 SmallVector<SDValue,16> pshufbMask;
11871 // If we have elements from both input vectors, set the high bit of the
11872 // shuffle mask element to zero out elements that come from V2 in the V1
11873 // mask, and elements that come from V1 in the V2 mask, so that the two
11874 // results can be OR'd together.
11875 bool TwoInputs = V1Used && V2Used;
11876 V1 = getPSHUFB(MaskVals, V1, dl, DAG);
11878 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
11880 // Calculate the shuffle mask for the second input, shuffle it, and
11881 // OR it with the first shuffled input.
11882 CommuteVectorShuffleMask(MaskVals, 8);
11883 V2 = getPSHUFB(MaskVals, V2, dl, DAG);
11884 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
11885 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
11888 // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order,
11889 // and update MaskVals with new element order.
11890 std::bitset<8> InOrder;
11891 if (BestLoQuad >= 0) {
11892 int MaskV[] = { -1, -1, -1, -1, 4, 5, 6, 7 };
11893 for (int i = 0; i != 4; ++i) {
11894 int idx = MaskVals[i];
11897 } else if ((idx / 4) == BestLoQuad) {
11898 MaskV[i] = idx & 3;
11902 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
11905 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) {
11906 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
11907 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16,
11908 NewV.getOperand(0),
11909 getShufflePSHUFLWImmediate(SVOp), DAG);
11913 // If BestHi >= 0, generate a pshufhw to put the high elements in order,
11914 // and update MaskVals with the new element order.
11915 if (BestHiQuad >= 0) {
11916 int MaskV[] = { 0, 1, 2, 3, -1, -1, -1, -1 };
11917 for (unsigned i = 4; i != 8; ++i) {
11918 int idx = MaskVals[i];
11921 } else if ((idx / 4) == BestHiQuad) {
11922 MaskV[i] = (idx & 3) + 4;
11926 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
11929 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) {
11930 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
11931 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16,
11932 NewV.getOperand(0),
11933 getShufflePSHUFHWImmediate(SVOp), DAG);
11937 // In case BestHi & BestLo were both -1, which means each quadword has a word
11938 // from each of the four input quadwords, calculate the InOrder bitvector now
11939 // before falling through to the insert/extract cleanup.
11940 if (BestLoQuad == -1 && BestHiQuad == -1) {
11942 for (int i = 0; i != 8; ++i)
11943 if (MaskVals[i] < 0 || MaskVals[i] == i)
11947 // The other elements are put in the right place using pextrw and pinsrw.
11948 for (unsigned i = 0; i != 8; ++i) {
11951 int EltIdx = MaskVals[i];
11954 SDValue ExtOp = (EltIdx < 8) ?
11955 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1,
11956 DAG.getIntPtrConstant(EltIdx)) :
11957 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2,
11958 DAG.getIntPtrConstant(EltIdx - 8));
11959 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp,
11960 DAG.getIntPtrConstant(i));
11965 /// \brief v16i16 shuffles
11967 /// FIXME: We only support generation of a single pshufb currently. We can
11968 /// generalize the other applicable cases from LowerVECTOR_SHUFFLEv8i16 as
11969 /// well (e.g 2 x pshufb + 1 x por).
11971 LowerVECTOR_SHUFFLEv16i16(SDValue Op, SelectionDAG &DAG) {
11972 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11973 SDValue V1 = SVOp->getOperand(0);
11974 SDValue V2 = SVOp->getOperand(1);
11977 if (V2.getOpcode() != ISD::UNDEF)
11980 SmallVector<int, 16> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
11981 return getPSHUFB(MaskVals, V1, dl, DAG);
11984 // v16i8 shuffles - Prefer shuffles in the following order:
11985 // 1. [ssse3] 1 x pshufb
11986 // 2. [ssse3] 2 x pshufb + 1 x por
11987 // 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw
11988 static SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
11989 const X86Subtarget* Subtarget,
11990 SelectionDAG &DAG) {
11991 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11992 SDValue V1 = SVOp->getOperand(0);
11993 SDValue V2 = SVOp->getOperand(1);
11995 ArrayRef<int> MaskVals = SVOp->getMask();
11997 // Promote splats to a larger type which usually leads to more efficient code.
11998 // FIXME: Is this true if pshufb is available?
11999 if (SVOp->isSplat())
12000 return PromoteSplat(SVOp, DAG);
12002 // If we have SSSE3, case 1 is generated when all result bytes come from
12003 // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is
12004 // present, fall back to case 3.
12006 // If SSSE3, use 1 pshufb instruction per vector with elements in the result.
12007 if (Subtarget->hasSSSE3()) {
12008 SmallVector<SDValue,16> pshufbMask;
12010 // If all result elements are from one input vector, then only translate
12011 // undef mask values to 0x80 (zero out result) in the pshufb mask.
12013 // Otherwise, we have elements from both input vectors, and must zero out
12014 // elements that come from V2 in the first mask, and V1 in the second mask
12015 // so that we can OR them together.
12016 for (unsigned i = 0; i != 16; ++i) {
12017 int EltIdx = MaskVals[i];
12018 if (EltIdx < 0 || EltIdx >= 16)
12020 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
12022 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
12023 DAG.getNode(ISD::BUILD_VECTOR, dl,
12024 MVT::v16i8, pshufbMask));
12026 // As PSHUFB will zero elements with negative indices, it's safe to ignore
12027 // the 2nd operand if it's undefined or zero.
12028 if (V2.getOpcode() == ISD::UNDEF ||
12029 ISD::isBuildVectorAllZeros(V2.getNode()))
12032 // Calculate the shuffle mask for the second input, shuffle it, and
12033 // OR it with the first shuffled input.
12034 pshufbMask.clear();
12035 for (unsigned i = 0; i != 16; ++i) {
12036 int EltIdx = MaskVals[i];
12037 EltIdx = (EltIdx < 16) ? 0x80 : EltIdx - 16;
12038 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
12040 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
12041 DAG.getNode(ISD::BUILD_VECTOR, dl,
12042 MVT::v16i8, pshufbMask));
12043 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
12046 // No SSSE3 - Calculate in place words and then fix all out of place words
12047 // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from
12048 // the 16 different words that comprise the two doublequadword input vectors.
12049 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
12050 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2);
12052 for (int i = 0; i != 8; ++i) {
12053 int Elt0 = MaskVals[i*2];
12054 int Elt1 = MaskVals[i*2+1];
12056 // This word of the result is all undef, skip it.
12057 if (Elt0 < 0 && Elt1 < 0)
12060 // This word of the result is already in the correct place, skip it.
12061 if ((Elt0 == i*2) && (Elt1 == i*2+1))
12064 SDValue Elt0Src = Elt0 < 16 ? V1 : V2;
12065 SDValue Elt1Src = Elt1 < 16 ? V1 : V2;
12068 // If Elt0 and Elt1 are defined, are consecutive, and can be load
12069 // using a single extract together, load it and store it.
12070 if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) {
12071 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
12072 DAG.getIntPtrConstant(Elt1 / 2));
12073 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
12074 DAG.getIntPtrConstant(i));
12078 // If Elt1 is defined, extract it from the appropriate source. If the
12079 // source byte is not also odd, shift the extracted word left 8 bits
12080 // otherwise clear the bottom 8 bits if we need to do an or.
12082 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
12083 DAG.getIntPtrConstant(Elt1 / 2));
12084 if ((Elt1 & 1) == 0)
12085 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt,
12087 TLI.getShiftAmountTy(InsElt.getValueType())));
12088 else if (Elt0 >= 0)
12089 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt,
12090 DAG.getConstant(0xFF00, MVT::i16));
12092 // If Elt0 is defined, extract it from the appropriate source. If the
12093 // source byte is not also even, shift the extracted word right 8 bits. If
12094 // Elt1 was also defined, OR the extracted values together before
12095 // inserting them in the result.
12097 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
12098 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2));
12099 if ((Elt0 & 1) != 0)
12100 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0,
12102 TLI.getShiftAmountTy(InsElt0.getValueType())));
12103 else if (Elt1 >= 0)
12104 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0,
12105 DAG.getConstant(0x00FF, MVT::i16));
12106 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0)
12109 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
12110 DAG.getIntPtrConstant(i));
12112 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV);
12115 // v32i8 shuffles - Translate to VPSHUFB if possible.
12117 SDValue LowerVECTOR_SHUFFLEv32i8(ShuffleVectorSDNode *SVOp,
12118 const X86Subtarget *Subtarget,
12119 SelectionDAG &DAG) {
12120 MVT VT = SVOp->getSimpleValueType(0);
12121 SDValue V1 = SVOp->getOperand(0);
12122 SDValue V2 = SVOp->getOperand(1);
12124 SmallVector<int, 32> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
12126 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
12127 bool V1IsAllZero = ISD::isBuildVectorAllZeros(V1.getNode());
12128 bool V2IsAllZero = ISD::isBuildVectorAllZeros(V2.getNode());
12130 // VPSHUFB may be generated if
12131 // (1) one of input vector is undefined or zeroinitializer.
12132 // The mask value 0x80 puts 0 in the corresponding slot of the vector.
12133 // And (2) the mask indexes don't cross the 128-bit lane.
12134 if (VT != MVT::v32i8 || !Subtarget->hasInt256() ||
12135 (!V2IsUndef && !V2IsAllZero && !V1IsAllZero))
12138 if (V1IsAllZero && !V2IsAllZero) {
12139 CommuteVectorShuffleMask(MaskVals, 32);
12142 return getPSHUFB(MaskVals, V1, dl, DAG);
12145 /// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide
12146 /// ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be
12147 /// done when every pair / quad of shuffle mask elements point to elements in
12148 /// the right sequence. e.g.
12149 /// vector_shuffle X, Y, <2, 3, | 10, 11, | 0, 1, | 14, 15>
12151 SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp,
12152 SelectionDAG &DAG) {
12153 MVT VT = SVOp->getSimpleValueType(0);
12155 unsigned NumElems = VT.getVectorNumElements();
12158 switch (VT.SimpleTy) {
12159 default: llvm_unreachable("Unexpected!");
12162 return SDValue(SVOp, 0);
12163 case MVT::v4f32: NewVT = MVT::v2f64; Scale = 2; break;
12164 case MVT::v4i32: NewVT = MVT::v2i64; Scale = 2; break;
12165 case MVT::v8i16: NewVT = MVT::v4i32; Scale = 2; break;
12166 case MVT::v16i8: NewVT = MVT::v4i32; Scale = 4; break;
12167 case MVT::v16i16: NewVT = MVT::v8i32; Scale = 2; break;
12168 case MVT::v32i8: NewVT = MVT::v8i32; Scale = 4; break;
12171 SmallVector<int, 8> MaskVec;
12172 for (unsigned i = 0; i != NumElems; i += Scale) {
12174 for (unsigned j = 0; j != Scale; ++j) {
12175 int EltIdx = SVOp->getMaskElt(i+j);
12179 StartIdx = (EltIdx / Scale);
12180 if (EltIdx != (int)(StartIdx*Scale + j))
12183 MaskVec.push_back(StartIdx);
12186 SDValue V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(0));
12187 SDValue V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(1));
12188 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]);
12191 /// getVZextMovL - Return a zero-extending vector move low node.
12193 static SDValue getVZextMovL(MVT VT, MVT OpVT,
12194 SDValue SrcOp, SelectionDAG &DAG,
12195 const X86Subtarget *Subtarget, SDLoc dl) {
12196 if (VT == MVT::v2f64 || VT == MVT::v4f32) {
12197 LoadSDNode *LD = nullptr;
12198 if (!isScalarLoadToVector(SrcOp.getNode(), &LD))
12199 LD = dyn_cast<LoadSDNode>(SrcOp);
12201 // movssrr and movsdrr do not clear top bits. Try to use movd, movq
12203 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32;
12204 if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) &&
12205 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR &&
12206 SrcOp.getOperand(0).getOpcode() == ISD::BITCAST &&
12207 SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) {
12209 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32;
12210 return DAG.getNode(ISD::BITCAST, dl, VT,
12211 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
12212 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
12214 SrcOp.getOperand(0)
12220 return DAG.getNode(ISD::BITCAST, dl, VT,
12221 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
12222 DAG.getNode(ISD::BITCAST, dl,
12226 /// LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles
12227 /// which could not be matched by any known target speficic shuffle
12229 LowerVECTOR_SHUFFLE_256(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
12231 SDValue NewOp = Compact8x32ShuffleNode(SVOp, DAG);
12232 if (NewOp.getNode())
12235 MVT VT = SVOp->getSimpleValueType(0);
12237 unsigned NumElems = VT.getVectorNumElements();
12238 unsigned NumLaneElems = NumElems / 2;
12241 MVT EltVT = VT.getVectorElementType();
12242 MVT NVT = MVT::getVectorVT(EltVT, NumLaneElems);
12245 SmallVector<int, 16> Mask;
12246 for (unsigned l = 0; l < 2; ++l) {
12247 // Build a shuffle mask for the output, discovering on the fly which
12248 // input vectors to use as shuffle operands (recorded in InputUsed).
12249 // If building a suitable shuffle vector proves too hard, then bail
12250 // out with UseBuildVector set.
12251 bool UseBuildVector = false;
12252 int InputUsed[2] = { -1, -1 }; // Not yet discovered.
12253 unsigned LaneStart = l * NumLaneElems;
12254 for (unsigned i = 0; i != NumLaneElems; ++i) {
12255 // The mask element. This indexes into the input.
12256 int Idx = SVOp->getMaskElt(i+LaneStart);
12258 // the mask element does not index into any input vector.
12259 Mask.push_back(-1);
12263 // The input vector this mask element indexes into.
12264 int Input = Idx / NumLaneElems;
12266 // Turn the index into an offset from the start of the input vector.
12267 Idx -= Input * NumLaneElems;
12269 // Find or create a shuffle vector operand to hold this input.
12271 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
12272 if (InputUsed[OpNo] == Input)
12273 // This input vector is already an operand.
12275 if (InputUsed[OpNo] < 0) {
12276 // Create a new operand for this input vector.
12277 InputUsed[OpNo] = Input;
12282 if (OpNo >= array_lengthof(InputUsed)) {
12283 // More than two input vectors used! Give up on trying to create a
12284 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
12285 UseBuildVector = true;
12289 // Add the mask index for the new shuffle vector.
12290 Mask.push_back(Idx + OpNo * NumLaneElems);
12293 if (UseBuildVector) {
12294 SmallVector<SDValue, 16> SVOps;
12295 for (unsigned i = 0; i != NumLaneElems; ++i) {
12296 // The mask element. This indexes into the input.
12297 int Idx = SVOp->getMaskElt(i+LaneStart);
12299 SVOps.push_back(DAG.getUNDEF(EltVT));
12303 // The input vector this mask element indexes into.
12304 int Input = Idx / NumElems;
12306 // Turn the index into an offset from the start of the input vector.
12307 Idx -= Input * NumElems;
12309 // Extract the vector element by hand.
12310 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
12311 SVOp->getOperand(Input),
12312 DAG.getIntPtrConstant(Idx)));
12315 // Construct the output using a BUILD_VECTOR.
12316 Output[l] = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, SVOps);
12317 } else if (InputUsed[0] < 0) {
12318 // No input vectors were used! The result is undefined.
12319 Output[l] = DAG.getUNDEF(NVT);
12321 SDValue Op0 = Extract128BitVector(SVOp->getOperand(InputUsed[0] / 2),
12322 (InputUsed[0] % 2) * NumLaneElems,
12324 // If only one input was used, use an undefined vector for the other.
12325 SDValue Op1 = (InputUsed[1] < 0) ? DAG.getUNDEF(NVT) :
12326 Extract128BitVector(SVOp->getOperand(InputUsed[1] / 2),
12327 (InputUsed[1] % 2) * NumLaneElems, DAG, dl);
12328 // At least one input vector was used. Create a new shuffle vector.
12329 Output[l] = DAG.getVectorShuffle(NVT, dl, Op0, Op1, &Mask[0]);
12335 // Concatenate the result back
12336 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Output[0], Output[1]);
12339 /// LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with
12340 /// 4 elements, and match them with several different shuffle types.
12342 LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
12343 SDValue V1 = SVOp->getOperand(0);
12344 SDValue V2 = SVOp->getOperand(1);
12346 MVT VT = SVOp->getSimpleValueType(0);
12348 assert(VT.is128BitVector() && "Unsupported vector size");
12350 std::pair<int, int> Locs[4];
12351 int Mask1[] = { -1, -1, -1, -1 };
12352 SmallVector<int, 8> PermMask(SVOp->getMask().begin(), SVOp->getMask().end());
12354 unsigned NumHi = 0;
12355 unsigned NumLo = 0;
12356 for (unsigned i = 0; i != 4; ++i) {
12357 int Idx = PermMask[i];
12359 Locs[i] = std::make_pair(-1, -1);
12361 assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!");
12363 Locs[i] = std::make_pair(0, NumLo);
12364 Mask1[NumLo] = Idx;
12367 Locs[i] = std::make_pair(1, NumHi);
12369 Mask1[2+NumHi] = Idx;
12375 if (NumLo <= 2 && NumHi <= 2) {
12376 // If no more than two elements come from either vector. This can be
12377 // implemented with two shuffles. First shuffle gather the elements.
12378 // The second shuffle, which takes the first shuffle as both of its
12379 // vector operands, put the elements into the right order.
12380 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
12382 int Mask2[] = { -1, -1, -1, -1 };
12384 for (unsigned i = 0; i != 4; ++i)
12385 if (Locs[i].first != -1) {
12386 unsigned Idx = (i < 2) ? 0 : 4;
12387 Idx += Locs[i].first * 2 + Locs[i].second;
12391 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]);
12394 if (NumLo == 3 || NumHi == 3) {
12395 // Otherwise, we must have three elements from one vector, call it X, and
12396 // one element from the other, call it Y. First, use a shufps to build an
12397 // intermediate vector with the one element from Y and the element from X
12398 // that will be in the same half in the final destination (the indexes don't
12399 // matter). Then, use a shufps to build the final vector, taking the half
12400 // containing the element from Y from the intermediate, and the other half
12403 // Normalize it so the 3 elements come from V1.
12404 CommuteVectorShuffleMask(PermMask, 4);
12408 // Find the element from V2.
12410 for (HiIndex = 0; HiIndex < 3; ++HiIndex) {
12411 int Val = PermMask[HiIndex];
12418 Mask1[0] = PermMask[HiIndex];
12420 Mask1[2] = PermMask[HiIndex^1];
12422 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
12424 if (HiIndex >= 2) {
12425 Mask1[0] = PermMask[0];
12426 Mask1[1] = PermMask[1];
12427 Mask1[2] = HiIndex & 1 ? 6 : 4;
12428 Mask1[3] = HiIndex & 1 ? 4 : 6;
12429 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
12432 Mask1[0] = HiIndex & 1 ? 2 : 0;
12433 Mask1[1] = HiIndex & 1 ? 0 : 2;
12434 Mask1[2] = PermMask[2];
12435 Mask1[3] = PermMask[3];
12440 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]);
12443 // Break it into (shuffle shuffle_hi, shuffle_lo).
12444 int LoMask[] = { -1, -1, -1, -1 };
12445 int HiMask[] = { -1, -1, -1, -1 };
12447 int *MaskPtr = LoMask;
12448 unsigned MaskIdx = 0;
12449 unsigned LoIdx = 0;
12450 unsigned HiIdx = 2;
12451 for (unsigned i = 0; i != 4; ++i) {
12458 int Idx = PermMask[i];
12460 Locs[i] = std::make_pair(-1, -1);
12461 } else if (Idx < 4) {
12462 Locs[i] = std::make_pair(MaskIdx, LoIdx);
12463 MaskPtr[LoIdx] = Idx;
12466 Locs[i] = std::make_pair(MaskIdx, HiIdx);
12467 MaskPtr[HiIdx] = Idx;
12472 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]);
12473 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]);
12474 int MaskOps[] = { -1, -1, -1, -1 };
12475 for (unsigned i = 0; i != 4; ++i)
12476 if (Locs[i].first != -1)
12477 MaskOps[i] = Locs[i].first * 4 + Locs[i].second;
12478 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]);
12481 static bool MayFoldVectorLoad(SDValue V) {
12482 while (V.hasOneUse() && V.getOpcode() == ISD::BITCAST)
12483 V = V.getOperand(0);
12485 if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR)
12486 V = V.getOperand(0);
12487 if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR &&
12488 V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF)
12489 // BUILD_VECTOR (load), undef
12490 V = V.getOperand(0);
12492 return MayFoldLoad(V);
12496 SDValue getMOVDDup(SDValue &Op, SDLoc &dl, SDValue V1, SelectionDAG &DAG) {
12497 MVT VT = Op.getSimpleValueType();
12499 // Canonicalize to v2f64.
12500 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1);
12501 return DAG.getNode(ISD::BITCAST, dl, VT,
12502 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64,
12507 SDValue getMOVLowToHigh(SDValue &Op, SDLoc &dl, SelectionDAG &DAG,
12509 SDValue V1 = Op.getOperand(0);
12510 SDValue V2 = Op.getOperand(1);
12511 MVT VT = Op.getSimpleValueType();
12513 assert(VT != MVT::v2i64 && "unsupported shuffle type");
12515 if (HasSSE2 && VT == MVT::v2f64)
12516 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
12518 // v4f32 or v4i32: canonicalize to v4f32 (which is legal for SSE1)
12519 return DAG.getNode(ISD::BITCAST, dl, VT,
12520 getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32,
12521 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1),
12522 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG));
12526 SDValue getMOVHighToLow(SDValue &Op, SDLoc &dl, SelectionDAG &DAG) {
12527 SDValue V1 = Op.getOperand(0);
12528 SDValue V2 = Op.getOperand(1);
12529 MVT VT = Op.getSimpleValueType();
12531 assert((VT == MVT::v4i32 || VT == MVT::v4f32) &&
12532 "unsupported shuffle type");
12534 if (V2.getOpcode() == ISD::UNDEF)
12538 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
12542 SDValue getMOVLP(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
12543 SDValue V1 = Op.getOperand(0);
12544 SDValue V2 = Op.getOperand(1);
12545 MVT VT = Op.getSimpleValueType();
12546 unsigned NumElems = VT.getVectorNumElements();
12548 // Use MOVLPS and MOVLPD in case V1 or V2 are loads. During isel, the second
12549 // operand of these instructions is only memory, so check if there's a
12550 // potencial load folding here, otherwise use SHUFPS or MOVSD to match the
12552 bool CanFoldLoad = false;
12554 // Trivial case, when V2 comes from a load.
12555 if (MayFoldVectorLoad(V2))
12556 CanFoldLoad = true;
12558 // When V1 is a load, it can be folded later into a store in isel, example:
12559 // (store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), addr:$src1)
12561 // (MOVLPSmr addr:$src1, VR128:$src2)
12562 // So, recognize this potential and also use MOVLPS or MOVLPD
12563 else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op))
12564 CanFoldLoad = true;
12566 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12568 if (HasSSE2 && NumElems == 2)
12569 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
12572 // If we don't care about the second element, proceed to use movss.
12573 if (SVOp->getMaskElt(1) != -1)
12574 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG);
12577 // movl and movlp will both match v2i64, but v2i64 is never matched by
12578 // movl earlier because we make it strict to avoid messing with the movlp load
12579 // folding logic (see the code above getMOVLP call). Match it here then,
12580 // this is horrible, but will stay like this until we move all shuffle
12581 // matching to x86 specific nodes. Note that for the 1st condition all
12582 // types are matched with movsd.
12584 // FIXME: isMOVLMask should be checked and matched before getMOVLP,
12585 // as to remove this logic from here, as much as possible
12586 if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT))
12587 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
12588 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
12591 assert(VT != MVT::v4i32 && "unsupported shuffle type");
12593 // Invert the operand order and use SHUFPS to match it.
12594 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1,
12595 getShuffleSHUFImmediate(SVOp), DAG);
12598 static SDValue NarrowVectorLoadToElement(LoadSDNode *Load, unsigned Index,
12599 SelectionDAG &DAG) {
12601 MVT VT = Load->getSimpleValueType(0);
12602 MVT EVT = VT.getVectorElementType();
12603 SDValue Addr = Load->getOperand(1);
12604 SDValue NewAddr = DAG.getNode(
12605 ISD::ADD, dl, Addr.getSimpleValueType(), Addr,
12606 DAG.getConstant(Index * EVT.getStoreSize(), Addr.getSimpleValueType()));
12609 DAG.getLoad(EVT, dl, Load->getChain(), NewAddr,
12610 DAG.getMachineFunction().getMachineMemOperand(
12611 Load->getMemOperand(), 0, EVT.getStoreSize()));
12615 // It is only safe to call this function if isINSERTPSMask is true for
12616 // this shufflevector mask.
12617 static SDValue getINSERTPS(ShuffleVectorSDNode *SVOp, SDLoc &dl,
12618 SelectionDAG &DAG) {
12619 // Generate an insertps instruction when inserting an f32 from memory onto a
12620 // v4f32 or when copying a member from one v4f32 to another.
12621 // We also use it for transferring i32 from one register to another,
12622 // since it simply copies the same bits.
12623 // If we're transferring an i32 from memory to a specific element in a
12624 // register, we output a generic DAG that will match the PINSRD
12626 MVT VT = SVOp->getSimpleValueType(0);
12627 MVT EVT = VT.getVectorElementType();
12628 SDValue V1 = SVOp->getOperand(0);
12629 SDValue V2 = SVOp->getOperand(1);
12630 auto Mask = SVOp->getMask();
12631 assert((VT == MVT::v4f32 || VT == MVT::v4i32) &&
12632 "unsupported vector type for insertps/pinsrd");
12634 auto FromV1Predicate = [](const int &i) { return i < 4 && i > -1; };
12635 auto FromV2Predicate = [](const int &i) { return i >= 4; };
12636 int FromV1 = std::count_if(Mask.begin(), Mask.end(), FromV1Predicate);
12640 unsigned DestIndex;
12644 DestIndex = std::find_if(Mask.begin(), Mask.end(), FromV1Predicate) -
12647 // If we have 1 element from each vector, we have to check if we're
12648 // changing V1's element's place. If so, we're done. Otherwise, we
12649 // should assume we're changing V2's element's place and behave
12651 int FromV2 = std::count_if(Mask.begin(), Mask.end(), FromV2Predicate);
12652 assert(DestIndex <= INT32_MAX && "truncated destination index");
12653 if (FromV1 == FromV2 &&
12654 static_cast<int>(DestIndex) == Mask[DestIndex] % 4) {
12658 std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin();
12661 assert(std::count_if(Mask.begin(), Mask.end(), FromV2Predicate) == 1 &&
12662 "More than one element from V1 and from V2, or no elements from one "
12663 "of the vectors. This case should not have returned true from "
12668 std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin();
12671 // Get an index into the source vector in the range [0,4) (the mask is
12672 // in the range [0,8) because it can address V1 and V2)
12673 unsigned SrcIndex = Mask[DestIndex] % 4;
12674 if (MayFoldLoad(From)) {
12675 // Trivial case, when From comes from a load and is only used by the
12676 // shuffle. Make it use insertps from the vector that we need from that
12679 NarrowVectorLoadToElement(cast<LoadSDNode>(From), SrcIndex, DAG);
12680 if (!NewLoad.getNode())
12683 if (EVT == MVT::f32) {
12684 // Create this as a scalar to vector to match the instruction pattern.
12685 SDValue LoadScalarToVector =
12686 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, NewLoad);
12687 SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4);
12688 return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, LoadScalarToVector,
12690 } else { // EVT == MVT::i32
12691 // If we're getting an i32 from memory, use an INSERT_VECTOR_ELT
12692 // instruction, to match the PINSRD instruction, which loads an i32 to a
12693 // certain vector element.
12694 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, To, NewLoad,
12695 DAG.getConstant(DestIndex, MVT::i32));
12699 // Vector-element-to-vector
12700 SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4 | SrcIndex << 6);
12701 return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, From, InsertpsMask);
12704 // Reduce a vector shuffle to zext.
12705 static SDValue LowerVectorIntExtend(SDValue Op, const X86Subtarget *Subtarget,
12706 SelectionDAG &DAG) {
12707 // PMOVZX is only available from SSE41.
12708 if (!Subtarget->hasSSE41())
12711 MVT VT = Op.getSimpleValueType();
12713 // Only AVX2 support 256-bit vector integer extending.
12714 if (!Subtarget->hasInt256() && VT.is256BitVector())
12717 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12719 SDValue V1 = Op.getOperand(0);
12720 SDValue V2 = Op.getOperand(1);
12721 unsigned NumElems = VT.getVectorNumElements();
12723 // Extending is an unary operation and the element type of the source vector
12724 // won't be equal to or larger than i64.
12725 if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() ||
12726 VT.getVectorElementType() == MVT::i64)
12729 // Find the expansion ratio, e.g. expanding from i8 to i32 has a ratio of 4.
12730 unsigned Shift = 1; // Start from 2, i.e. 1 << 1.
12731 while ((1U << Shift) < NumElems) {
12732 if (SVOp->getMaskElt(1U << Shift) == 1)
12735 // The maximal ratio is 8, i.e. from i8 to i64.
12740 // Check the shuffle mask.
12741 unsigned Mask = (1U << Shift) - 1;
12742 for (unsigned i = 0; i != NumElems; ++i) {
12743 int EltIdx = SVOp->getMaskElt(i);
12744 if ((i & Mask) != 0 && EltIdx != -1)
12746 if ((i & Mask) == 0 && (unsigned)EltIdx != (i >> Shift))
12750 unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift;
12751 MVT NeVT = MVT::getIntegerVT(NBits);
12752 MVT NVT = MVT::getVectorVT(NeVT, NumElems >> Shift);
12754 if (!DAG.getTargetLoweringInfo().isTypeLegal(NVT))
12757 return DAG.getNode(ISD::BITCAST, DL, VT,
12758 DAG.getNode(X86ISD::VZEXT, DL, NVT, V1));
12761 static SDValue NormalizeVectorShuffle(SDValue Op, const X86Subtarget *Subtarget,
12762 SelectionDAG &DAG) {
12763 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12764 MVT VT = Op.getSimpleValueType();
12766 SDValue V1 = Op.getOperand(0);
12767 SDValue V2 = Op.getOperand(1);
12769 if (isZeroShuffle(SVOp))
12770 return getZeroVector(VT, Subtarget, DAG, dl);
12772 // Handle splat operations
12773 if (SVOp->isSplat()) {
12774 // Use vbroadcast whenever the splat comes from a foldable load
12775 SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG);
12776 if (Broadcast.getNode())
12780 // Check integer expanding shuffles.
12781 SDValue NewOp = LowerVectorIntExtend(Op, Subtarget, DAG);
12782 if (NewOp.getNode())
12785 // If the shuffle can be profitably rewritten as a narrower shuffle, then
12787 if (VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v16i16 ||
12788 VT == MVT::v32i8) {
12789 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
12790 if (NewOp.getNode())
12791 return DAG.getNode(ISD::BITCAST, dl, VT, NewOp);
12792 } else if (VT.is128BitVector() && Subtarget->hasSSE2()) {
12793 // FIXME: Figure out a cleaner way to do this.
12794 if (ISD::isBuildVectorAllZeros(V2.getNode())) {
12795 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
12796 if (NewOp.getNode()) {
12797 MVT NewVT = NewOp.getSimpleValueType();
12798 if (isCommutedMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(),
12799 NewVT, true, false))
12800 return getVZextMovL(VT, NewVT, NewOp.getOperand(0), DAG, Subtarget,
12803 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) {
12804 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
12805 if (NewOp.getNode()) {
12806 MVT NewVT = NewOp.getSimpleValueType();
12807 if (isMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(), NewVT))
12808 return getVZextMovL(VT, NewVT, NewOp.getOperand(1), DAG, Subtarget,
12817 X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
12818 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12819 SDValue V1 = Op.getOperand(0);
12820 SDValue V2 = Op.getOperand(1);
12821 MVT VT = Op.getSimpleValueType();
12823 unsigned NumElems = VT.getVectorNumElements();
12824 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
12825 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
12826 bool V1IsSplat = false;
12827 bool V2IsSplat = false;
12828 bool HasSSE2 = Subtarget->hasSSE2();
12829 bool HasFp256 = Subtarget->hasFp256();
12830 bool HasInt256 = Subtarget->hasInt256();
12831 MachineFunction &MF = DAG.getMachineFunction();
12833 MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
12835 // Check if we should use the experimental vector shuffle lowering. If so,
12836 // delegate completely to that code path.
12837 if (ExperimentalVectorShuffleLowering)
12838 return lowerVectorShuffle(Op, Subtarget, DAG);
12840 assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
12842 if (V1IsUndef && V2IsUndef)
12843 return DAG.getUNDEF(VT);
12845 // When we create a shuffle node we put the UNDEF node to second operand,
12846 // but in some cases the first operand may be transformed to UNDEF.
12847 // In this case we should just commute the node.
12849 return DAG.getCommutedVectorShuffle(*SVOp);
12851 // Vector shuffle lowering takes 3 steps:
12853 // 1) Normalize the input vectors. Here splats, zeroed vectors, profitable
12854 // narrowing and commutation of operands should be handled.
12855 // 2) Matching of shuffles with known shuffle masks to x86 target specific
12857 // 3) Rewriting of unmatched masks into new generic shuffle operations,
12858 // so the shuffle can be broken into other shuffles and the legalizer can
12859 // try the lowering again.
12861 // The general idea is that no vector_shuffle operation should be left to
12862 // be matched during isel, all of them must be converted to a target specific
12865 // Normalize the input vectors. Here splats, zeroed vectors, profitable
12866 // narrowing and commutation of operands should be handled. The actual code
12867 // doesn't include all of those, work in progress...
12868 SDValue NewOp = NormalizeVectorShuffle(Op, Subtarget, DAG);
12869 if (NewOp.getNode())
12872 SmallVector<int, 8> M(SVOp->getMask().begin(), SVOp->getMask().end());
12874 // NOTE: isPSHUFDMask can also match both masks below (unpckl_undef and
12875 // unpckh_undef). Only use pshufd if speed is more important than size.
12876 if (OptForSize && isUNPCKL_v_undef_Mask(M, VT, HasInt256))
12877 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
12878 if (OptForSize && isUNPCKH_v_undef_Mask(M, VT, HasInt256))
12879 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
12881 if (isMOVDDUPMask(M, VT) && Subtarget->hasSSE3() &&
12882 V2IsUndef && MayFoldVectorLoad(V1))
12883 return getMOVDDup(Op, dl, V1, DAG);
12885 if (isMOVHLPS_v_undef_Mask(M, VT))
12886 return getMOVHighToLow(Op, dl, DAG);
12888 // Use to match splats
12889 if (HasSSE2 && isUNPCKHMask(M, VT, HasInt256) && V2IsUndef &&
12890 (VT == MVT::v2f64 || VT == MVT::v2i64))
12891 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
12893 if (isPSHUFDMask(M, VT)) {
12894 // The actual implementation will match the mask in the if above and then
12895 // during isel it can match several different instructions, not only pshufd
12896 // as its name says, sad but true, emulate the behavior for now...
12897 if (isMOVDDUPMask(M, VT) && ((VT == MVT::v4f32 || VT == MVT::v2i64)))
12898 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG);
12900 unsigned TargetMask = getShuffleSHUFImmediate(SVOp);
12902 if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32))
12903 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG);
12905 if (HasFp256 && (VT == MVT::v4f32 || VT == MVT::v2f64))
12906 return getTargetShuffleNode(X86ISD::VPERMILPI, dl, VT, V1, TargetMask,
12909 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V1,
12913 if (isPALIGNRMask(M, VT, Subtarget))
12914 return getTargetShuffleNode(X86ISD::PALIGNR, dl, VT, V1, V2,
12915 getShufflePALIGNRImmediate(SVOp),
12918 if (isVALIGNMask(M, VT, Subtarget))
12919 return getTargetShuffleNode(X86ISD::VALIGN, dl, VT, V1, V2,
12920 getShuffleVALIGNImmediate(SVOp),
12923 // Check if this can be converted into a logical shift.
12924 bool isLeft = false;
12925 unsigned ShAmt = 0;
12927 bool isShift = HasSSE2 && isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt);
12928 if (isShift && ShVal.hasOneUse()) {
12929 // If the shifted value has multiple uses, it may be cheaper to use
12930 // v_set0 + movlhps or movhlps, etc.
12931 MVT EltVT = VT.getVectorElementType();
12932 ShAmt *= EltVT.getSizeInBits();
12933 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
12936 if (isMOVLMask(M, VT)) {
12937 if (ISD::isBuildVectorAllZeros(V1.getNode()))
12938 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl);
12939 if (!isMOVLPMask(M, VT)) {
12940 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64))
12941 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
12943 if (VT == MVT::v4i32 || VT == MVT::v4f32)
12944 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
12948 // FIXME: fold these into legal mask.
12949 if (isMOVLHPSMask(M, VT) && !isUNPCKLMask(M, VT, HasInt256))
12950 return getMOVLowToHigh(Op, dl, DAG, HasSSE2);
12952 if (isMOVHLPSMask(M, VT))
12953 return getMOVHighToLow(Op, dl, DAG);
12955 if (V2IsUndef && isMOVSHDUPMask(M, VT, Subtarget))
12956 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG);
12958 if (V2IsUndef && isMOVSLDUPMask(M, VT, Subtarget))
12959 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG);
12961 if (isMOVLPMask(M, VT))
12962 return getMOVLP(Op, dl, DAG, HasSSE2);
12964 if (ShouldXformToMOVHLPS(M, VT) ||
12965 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), M, VT))
12966 return DAG.getCommutedVectorShuffle(*SVOp);
12969 // No better options. Use a vshldq / vsrldq.
12970 MVT EltVT = VT.getVectorElementType();
12971 ShAmt *= EltVT.getSizeInBits();
12972 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
12975 bool Commuted = false;
12976 // FIXME: This should also accept a bitcast of a splat? Be careful, not
12977 // 1,1,1,1 -> v8i16 though.
12978 BitVector UndefElements;
12979 if (auto *BVOp = dyn_cast<BuildVectorSDNode>(V1.getNode()))
12980 if (BVOp->getConstantSplatNode(&UndefElements) && UndefElements.none())
12982 if (auto *BVOp = dyn_cast<BuildVectorSDNode>(V2.getNode()))
12983 if (BVOp->getConstantSplatNode(&UndefElements) && UndefElements.none())
12986 // Canonicalize the splat or undef, if present, to be on the RHS.
12987 if (!V2IsUndef && V1IsSplat && !V2IsSplat) {
12988 CommuteVectorShuffleMask(M, NumElems);
12990 std::swap(V1IsSplat, V2IsSplat);
12994 if (isCommutedMOVLMask(M, VT, V2IsSplat, V2IsUndef)) {
12995 // Shuffling low element of v1 into undef, just return v1.
12998 // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which
12999 // the instruction selector will not match, so get a canonical MOVL with
13000 // swapped operands to undo the commute.
13001 return getMOVL(DAG, dl, VT, V2, V1);
13004 if (isUNPCKLMask(M, VT, HasInt256))
13005 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
13007 if (isUNPCKHMask(M, VT, HasInt256))
13008 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
13011 // Normalize mask so all entries that point to V2 points to its first
13012 // element then try to match unpck{h|l} again. If match, return a
13013 // new vector_shuffle with the corrected mask.p
13014 SmallVector<int, 8> NewMask(M.begin(), M.end());
13015 NormalizeMask(NewMask, NumElems);
13016 if (isUNPCKLMask(NewMask, VT, HasInt256, true))
13017 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
13018 if (isUNPCKHMask(NewMask, VT, HasInt256, true))
13019 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
13023 // Commute is back and try unpck* again.
13024 // FIXME: this seems wrong.
13025 CommuteVectorShuffleMask(M, NumElems);
13027 std::swap(V1IsSplat, V2IsSplat);
13029 if (isUNPCKLMask(M, VT, HasInt256))
13030 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
13032 if (isUNPCKHMask(M, VT, HasInt256))
13033 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
13036 // Normalize the node to match x86 shuffle ops if needed
13037 if (!V2IsUndef && (isSHUFPMask(M, VT, /* Commuted */ true)))
13038 return DAG.getCommutedVectorShuffle(*SVOp);
13040 // The checks below are all present in isShuffleMaskLegal, but they are
13041 // inlined here right now to enable us to directly emit target specific
13042 // nodes, and remove one by one until they don't return Op anymore.
13044 if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) &&
13045 SVOp->getSplatIndex() == 0 && V2IsUndef) {
13046 if (VT == MVT::v2f64 || VT == MVT::v2i64)
13047 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
13050 if (isPSHUFHWMask(M, VT, HasInt256))
13051 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1,
13052 getShufflePSHUFHWImmediate(SVOp),
13055 if (isPSHUFLWMask(M, VT, HasInt256))
13056 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1,
13057 getShufflePSHUFLWImmediate(SVOp),
13060 unsigned MaskValue;
13061 if (isBlendMask(M, VT, Subtarget->hasSSE41(), HasInt256, &MaskValue))
13062 return LowerVECTOR_SHUFFLEtoBlend(SVOp, MaskValue, Subtarget, DAG);
13064 if (isSHUFPMask(M, VT))
13065 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V2,
13066 getShuffleSHUFImmediate(SVOp), DAG);
13068 if (isUNPCKL_v_undef_Mask(M, VT, HasInt256))
13069 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
13070 if (isUNPCKH_v_undef_Mask(M, VT, HasInt256))
13071 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
13073 //===--------------------------------------------------------------------===//
13074 // Generate target specific nodes for 128 or 256-bit shuffles only
13075 // supported in the AVX instruction set.
13078 // Handle VMOVDDUPY permutations
13079 if (V2IsUndef && isMOVDDUPYMask(M, VT, HasFp256))
13080 return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG);
13082 // Handle VPERMILPS/D* permutations
13083 if (isVPERMILPMask(M, VT)) {
13084 if ((HasInt256 && VT == MVT::v8i32) || VT == MVT::v16i32)
13085 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1,
13086 getShuffleSHUFImmediate(SVOp), DAG);
13087 return getTargetShuffleNode(X86ISD::VPERMILPI, dl, VT, V1,
13088 getShuffleSHUFImmediate(SVOp), DAG);
13092 if (VT.is512BitVector() && isINSERT64x4Mask(M, VT, &Idx))
13093 return Insert256BitVector(V1, Extract256BitVector(V2, 0, DAG, dl),
13094 Idx*(NumElems/2), DAG, dl);
13096 // Handle VPERM2F128/VPERM2I128 permutations
13097 if (isVPERM2X128Mask(M, VT, HasFp256))
13098 return getTargetShuffleNode(X86ISD::VPERM2X128, dl, VT, V1,
13099 V2, getShuffleVPERM2X128Immediate(SVOp), DAG);
13101 if (Subtarget->hasSSE41() && isINSERTPSMask(M, VT))
13102 return getINSERTPS(SVOp, dl, DAG);
13105 if (V2IsUndef && HasInt256 && isPermImmMask(M, VT, Imm8))
13106 return getTargetShuffleNode(X86ISD::VPERMI, dl, VT, V1, Imm8, DAG);
13108 if ((V2IsUndef && HasInt256 && VT.is256BitVector() && NumElems == 8) ||
13109 VT.is512BitVector()) {
13110 MVT MaskEltVT = MVT::getIntegerVT(VT.getVectorElementType().getSizeInBits());
13111 MVT MaskVectorVT = MVT::getVectorVT(MaskEltVT, NumElems);
13112 SmallVector<SDValue, 16> permclMask;
13113 for (unsigned i = 0; i != NumElems; ++i) {
13114 permclMask.push_back(DAG.getConstant((M[i]>=0) ? M[i] : 0, MaskEltVT));
13117 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVectorVT, permclMask);
13119 // Bitcast is for VPERMPS since mask is v8i32 but node takes v8f32
13120 return DAG.getNode(X86ISD::VPERMV, dl, VT,
13121 DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1);
13122 return DAG.getNode(X86ISD::VPERMV3, dl, VT, V1,
13123 DAG.getNode(ISD::BITCAST, dl, VT, Mask), V2);
13126 //===--------------------------------------------------------------------===//
13127 // Since no target specific shuffle was selected for this generic one,
13128 // lower it into other known shuffles. FIXME: this isn't true yet, but
13129 // this is the plan.
13132 // Handle v8i16 specifically since SSE can do byte extraction and insertion.
13133 if (VT == MVT::v8i16) {
13134 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, Subtarget, DAG);
13135 if (NewOp.getNode())
13139 if (VT == MVT::v16i16 && HasInt256) {
13140 SDValue NewOp = LowerVECTOR_SHUFFLEv16i16(Op, DAG);
13141 if (NewOp.getNode())
13145 if (VT == MVT::v16i8) {
13146 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, Subtarget, DAG);
13147 if (NewOp.getNode())
13151 if (VT == MVT::v32i8) {
13152 SDValue NewOp = LowerVECTOR_SHUFFLEv32i8(SVOp, Subtarget, DAG);
13153 if (NewOp.getNode())
13157 // Handle all 128-bit wide vectors with 4 elements, and match them with
13158 // several different shuffle types.
13159 if (NumElems == 4 && VT.is128BitVector())
13160 return LowerVECTOR_SHUFFLE_128v4(SVOp, DAG);
13162 // Handle general 256-bit shuffles
13163 if (VT.is256BitVector())
13164 return LowerVECTOR_SHUFFLE_256(SVOp, DAG);
13169 // This function assumes its argument is a BUILD_VECTOR of constants or
13170 // undef SDNodes. i.e: ISD::isBuildVectorOfConstantSDNodes(BuildVector) is
13172 static bool BUILD_VECTORtoBlendMask(BuildVectorSDNode *BuildVector,
13173 unsigned &MaskValue) {
13175 unsigned NumElems = BuildVector->getNumOperands();
13176 // There are 2 lanes if (NumElems > 8), and 1 lane otherwise.
13177 unsigned NumLanes = (NumElems - 1) / 8 + 1;
13178 unsigned NumElemsInLane = NumElems / NumLanes;
13180 // Blend for v16i16 should be symetric for the both lanes.
13181 for (unsigned i = 0; i < NumElemsInLane; ++i) {
13182 SDValue EltCond = BuildVector->getOperand(i);
13183 SDValue SndLaneEltCond =
13184 (NumLanes == 2) ? BuildVector->getOperand(i + NumElemsInLane) : EltCond;
13186 int Lane1Cond = -1, Lane2Cond = -1;
13187 if (isa<ConstantSDNode>(EltCond))
13188 Lane1Cond = !isZero(EltCond);
13189 if (isa<ConstantSDNode>(SndLaneEltCond))
13190 Lane2Cond = !isZero(SndLaneEltCond);
13192 if (Lane1Cond == Lane2Cond || Lane2Cond < 0)
13193 // Lane1Cond != 0, means we want the first argument.
13194 // Lane1Cond == 0, means we want the second argument.
13195 // The encoding of this argument is 0 for the first argument, 1
13196 // for the second. Therefore, invert the condition.
13197 MaskValue |= !Lane1Cond << i;
13198 else if (Lane1Cond < 0)
13199 MaskValue |= !Lane2Cond << i;
13206 /// \brief Try to lower a VSELECT instruction to an immediate-controlled blend
13208 static SDValue lowerVSELECTtoBLENDI(SDValue Op, const X86Subtarget *Subtarget,
13209 SelectionDAG &DAG) {
13210 SDValue Cond = Op.getOperand(0);
13211 SDValue LHS = Op.getOperand(1);
13212 SDValue RHS = Op.getOperand(2);
13214 MVT VT = Op.getSimpleValueType();
13215 MVT EltVT = VT.getVectorElementType();
13216 unsigned NumElems = VT.getVectorNumElements();
13218 // There is no blend with immediate in AVX-512.
13219 if (VT.is512BitVector())
13222 if (!Subtarget->hasSSE41() || EltVT == MVT::i8)
13224 if (!Subtarget->hasInt256() && VT == MVT::v16i16)
13227 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
13230 // Check the mask for BLEND and build the value.
13231 unsigned MaskValue = 0;
13232 if (!BUILD_VECTORtoBlendMask(cast<BuildVectorSDNode>(Cond), MaskValue))
13235 // Convert i32 vectors to floating point if it is not AVX2.
13236 // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors.
13238 if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) {
13239 BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()),
13241 LHS = DAG.getNode(ISD::BITCAST, dl, VT, LHS);
13242 RHS = DAG.getNode(ISD::BITCAST, dl, VT, RHS);
13245 SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, LHS, RHS,
13246 DAG.getConstant(MaskValue, MVT::i32));
13247 return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
13250 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
13251 // A vselect where all conditions and data are constants can be optimized into
13252 // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
13253 if (ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(0).getNode()) &&
13254 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(1).getNode()) &&
13255 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(2).getNode()))
13258 SDValue BlendOp = lowerVSELECTtoBLENDI(Op, Subtarget, DAG);
13259 if (BlendOp.getNode())
13262 // Some types for vselect were previously set to Expand, not Legal or
13263 // Custom. Return an empty SDValue so we fall-through to Expand, after
13264 // the Custom lowering phase.
13265 MVT VT = Op.getSimpleValueType();
13266 switch (VT.SimpleTy) {
13271 if (Subtarget->hasBWI() && Subtarget->hasVLX())
13276 // We couldn't create a "Blend with immediate" node.
13277 // This node should still be legal, but we'll have to emit a blendv*
13282 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
13283 MVT VT = Op.getSimpleValueType();
13286 if (!Op.getOperand(0).getSimpleValueType().is128BitVector())
13289 if (VT.getSizeInBits() == 8) {
13290 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
13291 Op.getOperand(0), Op.getOperand(1));
13292 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
13293 DAG.getValueType(VT));
13294 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
13297 if (VT.getSizeInBits() == 16) {
13298 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13299 // If Idx is 0, it's cheaper to do a move instead of a pextrw.
13301 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
13302 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
13303 DAG.getNode(ISD::BITCAST, dl,
13306 Op.getOperand(1)));
13307 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
13308 Op.getOperand(0), Op.getOperand(1));
13309 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
13310 DAG.getValueType(VT));
13311 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
13314 if (VT == MVT::f32) {
13315 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
13316 // the result back to FR32 register. It's only worth matching if the
13317 // result has a single use which is a store or a bitcast to i32. And in
13318 // the case of a store, it's not worth it if the index is a constant 0,
13319 // because a MOVSSmr can be used instead, which is smaller and faster.
13320 if (!Op.hasOneUse())
13322 SDNode *User = *Op.getNode()->use_begin();
13323 if ((User->getOpcode() != ISD::STORE ||
13324 (isa<ConstantSDNode>(Op.getOperand(1)) &&
13325 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) &&
13326 (User->getOpcode() != ISD::BITCAST ||
13327 User->getValueType(0) != MVT::i32))
13329 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
13330 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32,
13333 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract);
13336 if (VT == MVT::i32 || VT == MVT::i64) {
13337 // ExtractPS/pextrq works with constant index.
13338 if (isa<ConstantSDNode>(Op.getOperand(1)))
13344 /// Extract one bit from mask vector, like v16i1 or v8i1.
13345 /// AVX-512 feature.
13347 X86TargetLowering::ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG) const {
13348 SDValue Vec = Op.getOperand(0);
13350 MVT VecVT = Vec.getSimpleValueType();
13351 SDValue Idx = Op.getOperand(1);
13352 MVT EltVT = Op.getSimpleValueType();
13354 assert((EltVT == MVT::i1) && "Unexpected operands in ExtractBitFromMaskVector");
13355 assert((VecVT.getVectorNumElements() <= 16 || Subtarget->hasBWI()) &&
13356 "Unexpected vector type in ExtractBitFromMaskVector");
13358 // variable index can't be handled in mask registers,
13359 // extend vector to VR512
13360 if (!isa<ConstantSDNode>(Idx)) {
13361 MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32);
13362 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Vec);
13363 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
13364 ExtVT.getVectorElementType(), Ext, Idx);
13365 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
13368 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13369 const TargetRegisterClass* rc = getRegClassFor(VecVT);
13370 if (!Subtarget->hasDQI() && (VecVT.getVectorNumElements() <= 8))
13371 rc = getRegClassFor(MVT::v16i1);
13372 unsigned MaxSift = rc->getSize()*8 - 1;
13373 Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec,
13374 DAG.getConstant(MaxSift - IdxVal, MVT::i8));
13375 Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec,
13376 DAG.getConstant(MaxSift, MVT::i8));
13377 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec,
13378 DAG.getIntPtrConstant(0));
13382 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
13383 SelectionDAG &DAG) const {
13385 SDValue Vec = Op.getOperand(0);
13386 MVT VecVT = Vec.getSimpleValueType();
13387 SDValue Idx = Op.getOperand(1);
13389 if (Op.getSimpleValueType() == MVT::i1)
13390 return ExtractBitFromMaskVector(Op, DAG);
13392 if (!isa<ConstantSDNode>(Idx)) {
13393 if (VecVT.is512BitVector() ||
13394 (VecVT.is256BitVector() && Subtarget->hasInt256() &&
13395 VecVT.getVectorElementType().getSizeInBits() == 32)) {
13398 MVT::getIntegerVT(VecVT.getVectorElementType().getSizeInBits());
13399 MVT MaskVT = MVT::getVectorVT(MaskEltVT, VecVT.getSizeInBits() /
13400 MaskEltVT.getSizeInBits());
13402 Idx = DAG.getZExtOrTrunc(Idx, dl, MaskEltVT);
13403 SDValue Mask = DAG.getNode(X86ISD::VINSERT, dl, MaskVT,
13404 getZeroVector(MaskVT, Subtarget, DAG, dl),
13405 Idx, DAG.getConstant(0, getPointerTy()));
13406 SDValue Perm = DAG.getNode(X86ISD::VPERMV, dl, VecVT, Mask, Vec);
13407 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(),
13408 Perm, DAG.getConstant(0, getPointerTy()));
13413 // If this is a 256-bit vector result, first extract the 128-bit vector and
13414 // then extract the element from the 128-bit vector.
13415 if (VecVT.is256BitVector() || VecVT.is512BitVector()) {
13417 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13418 // Get the 128-bit vector.
13419 Vec = Extract128BitVector(Vec, IdxVal, DAG, dl);
13420 MVT EltVT = VecVT.getVectorElementType();
13422 unsigned ElemsPerChunk = 128 / EltVT.getSizeInBits();
13424 //if (IdxVal >= NumElems/2)
13425 // IdxVal -= NumElems/2;
13426 IdxVal -= (IdxVal/ElemsPerChunk)*ElemsPerChunk;
13427 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
13428 DAG.getConstant(IdxVal, MVT::i32));
13431 assert(VecVT.is128BitVector() && "Unexpected vector length");
13433 if (Subtarget->hasSSE41()) {
13434 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
13439 MVT VT = Op.getSimpleValueType();
13440 // TODO: handle v16i8.
13441 if (VT.getSizeInBits() == 16) {
13442 SDValue Vec = Op.getOperand(0);
13443 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13445 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
13446 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
13447 DAG.getNode(ISD::BITCAST, dl,
13449 Op.getOperand(1)));
13450 // Transform it so it match pextrw which produces a 32-bit result.
13451 MVT EltVT = MVT::i32;
13452 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT,
13453 Op.getOperand(0), Op.getOperand(1));
13454 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract,
13455 DAG.getValueType(VT));
13456 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
13459 if (VT.getSizeInBits() == 32) {
13460 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13464 // SHUFPS the element to the lowest double word, then movss.
13465 int Mask[4] = { static_cast<int>(Idx), -1, -1, -1 };
13466 MVT VVT = Op.getOperand(0).getSimpleValueType();
13467 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
13468 DAG.getUNDEF(VVT), Mask);
13469 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
13470 DAG.getIntPtrConstant(0));
13473 if (VT.getSizeInBits() == 64) {
13474 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
13475 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
13476 // to match extract_elt for f64.
13477 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13481 // UNPCKHPD the element to the lowest double word, then movsd.
13482 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
13483 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
13484 int Mask[2] = { 1, -1 };
13485 MVT VVT = Op.getOperand(0).getSimpleValueType();
13486 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
13487 DAG.getUNDEF(VVT), Mask);
13488 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
13489 DAG.getIntPtrConstant(0));
13495 /// Insert one bit to mask vector, like v16i1 or v8i1.
13496 /// AVX-512 feature.
13498 X86TargetLowering::InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG) const {
13500 SDValue Vec = Op.getOperand(0);
13501 SDValue Elt = Op.getOperand(1);
13502 SDValue Idx = Op.getOperand(2);
13503 MVT VecVT = Vec.getSimpleValueType();
13505 if (!isa<ConstantSDNode>(Idx)) {
13506 // Non constant index. Extend source and destination,
13507 // insert element and then truncate the result.
13508 MVT ExtVecVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32);
13509 MVT ExtEltVT = (VecVT == MVT::v8i1 ? MVT::i64 : MVT::i32);
13510 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
13511 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVecVT, Vec),
13512 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtEltVT, Elt), Idx);
13513 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
13516 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13517 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Elt);
13518 if (Vec.getOpcode() == ISD::UNDEF)
13519 return DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
13520 DAG.getConstant(IdxVal, MVT::i8));
13521 const TargetRegisterClass* rc = getRegClassFor(VecVT);
13522 unsigned MaxSift = rc->getSize()*8 - 1;
13523 EltInVec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
13524 DAG.getConstant(MaxSift, MVT::i8));
13525 EltInVec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, EltInVec,
13526 DAG.getConstant(MaxSift - IdxVal, MVT::i8));
13527 return DAG.getNode(ISD::OR, dl, VecVT, Vec, EltInVec);
13530 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
13531 SelectionDAG &DAG) const {
13532 MVT VT = Op.getSimpleValueType();
13533 MVT EltVT = VT.getVectorElementType();
13535 if (EltVT == MVT::i1)
13536 return InsertBitToMaskVector(Op, DAG);
13539 SDValue N0 = Op.getOperand(0);
13540 SDValue N1 = Op.getOperand(1);
13541 SDValue N2 = Op.getOperand(2);
13542 if (!isa<ConstantSDNode>(N2))
13544 auto *N2C = cast<ConstantSDNode>(N2);
13545 unsigned IdxVal = N2C->getZExtValue();
13547 // If the vector is wider than 128 bits, extract the 128-bit subvector, insert
13548 // into that, and then insert the subvector back into the result.
13549 if (VT.is256BitVector() || VT.is512BitVector()) {
13550 // Get the desired 128-bit vector half.
13551 SDValue V = Extract128BitVector(N0, IdxVal, DAG, dl);
13553 // Insert the element into the desired half.
13554 unsigned NumEltsIn128 = 128 / EltVT.getSizeInBits();
13555 unsigned IdxIn128 = IdxVal - (IdxVal / NumEltsIn128) * NumEltsIn128;
13557 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
13558 DAG.getConstant(IdxIn128, MVT::i32));
13560 // Insert the changed part back to the 256-bit vector
13561 return Insert128BitVector(N0, V, IdxVal, DAG, dl);
13563 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!");
13565 if (Subtarget->hasSSE41()) {
13566 if (EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) {
13568 if (VT == MVT::v8i16) {
13569 Opc = X86ISD::PINSRW;
13571 assert(VT == MVT::v16i8);
13572 Opc = X86ISD::PINSRB;
13575 // Transform it so it match pinsr{b,w} which expects a GR32 as its second
13577 if (N1.getValueType() != MVT::i32)
13578 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
13579 if (N2.getValueType() != MVT::i32)
13580 N2 = DAG.getIntPtrConstant(IdxVal);
13581 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
13584 if (EltVT == MVT::f32) {
13585 // Bits [7:6] of the constant are the source select. This will always be
13586 // zero here. The DAG Combiner may combine an extract_elt index into
13588 // bits. For example (insert (extract, 3), 2) could be matched by
13590 // the '3' into bits [7:6] of X86ISD::INSERTPS.
13591 // Bits [5:4] of the constant are the destination select. This is the
13592 // value of the incoming immediate.
13593 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
13594 // combine either bitwise AND or insert of float 0.0 to set these bits.
13595 N2 = DAG.getIntPtrConstant(IdxVal << 4);
13596 // Create this as a scalar to vector..
13597 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
13598 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
13601 if (EltVT == MVT::i32 || EltVT == MVT::i64) {
13602 // PINSR* works with constant index.
13607 if (EltVT == MVT::i8)
13610 if (EltVT.getSizeInBits() == 16) {
13611 // Transform it so it match pinsrw which expects a 16-bit value in a GR32
13612 // as its second argument.
13613 if (N1.getValueType() != MVT::i32)
13614 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
13615 if (N2.getValueType() != MVT::i32)
13616 N2 = DAG.getIntPtrConstant(IdxVal);
13617 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
13622 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
13624 MVT OpVT = Op.getSimpleValueType();
13626 // If this is a 256-bit vector result, first insert into a 128-bit
13627 // vector and then insert into the 256-bit vector.
13628 if (!OpVT.is128BitVector()) {
13629 // Insert into a 128-bit vector.
13630 unsigned SizeFactor = OpVT.getSizeInBits()/128;
13631 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
13632 OpVT.getVectorNumElements() / SizeFactor);
13634 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
13636 // Insert the 128-bit vector.
13637 return Insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
13640 if (OpVT == MVT::v1i64 &&
13641 Op.getOperand(0).getValueType() == MVT::i64)
13642 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
13644 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
13645 assert(OpVT.is128BitVector() && "Expected an SSE type!");
13646 return DAG.getNode(ISD::BITCAST, dl, OpVT,
13647 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt));
13650 // Lower a node with an EXTRACT_SUBVECTOR opcode. This may result in
13651 // a simple subregister reference or explicit instructions to grab
13652 // upper bits of a vector.
13653 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
13654 SelectionDAG &DAG) {
13656 SDValue In = Op.getOperand(0);
13657 SDValue Idx = Op.getOperand(1);
13658 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13659 MVT ResVT = Op.getSimpleValueType();
13660 MVT InVT = In.getSimpleValueType();
13662 if (Subtarget->hasFp256()) {
13663 if (ResVT.is128BitVector() &&
13664 (InVT.is256BitVector() || InVT.is512BitVector()) &&
13665 isa<ConstantSDNode>(Idx)) {
13666 return Extract128BitVector(In, IdxVal, DAG, dl);
13668 if (ResVT.is256BitVector() && InVT.is512BitVector() &&
13669 isa<ConstantSDNode>(Idx)) {
13670 return Extract256BitVector(In, IdxVal, DAG, dl);
13676 // Lower a node with an INSERT_SUBVECTOR opcode. This may result in a
13677 // simple superregister reference or explicit instructions to insert
13678 // the upper bits of a vector.
13679 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
13680 SelectionDAG &DAG) {
13681 if (!Subtarget->hasAVX())
13685 SDValue Vec = Op.getOperand(0);
13686 SDValue SubVec = Op.getOperand(1);
13687 SDValue Idx = Op.getOperand(2);
13689 if (!isa<ConstantSDNode>(Idx))
13692 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13693 MVT OpVT = Op.getSimpleValueType();
13694 MVT SubVecVT = SubVec.getSimpleValueType();
13696 // Fold two 16-byte subvector loads into one 32-byte load:
13697 // (insert_subvector (insert_subvector undef, (load addr), 0),
13698 // (load addr + 16), Elts/2)
13700 if ((IdxVal == OpVT.getVectorNumElements() / 2) &&
13701 Vec.getOpcode() == ISD::INSERT_SUBVECTOR &&
13702 OpVT.is256BitVector() && SubVecVT.is128BitVector() &&
13703 !Subtarget->isUnalignedMem32Slow()) {
13704 SDValue SubVec2 = Vec.getOperand(1);
13705 if (auto *Idx2 = dyn_cast<ConstantSDNode>(Vec.getOperand(2))) {
13706 if (Idx2->getZExtValue() == 0) {
13707 SDValue Ops[] = { SubVec2, SubVec };
13708 SDValue LD = EltsFromConsecutiveLoads(OpVT, Ops, dl, DAG, false);
13715 if ((OpVT.is256BitVector() || OpVT.is512BitVector()) &&
13716 SubVecVT.is128BitVector())
13717 return Insert128BitVector(Vec, SubVec, IdxVal, DAG, dl);
13719 if (OpVT.is512BitVector() && SubVecVT.is256BitVector())
13720 return Insert256BitVector(Vec, SubVec, IdxVal, DAG, dl);
13725 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
13726 // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
13727 // one of the above mentioned nodes. It has to be wrapped because otherwise
13728 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
13729 // be used to form addressing mode. These wrapped nodes will be selected
13732 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
13733 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
13735 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
13736 // global base reg.
13737 unsigned char OpFlag = 0;
13738 unsigned WrapperKind = X86ISD::Wrapper;
13739 CodeModel::Model M = DAG.getTarget().getCodeModel();
13741 if (Subtarget->isPICStyleRIPRel() &&
13742 (M == CodeModel::Small || M == CodeModel::Kernel))
13743 WrapperKind = X86ISD::WrapperRIP;
13744 else if (Subtarget->isPICStyleGOT())
13745 OpFlag = X86II::MO_GOTOFF;
13746 else if (Subtarget->isPICStyleStubPIC())
13747 OpFlag = X86II::MO_PIC_BASE_OFFSET;
13749 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(),
13750 CP->getAlignment(),
13751 CP->getOffset(), OpFlag);
13753 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
13754 // With PIC, the address is actually $g + Offset.
13756 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
13757 DAG.getNode(X86ISD::GlobalBaseReg,
13758 SDLoc(), getPointerTy()),
13765 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
13766 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
13768 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
13769 // global base reg.
13770 unsigned char OpFlag = 0;
13771 unsigned WrapperKind = X86ISD::Wrapper;
13772 CodeModel::Model M = DAG.getTarget().getCodeModel();
13774 if (Subtarget->isPICStyleRIPRel() &&
13775 (M == CodeModel::Small || M == CodeModel::Kernel))
13776 WrapperKind = X86ISD::WrapperRIP;
13777 else if (Subtarget->isPICStyleGOT())
13778 OpFlag = X86II::MO_GOTOFF;
13779 else if (Subtarget->isPICStyleStubPIC())
13780 OpFlag = X86II::MO_PIC_BASE_OFFSET;
13782 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(),
13785 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
13787 // With PIC, the address is actually $g + Offset.
13789 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
13790 DAG.getNode(X86ISD::GlobalBaseReg,
13791 SDLoc(), getPointerTy()),
13798 X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
13799 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
13801 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
13802 // global base reg.
13803 unsigned char OpFlag = 0;
13804 unsigned WrapperKind = X86ISD::Wrapper;
13805 CodeModel::Model M = DAG.getTarget().getCodeModel();
13807 if (Subtarget->isPICStyleRIPRel() &&
13808 (M == CodeModel::Small || M == CodeModel::Kernel)) {
13809 if (Subtarget->isTargetDarwin() || Subtarget->isTargetELF())
13810 OpFlag = X86II::MO_GOTPCREL;
13811 WrapperKind = X86ISD::WrapperRIP;
13812 } else if (Subtarget->isPICStyleGOT()) {
13813 OpFlag = X86II::MO_GOT;
13814 } else if (Subtarget->isPICStyleStubPIC()) {
13815 OpFlag = X86II::MO_DARWIN_NONLAZY_PIC_BASE;
13816 } else if (Subtarget->isPICStyleStubNoDynamic()) {
13817 OpFlag = X86II::MO_DARWIN_NONLAZY;
13820 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag);
13823 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
13825 // With PIC, the address is actually $g + Offset.
13826 if (DAG.getTarget().getRelocationModel() == Reloc::PIC_ &&
13827 !Subtarget->is64Bit()) {
13828 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
13829 DAG.getNode(X86ISD::GlobalBaseReg,
13830 SDLoc(), getPointerTy()),
13834 // For symbols that require a load from a stub to get the address, emit the
13836 if (isGlobalStubReference(OpFlag))
13837 Result = DAG.getLoad(getPointerTy(), DL, DAG.getEntryNode(), Result,
13838 MachinePointerInfo::getGOT(), false, false, false, 0);
13844 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
13845 // Create the TargetBlockAddressAddress node.
13846 unsigned char OpFlags =
13847 Subtarget->ClassifyBlockAddressReference();
13848 CodeModel::Model M = DAG.getTarget().getCodeModel();
13849 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
13850 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
13852 SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy(), Offset,
13855 if (Subtarget->isPICStyleRIPRel() &&
13856 (M == CodeModel::Small || M == CodeModel::Kernel))
13857 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
13859 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
13861 // With PIC, the address is actually $g + Offset.
13862 if (isGlobalRelativeToPICBase(OpFlags)) {
13863 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
13864 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
13872 X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, SDLoc dl,
13873 int64_t Offset, SelectionDAG &DAG) const {
13874 // Create the TargetGlobalAddress node, folding in the constant
13875 // offset if it is legal.
13876 unsigned char OpFlags =
13877 Subtarget->ClassifyGlobalReference(GV, DAG.getTarget());
13878 CodeModel::Model M = DAG.getTarget().getCodeModel();
13880 if (OpFlags == X86II::MO_NO_FLAG &&
13881 X86::isOffsetSuitableForCodeModel(Offset, M)) {
13882 // A direct static reference to a global.
13883 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset);
13886 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags);
13889 if (Subtarget->isPICStyleRIPRel() &&
13890 (M == CodeModel::Small || M == CodeModel::Kernel))
13891 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
13893 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
13895 // With PIC, the address is actually $g + Offset.
13896 if (isGlobalRelativeToPICBase(OpFlags)) {
13897 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
13898 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
13902 // For globals that require a load from a stub to get the address, emit the
13904 if (isGlobalStubReference(OpFlags))
13905 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result,
13906 MachinePointerInfo::getGOT(), false, false, false, 0);
13908 // If there was a non-zero offset that we didn't fold, create an explicit
13909 // addition for it.
13911 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result,
13912 DAG.getConstant(Offset, getPointerTy()));
13918 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
13919 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
13920 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
13921 return LowerGlobalAddress(GV, SDLoc(Op), Offset, DAG);
13925 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
13926 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
13927 unsigned char OperandFlags, bool LocalDynamic = false) {
13928 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
13929 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
13931 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
13932 GA->getValueType(0),
13936 X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
13940 SDValue Ops[] = { Chain, TGA, *InFlag };
13941 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
13943 SDValue Ops[] = { Chain, TGA };
13944 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
13947 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
13948 MFI->setAdjustsStack(true);
13949 MFI->setHasCalls(true);
13951 SDValue Flag = Chain.getValue(1);
13952 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag);
13955 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
13957 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
13960 SDLoc dl(GA); // ? function entry point might be better
13961 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
13962 DAG.getNode(X86ISD::GlobalBaseReg,
13963 SDLoc(), PtrVT), InFlag);
13964 InFlag = Chain.getValue(1);
13966 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
13969 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit
13971 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
13973 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
13974 X86::RAX, X86II::MO_TLSGD);
13977 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
13983 // Get the start address of the TLS block for this module.
13984 X86MachineFunctionInfo* MFI = DAG.getMachineFunction()
13985 .getInfo<X86MachineFunctionInfo>();
13986 MFI->incNumLocalDynamicTLSAccesses();
13990 Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, X86::RAX,
13991 X86II::MO_TLSLD, /*LocalDynamic=*/true);
13994 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
13995 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
13996 InFlag = Chain.getValue(1);
13997 Base = GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX,
13998 X86II::MO_TLSLDM, /*LocalDynamic=*/true);
14001 // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
14005 unsigned char OperandFlags = X86II::MO_DTPOFF;
14006 unsigned WrapperKind = X86ISD::Wrapper;
14007 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
14008 GA->getValueType(0),
14009 GA->getOffset(), OperandFlags);
14010 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
14012 // Add x@dtpoff with the base.
14013 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
14016 // Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
14017 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
14018 const EVT PtrVT, TLSModel::Model model,
14019 bool is64Bit, bool isPIC) {
14022 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
14023 Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(),
14024 is64Bit ? 257 : 256));
14026 SDValue ThreadPointer =
14027 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0),
14028 MachinePointerInfo(Ptr), false, false, false, 0);
14030 unsigned char OperandFlags = 0;
14031 // Most TLS accesses are not RIP relative, even on x86-64. One exception is
14033 unsigned WrapperKind = X86ISD::Wrapper;
14034 if (model == TLSModel::LocalExec) {
14035 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
14036 } else if (model == TLSModel::InitialExec) {
14038 OperandFlags = X86II::MO_GOTTPOFF;
14039 WrapperKind = X86ISD::WrapperRIP;
14041 OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
14044 llvm_unreachable("Unexpected model");
14047 // emit "addl x@ntpoff,%eax" (local exec)
14048 // or "addl x@indntpoff,%eax" (initial exec)
14049 // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
14051 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
14052 GA->getOffset(), OperandFlags);
14053 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
14055 if (model == TLSModel::InitialExec) {
14056 if (isPIC && !is64Bit) {
14057 Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
14058 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
14062 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
14063 MachinePointerInfo::getGOT(), false, false, false, 0);
14066 // The address of the thread local variable is the add of the thread
14067 // pointer with the offset of the variable.
14068 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
14072 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
14074 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
14075 const GlobalValue *GV = GA->getGlobal();
14077 if (Subtarget->isTargetELF()) {
14078 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
14081 case TLSModel::GeneralDynamic:
14082 if (Subtarget->is64Bit())
14083 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy());
14084 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy());
14085 case TLSModel::LocalDynamic:
14086 return LowerToTLSLocalDynamicModel(GA, DAG, getPointerTy(),
14087 Subtarget->is64Bit());
14088 case TLSModel::InitialExec:
14089 case TLSModel::LocalExec:
14090 return LowerToTLSExecModel(
14091 GA, DAG, getPointerTy(), model, Subtarget->is64Bit(),
14092 DAG.getTarget().getRelocationModel() == Reloc::PIC_);
14094 llvm_unreachable("Unknown TLS model.");
14097 if (Subtarget->isTargetDarwin()) {
14098 // Darwin only has one model of TLS. Lower to that.
14099 unsigned char OpFlag = 0;
14100 unsigned WrapperKind = Subtarget->isPICStyleRIPRel() ?
14101 X86ISD::WrapperRIP : X86ISD::Wrapper;
14103 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
14104 // global base reg.
14105 bool PIC32 = (DAG.getTarget().getRelocationModel() == Reloc::PIC_) &&
14106 !Subtarget->is64Bit();
14108 OpFlag = X86II::MO_TLVP_PIC_BASE;
14110 OpFlag = X86II::MO_TLVP;
14112 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
14113 GA->getValueType(0),
14114 GA->getOffset(), OpFlag);
14115 SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
14117 // With PIC32, the address is actually $g + Offset.
14119 Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(),
14120 DAG.getNode(X86ISD::GlobalBaseReg,
14121 SDLoc(), getPointerTy()),
14124 // Lowering the machine isd will make sure everything is in the right
14126 SDValue Chain = DAG.getEntryNode();
14127 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
14128 SDValue Args[] = { Chain, Offset };
14129 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
14131 // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
14132 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
14133 MFI->setAdjustsStack(true);
14135 // And our return value (tls address) is in the standard call return value
14137 unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
14138 return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(),
14139 Chain.getValue(1));
14142 if (Subtarget->isTargetKnownWindowsMSVC() ||
14143 Subtarget->isTargetWindowsGNU()) {
14144 // Just use the implicit TLS architecture
14145 // Need to generate someting similar to:
14146 // mov rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage
14148 // mov ecx, dword [rel _tls_index]: Load index (from C runtime)
14149 // mov rcx, qword [rdx+rcx*8]
14150 // mov eax, .tls$:tlsvar
14151 // [rax+rcx] contains the address
14152 // Windows 64bit: gs:0x58
14153 // Windows 32bit: fs:__tls_array
14156 SDValue Chain = DAG.getEntryNode();
14158 // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or
14159 // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly
14160 // use its literal value of 0x2C.
14161 Value *Ptr = Constant::getNullValue(Subtarget->is64Bit()
14162 ? Type::getInt8PtrTy(*DAG.getContext(),
14164 : Type::getInt32PtrTy(*DAG.getContext(),
14168 Subtarget->is64Bit()
14169 ? DAG.getIntPtrConstant(0x58)
14170 : (Subtarget->isTargetWindowsGNU()
14171 ? DAG.getIntPtrConstant(0x2C)
14172 : DAG.getExternalSymbol("_tls_array", getPointerTy()));
14174 SDValue ThreadPointer =
14175 DAG.getLoad(getPointerTy(), dl, Chain, TlsArray,
14176 MachinePointerInfo(Ptr), false, false, false, 0);
14178 // Load the _tls_index variable
14179 SDValue IDX = DAG.getExternalSymbol("_tls_index", getPointerTy());
14180 if (Subtarget->is64Bit())
14181 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, getPointerTy(), Chain,
14182 IDX, MachinePointerInfo(), MVT::i32,
14183 false, false, false, 0);
14185 IDX = DAG.getLoad(getPointerTy(), dl, Chain, IDX, MachinePointerInfo(),
14186 false, false, false, 0);
14188 SDValue Scale = DAG.getConstant(Log2_64_Ceil(TD->getPointerSize()),
14190 IDX = DAG.getNode(ISD::SHL, dl, getPointerTy(), IDX, Scale);
14192 SDValue res = DAG.getNode(ISD::ADD, dl, getPointerTy(), ThreadPointer, IDX);
14193 res = DAG.getLoad(getPointerTy(), dl, Chain, res, MachinePointerInfo(),
14194 false, false, false, 0);
14196 // Get the offset of start of .tls section
14197 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
14198 GA->getValueType(0),
14199 GA->getOffset(), X86II::MO_SECREL);
14200 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), TGA);
14202 // The address of the thread local variable is the add of the thread
14203 // pointer with the offset of the variable.
14204 return DAG.getNode(ISD::ADD, dl, getPointerTy(), res, Offset);
14207 llvm_unreachable("TLS not implemented for this target.");
14210 /// LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values
14211 /// and take a 2 x i32 value to shift plus a shift amount.
14212 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
14213 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
14214 MVT VT = Op.getSimpleValueType();
14215 unsigned VTBits = VT.getSizeInBits();
14217 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
14218 SDValue ShOpLo = Op.getOperand(0);
14219 SDValue ShOpHi = Op.getOperand(1);
14220 SDValue ShAmt = Op.getOperand(2);
14221 // X86ISD::SHLD and X86ISD::SHRD have defined overflow behavior but the
14222 // generic ISD nodes haven't. Insert an AND to be safe, it's optimized away
14224 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
14225 DAG.getConstant(VTBits - 1, MVT::i8));
14226 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
14227 DAG.getConstant(VTBits - 1, MVT::i8))
14228 : DAG.getConstant(0, VT);
14230 SDValue Tmp2, Tmp3;
14231 if (Op.getOpcode() == ISD::SHL_PARTS) {
14232 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
14233 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
14235 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
14236 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
14239 // If the shift amount is larger or equal than the width of a part we can't
14240 // rely on the results of shld/shrd. Insert a test and select the appropriate
14241 // values for large shift amounts.
14242 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
14243 DAG.getConstant(VTBits, MVT::i8));
14244 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
14245 AndNode, DAG.getConstant(0, MVT::i8));
14248 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8);
14249 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond };
14250 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond };
14252 if (Op.getOpcode() == ISD::SHL_PARTS) {
14253 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
14254 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
14256 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
14257 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
14260 SDValue Ops[2] = { Lo, Hi };
14261 return DAG.getMergeValues(Ops, dl);
14264 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
14265 SelectionDAG &DAG) const {
14266 MVT SrcVT = Op.getOperand(0).getSimpleValueType();
14269 if (SrcVT.isVector()) {
14270 if (SrcVT.getVectorElementType() == MVT::i1) {
14271 MVT IntegerVT = MVT::getVectorVT(MVT::i32, SrcVT.getVectorNumElements());
14272 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
14273 DAG.getNode(ISD::SIGN_EXTEND, dl, IntegerVT,
14274 Op.getOperand(0)));
14279 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
14280 "Unknown SINT_TO_FP to lower!");
14282 // These are really Legal; return the operand so the caller accepts it as
14284 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType()))
14286 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
14287 Subtarget->is64Bit()) {
14291 unsigned Size = SrcVT.getSizeInBits()/8;
14292 MachineFunction &MF = DAG.getMachineFunction();
14293 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size, false);
14294 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14295 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
14297 MachinePointerInfo::getFixedStack(SSFI),
14299 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
14302 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
14304 SelectionDAG &DAG) const {
14308 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
14310 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue);
14312 Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
14314 unsigned ByteSize = SrcVT.getSizeInBits()/8;
14316 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(StackSlot);
14317 MachineMemOperand *MMO;
14319 int SSFI = FI->getIndex();
14321 DAG.getMachineFunction()
14322 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14323 MachineMemOperand::MOLoad, ByteSize, ByteSize);
14325 MMO = cast<LoadSDNode>(StackSlot)->getMemOperand();
14326 StackSlot = StackSlot.getOperand(1);
14328 SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) };
14329 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
14331 Tys, Ops, SrcVT, MMO);
14334 Chain = Result.getValue(1);
14335 SDValue InFlag = Result.getValue(2);
14337 // FIXME: Currently the FST is flagged to the FILD_FLAG. This
14338 // shouldn't be necessary except that RFP cannot be live across
14339 // multiple blocks. When stackifier is fixed, they can be uncoupled.
14340 MachineFunction &MF = DAG.getMachineFunction();
14341 unsigned SSFISize = Op.getValueType().getSizeInBits()/8;
14342 int SSFI = MF.getFrameInfo()->CreateStackObject(SSFISize, SSFISize, false);
14343 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14344 Tys = DAG.getVTList(MVT::Other);
14346 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
14348 MachineMemOperand *MMO =
14349 DAG.getMachineFunction()
14350 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14351 MachineMemOperand::MOStore, SSFISize, SSFISize);
14353 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys,
14354 Ops, Op.getValueType(), MMO);
14355 Result = DAG.getLoad(Op.getValueType(), DL, Chain, StackSlot,
14356 MachinePointerInfo::getFixedStack(SSFI),
14357 false, false, false, 0);
14363 // LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion.
14364 SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
14365 SelectionDAG &DAG) const {
14366 // This algorithm is not obvious. Here it is what we're trying to output:
14369 punpckldq (c0), %xmm0 // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U }
14370 subpd (c1), %xmm0 // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 }
14372 haddpd %xmm0, %xmm0
14374 pshufd $0x4e, %xmm0, %xmm1
14380 LLVMContext *Context = DAG.getContext();
14382 // Build some magic constants.
14383 static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
14384 Constant *C0 = ConstantDataVector::get(*Context, CV0);
14385 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16);
14387 SmallVector<Constant*,2> CV1;
14389 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
14390 APInt(64, 0x4330000000000000ULL))));
14392 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
14393 APInt(64, 0x4530000000000000ULL))));
14394 Constant *C1 = ConstantVector::get(CV1);
14395 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16);
14397 // Load the 64-bit value into an XMM register.
14398 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
14400 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
14401 MachinePointerInfo::getConstantPool(),
14402 false, false, false, 16);
14403 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32,
14404 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1),
14407 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
14408 MachinePointerInfo::getConstantPool(),
14409 false, false, false, 16);
14410 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1);
14411 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
14414 if (Subtarget->hasSSE3()) {
14415 // FIXME: The 'haddpd' instruction may be slower than 'movhlps + addsd'.
14416 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
14418 SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub);
14419 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32,
14421 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64,
14422 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle),
14426 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
14427 DAG.getIntPtrConstant(0));
14430 // LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion.
14431 SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
14432 SelectionDAG &DAG) const {
14434 // FP constant to bias correct the final result.
14435 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
14438 // Load the 32-bit value into an XMM register.
14439 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
14442 // Zero out the upper parts of the register.
14443 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
14445 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
14446 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load),
14447 DAG.getIntPtrConstant(0));
14449 // Or the load with the bias.
14450 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64,
14451 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
14452 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
14453 MVT::v2f64, Load)),
14454 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
14455 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
14456 MVT::v2f64, Bias)));
14457 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
14458 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or),
14459 DAG.getIntPtrConstant(0));
14461 // Subtract the bias.
14462 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
14464 // Handle final rounding.
14465 EVT DestVT = Op.getValueType();
14467 if (DestVT.bitsLT(MVT::f64))
14468 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
14469 DAG.getIntPtrConstant(0));
14470 if (DestVT.bitsGT(MVT::f64))
14471 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
14473 // Handle final rounding.
14477 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
14478 const X86Subtarget &Subtarget) {
14479 // The algorithm is the following:
14480 // #ifdef __SSE4_1__
14481 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
14482 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
14483 // (uint4) 0x53000000, 0xaa);
14485 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
14486 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
14488 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
14489 // return (float4) lo + fhi;
14492 SDValue V = Op->getOperand(0);
14493 EVT VecIntVT = V.getValueType();
14494 bool Is128 = VecIntVT == MVT::v4i32;
14495 EVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
14496 // If we convert to something else than the supported type, e.g., to v4f64,
14498 if (VecFloatVT != Op->getValueType(0))
14501 unsigned NumElts = VecIntVT.getVectorNumElements();
14502 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
14503 "Unsupported custom type");
14504 assert(NumElts <= 8 && "The size of the constant array must be fixed");
14506 // In the #idef/#else code, we have in common:
14507 // - The vector of constants:
14513 // Create the splat vector for 0x4b000000.
14514 SDValue CstLow = DAG.getConstant(0x4b000000, MVT::i32);
14515 SDValue CstLowArray[] = {CstLow, CstLow, CstLow, CstLow,
14516 CstLow, CstLow, CstLow, CstLow};
14517 SDValue VecCstLow = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
14518 makeArrayRef(&CstLowArray[0], NumElts));
14519 // Create the splat vector for 0x53000000.
14520 SDValue CstHigh = DAG.getConstant(0x53000000, MVT::i32);
14521 SDValue CstHighArray[] = {CstHigh, CstHigh, CstHigh, CstHigh,
14522 CstHigh, CstHigh, CstHigh, CstHigh};
14523 SDValue VecCstHigh = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
14524 makeArrayRef(&CstHighArray[0], NumElts));
14526 // Create the right shift.
14527 SDValue CstShift = DAG.getConstant(16, MVT::i32);
14528 SDValue CstShiftArray[] = {CstShift, CstShift, CstShift, CstShift,
14529 CstShift, CstShift, CstShift, CstShift};
14530 SDValue VecCstShift = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
14531 makeArrayRef(&CstShiftArray[0], NumElts));
14532 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
14535 if (Subtarget.hasSSE41()) {
14536 EVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
14537 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
14538 SDValue VecCstLowBitcast =
14539 DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstLow);
14540 SDValue VecBitcast = DAG.getNode(ISD::BITCAST, DL, VecI16VT, V);
14541 // Low will be bitcasted right away, so do not bother bitcasting back to its
14543 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
14544 VecCstLowBitcast, DAG.getConstant(0xaa, MVT::i32));
14545 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
14546 // (uint4) 0x53000000, 0xaa);
14547 SDValue VecCstHighBitcast =
14548 DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstHigh);
14549 SDValue VecShiftBitcast =
14550 DAG.getNode(ISD::BITCAST, DL, VecI16VT, HighShift);
14551 // High will be bitcasted right away, so do not bother bitcasting back to
14552 // its original type.
14553 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
14554 VecCstHighBitcast, DAG.getConstant(0xaa, MVT::i32));
14556 SDValue CstMask = DAG.getConstant(0xffff, MVT::i32);
14557 SDValue VecCstMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, CstMask,
14558 CstMask, CstMask, CstMask);
14559 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
14560 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
14561 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
14563 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
14564 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
14567 // Create the vector constant for -(0x1.0p39f + 0x1.0p23f).
14568 SDValue CstFAdd = DAG.getConstantFP(
14569 APFloat(APFloat::IEEEsingle, APInt(32, 0xD3000080)), MVT::f32);
14570 SDValue CstFAddArray[] = {CstFAdd, CstFAdd, CstFAdd, CstFAdd,
14571 CstFAdd, CstFAdd, CstFAdd, CstFAdd};
14572 SDValue VecCstFAdd = DAG.getNode(ISD::BUILD_VECTOR, DL, VecFloatVT,
14573 makeArrayRef(&CstFAddArray[0], NumElts));
14575 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
14576 SDValue HighBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, High);
14578 DAG.getNode(ISD::FADD, DL, VecFloatVT, HighBitcast, VecCstFAdd);
14579 // return (float4) lo + fhi;
14580 SDValue LowBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, Low);
14581 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
14584 SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op,
14585 SelectionDAG &DAG) const {
14586 SDValue N0 = Op.getOperand(0);
14587 MVT SVT = N0.getSimpleValueType();
14590 switch (SVT.SimpleTy) {
14592 llvm_unreachable("Custom UINT_TO_FP is not supported!");
14597 MVT NVT = MVT::getVectorVT(MVT::i32, SVT.getVectorNumElements());
14598 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
14599 DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N0));
14603 return lowerUINT_TO_FP_vXi32(Op, DAG, *Subtarget);
14605 llvm_unreachable(nullptr);
14608 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
14609 SelectionDAG &DAG) const {
14610 SDValue N0 = Op.getOperand(0);
14613 if (Op.getValueType().isVector())
14614 return lowerUINT_TO_FP_vec(Op, DAG);
14616 // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
14617 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
14618 // the optimization here.
14619 if (DAG.SignBitIsZero(N0))
14620 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0);
14622 MVT SrcVT = N0.getSimpleValueType();
14623 MVT DstVT = Op.getSimpleValueType();
14624 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64)
14625 return LowerUINT_TO_FP_i64(Op, DAG);
14626 if (SrcVT == MVT::i32 && X86ScalarSSEf64)
14627 return LowerUINT_TO_FP_i32(Op, DAG);
14628 if (Subtarget->is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32)
14631 // Make a 64-bit buffer, and use it to build an FILD.
14632 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
14633 if (SrcVT == MVT::i32) {
14634 SDValue WordOff = DAG.getConstant(4, getPointerTy());
14635 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl,
14636 getPointerTy(), StackSlot, WordOff);
14637 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
14638 StackSlot, MachinePointerInfo(),
14640 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32),
14641 OffsetSlot, MachinePointerInfo(),
14643 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
14647 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");
14648 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
14649 StackSlot, MachinePointerInfo(),
14651 // For i64 source, we need to add the appropriate power of 2 if the input
14652 // was negative. This is the same as the optimization in
14653 // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here,
14654 // we must be careful to do the computation in x87 extended precision, not
14655 // in SSE. (The generic code can't know it's OK to do this, or how to.)
14656 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex();
14657 MachineMemOperand *MMO =
14658 DAG.getMachineFunction()
14659 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14660 MachineMemOperand::MOLoad, 8, 8);
14662 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
14663 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) };
14664 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
14667 APInt FF(32, 0x5F800000ULL);
14669 // Check whether the sign bit is set.
14670 SDValue SignSet = DAG.getSetCC(dl,
14671 getSetCCResultType(*DAG.getContext(), MVT::i64),
14672 Op.getOperand(0), DAG.getConstant(0, MVT::i64),
14675 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
14676 SDValue FudgePtr = DAG.getConstantPool(
14677 ConstantInt::get(*DAG.getContext(), FF.zext(64)),
14680 // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
14681 SDValue Zero = DAG.getIntPtrConstant(0);
14682 SDValue Four = DAG.getIntPtrConstant(4);
14683 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet,
14685 FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset);
14687 // Load the value out, extending it from f32 to f80.
14688 // FIXME: Avoid the extend by constructing the right constant pool?
14689 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(),
14690 FudgePtr, MachinePointerInfo::getConstantPool(),
14691 MVT::f32, false, false, false, 4);
14692 // Extend everything to 80 bits to force it to be done on x87.
14693 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
14694 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0));
14697 std::pair<SDValue,SDValue>
14698 X86TargetLowering:: FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
14699 bool IsSigned, bool IsReplace) const {
14702 EVT DstTy = Op.getValueType();
14704 if (!IsSigned && !isIntegerTypeFTOL(DstTy)) {
14705 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
14709 assert(DstTy.getSimpleVT() <= MVT::i64 &&
14710 DstTy.getSimpleVT() >= MVT::i16 &&
14711 "Unknown FP_TO_INT to lower!");
14713 // These are really Legal.
14714 if (DstTy == MVT::i32 &&
14715 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
14716 return std::make_pair(SDValue(), SDValue());
14717 if (Subtarget->is64Bit() &&
14718 DstTy == MVT::i64 &&
14719 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
14720 return std::make_pair(SDValue(), SDValue());
14722 // We lower FP->int64 either into FISTP64 followed by a load from a temporary
14723 // stack slot, or into the FTOL runtime function.
14724 MachineFunction &MF = DAG.getMachineFunction();
14725 unsigned MemSize = DstTy.getSizeInBits()/8;
14726 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
14727 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14730 if (!IsSigned && isIntegerTypeFTOL(DstTy))
14731 Opc = X86ISD::WIN_FTOL;
14733 switch (DstTy.getSimpleVT().SimpleTy) {
14734 default: llvm_unreachable("Invalid FP_TO_SINT to lower!");
14735 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
14736 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
14737 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
14740 SDValue Chain = DAG.getEntryNode();
14741 SDValue Value = Op.getOperand(0);
14742 EVT TheVT = Op.getOperand(0).getValueType();
14743 // FIXME This causes a redundant load/store if the SSE-class value is already
14744 // in memory, such as if it is on the callstack.
14745 if (isScalarFPTypeInSSEReg(TheVT)) {
14746 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
14747 Chain = DAG.getStore(Chain, DL, Value, StackSlot,
14748 MachinePointerInfo::getFixedStack(SSFI),
14750 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
14752 Chain, StackSlot, DAG.getValueType(TheVT)
14755 MachineMemOperand *MMO =
14756 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14757 MachineMemOperand::MOLoad, MemSize, MemSize);
14758 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
14759 Chain = Value.getValue(1);
14760 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
14761 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14764 MachineMemOperand *MMO =
14765 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14766 MachineMemOperand::MOStore, MemSize, MemSize);
14768 if (Opc != X86ISD::WIN_FTOL) {
14769 // Build the FP_TO_INT*_IN_MEM
14770 SDValue Ops[] = { Chain, Value, StackSlot };
14771 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other),
14773 return std::make_pair(FIST, StackSlot);
14775 SDValue ftol = DAG.getNode(X86ISD::WIN_FTOL, DL,
14776 DAG.getVTList(MVT::Other, MVT::Glue),
14778 SDValue eax = DAG.getCopyFromReg(ftol, DL, X86::EAX,
14779 MVT::i32, ftol.getValue(1));
14780 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), DL, X86::EDX,
14781 MVT::i32, eax.getValue(2));
14782 SDValue Ops[] = { eax, edx };
14783 SDValue pair = IsReplace
14784 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops)
14785 : DAG.getMergeValues(Ops, DL);
14786 return std::make_pair(pair, SDValue());
14790 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
14791 const X86Subtarget *Subtarget) {
14792 MVT VT = Op->getSimpleValueType(0);
14793 SDValue In = Op->getOperand(0);
14794 MVT InVT = In.getSimpleValueType();
14797 // Optimize vectors in AVX mode:
14800 // Use vpunpcklwd for 4 lower elements v8i16 -> v4i32.
14801 // Use vpunpckhwd for 4 upper elements v8i16 -> v4i32.
14802 // Concat upper and lower parts.
14805 // Use vpunpckldq for 4 lower elements v4i32 -> v2i64.
14806 // Use vpunpckhdq for 4 upper elements v4i32 -> v2i64.
14807 // Concat upper and lower parts.
14810 if (((VT != MVT::v16i16) || (InVT != MVT::v16i8)) &&
14811 ((VT != MVT::v8i32) || (InVT != MVT::v8i16)) &&
14812 ((VT != MVT::v4i64) || (InVT != MVT::v4i32)))
14815 if (Subtarget->hasInt256())
14816 return DAG.getNode(X86ISD::VZEXT, dl, VT, In);
14818 SDValue ZeroVec = getZeroVector(InVT, Subtarget, DAG, dl);
14819 SDValue Undef = DAG.getUNDEF(InVT);
14820 bool NeedZero = Op.getOpcode() == ISD::ZERO_EXTEND;
14821 SDValue OpLo = getUnpackl(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
14822 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
14824 MVT HVT = MVT::getVectorVT(VT.getVectorElementType(),
14825 VT.getVectorNumElements()/2);
14827 OpLo = DAG.getNode(ISD::BITCAST, dl, HVT, OpLo);
14828 OpHi = DAG.getNode(ISD::BITCAST, dl, HVT, OpHi);
14830 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
14833 static SDValue LowerZERO_EXTEND_AVX512(SDValue Op,
14834 SelectionDAG &DAG) {
14835 MVT VT = Op->getSimpleValueType(0);
14836 SDValue In = Op->getOperand(0);
14837 MVT InVT = In.getSimpleValueType();
14839 unsigned int NumElts = VT.getVectorNumElements();
14840 if (NumElts != 8 && NumElts != 16)
14843 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1)
14844 return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
14846 EVT ExtVT = (NumElts == 8)? MVT::v8i64 : MVT::v16i32;
14847 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14848 // Now we have only mask extension
14849 assert(InVT.getVectorElementType() == MVT::i1);
14850 SDValue Cst = DAG.getTargetConstant(1, ExtVT.getScalarType());
14851 const Constant *C = (dyn_cast<ConstantSDNode>(Cst))->getConstantIntValue();
14852 SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
14853 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
14854 SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
14855 MachinePointerInfo::getConstantPool(),
14856 false, false, false, Alignment);
14858 SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, DL, ExtVT, In, Ld);
14859 if (VT.is512BitVector())
14861 return DAG.getNode(X86ISD::VTRUNC, DL, VT, Brcst);
14864 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
14865 SelectionDAG &DAG) {
14866 if (Subtarget->hasFp256()) {
14867 SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
14875 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
14876 SelectionDAG &DAG) {
14878 MVT VT = Op.getSimpleValueType();
14879 SDValue In = Op.getOperand(0);
14880 MVT SVT = In.getSimpleValueType();
14882 if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1)
14883 return LowerZERO_EXTEND_AVX512(Op, DAG);
14885 if (Subtarget->hasFp256()) {
14886 SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
14891 assert(!VT.is256BitVector() || !SVT.is128BitVector() ||
14892 VT.getVectorNumElements() != SVT.getVectorNumElements());
14896 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
14898 MVT VT = Op.getSimpleValueType();
14899 SDValue In = Op.getOperand(0);
14900 MVT InVT = In.getSimpleValueType();
14902 if (VT == MVT::i1) {
14903 assert((InVT.isInteger() && (InVT.getSizeInBits() <= 64)) &&
14904 "Invalid scalar TRUNCATE operation");
14905 if (InVT.getSizeInBits() >= 32)
14907 In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In);
14908 return DAG.getNode(ISD::TRUNCATE, DL, VT, In);
14910 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
14911 "Invalid TRUNCATE operation");
14913 if (InVT.is512BitVector() || VT.getVectorElementType() == MVT::i1) {
14914 if (VT.getVectorElementType().getSizeInBits() >=8)
14915 return DAG.getNode(X86ISD::VTRUNC, DL, VT, In);
14917 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
14918 unsigned NumElts = InVT.getVectorNumElements();
14919 assert ((NumElts == 8 || NumElts == 16) && "Unexpected vector type");
14920 if (InVT.getSizeInBits() < 512) {
14921 MVT ExtVT = (NumElts == 16)? MVT::v16i32 : MVT::v8i64;
14922 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
14926 SDValue Cst = DAG.getTargetConstant(1, InVT.getVectorElementType());
14927 const Constant *C = (dyn_cast<ConstantSDNode>(Cst))->getConstantIntValue();
14928 SDValue CP = DAG.getConstantPool(C, getPointerTy());
14929 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
14930 SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
14931 MachinePointerInfo::getConstantPool(),
14932 false, false, false, Alignment);
14933 SDValue OneV = DAG.getNode(X86ISD::VBROADCAST, DL, InVT, Ld);
14934 SDValue And = DAG.getNode(ISD::AND, DL, InVT, OneV, In);
14935 return DAG.getNode(X86ISD::TESTM, DL, VT, And, And);
14938 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
14939 // On AVX2, v4i64 -> v4i32 becomes VPERMD.
14940 if (Subtarget->hasInt256()) {
14941 static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
14942 In = DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, In);
14943 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, DAG.getUNDEF(MVT::v8i32),
14945 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
14946 DAG.getIntPtrConstant(0));
14949 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
14950 DAG.getIntPtrConstant(0));
14951 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
14952 DAG.getIntPtrConstant(2));
14953 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo);
14954 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi);
14955 static const int ShufMask[] = {0, 2, 4, 6};
14956 return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);
14959 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
14960 // On AVX2, v8i32 -> v8i16 becomed PSHUFB.
14961 if (Subtarget->hasInt256()) {
14962 In = DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, In);
14964 SmallVector<SDValue,32> pshufbMask;
14965 for (unsigned i = 0; i < 2; ++i) {
14966 pshufbMask.push_back(DAG.getConstant(0x0, MVT::i8));
14967 pshufbMask.push_back(DAG.getConstant(0x1, MVT::i8));
14968 pshufbMask.push_back(DAG.getConstant(0x4, MVT::i8));
14969 pshufbMask.push_back(DAG.getConstant(0x5, MVT::i8));
14970 pshufbMask.push_back(DAG.getConstant(0x8, MVT::i8));
14971 pshufbMask.push_back(DAG.getConstant(0x9, MVT::i8));
14972 pshufbMask.push_back(DAG.getConstant(0xc, MVT::i8));
14973 pshufbMask.push_back(DAG.getConstant(0xd, MVT::i8));
14974 for (unsigned j = 0; j < 8; ++j)
14975 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
14977 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask);
14978 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV);
14979 In = DAG.getNode(ISD::BITCAST, DL, MVT::v4i64, In);
14981 static const int ShufMask[] = {0, 2, -1, -1};
14982 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, DAG.getUNDEF(MVT::v4i64),
14984 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
14985 DAG.getIntPtrConstant(0));
14986 return DAG.getNode(ISD::BITCAST, DL, VT, In);
14989 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
14990 DAG.getIntPtrConstant(0));
14992 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
14993 DAG.getIntPtrConstant(4));
14995 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpLo);
14996 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpHi);
14998 // The PSHUFB mask:
14999 static const int ShufMask1[] = {0, 1, 4, 5, 8, 9, 12, 13,
15000 -1, -1, -1, -1, -1, -1, -1, -1};
15002 SDValue Undef = DAG.getUNDEF(MVT::v16i8);
15003 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, Undef, ShufMask1);
15004 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, Undef, ShufMask1);
15006 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo);
15007 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi);
15009 // The MOVLHPS Mask:
15010 static const int ShufMask2[] = {0, 1, 4, 5};
15011 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
15012 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, res);
15015 // Handle truncation of V256 to V128 using shuffles.
15016 if (!VT.is128BitVector() || !InVT.is256BitVector())
15019 assert(Subtarget->hasFp256() && "256-bit vector without AVX!");
15021 unsigned NumElems = VT.getVectorNumElements();
15022 MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2);
15024 SmallVector<int, 16> MaskVec(NumElems * 2, -1);
15025 // Prepare truncation shuffle mask
15026 for (unsigned i = 0; i != NumElems; ++i)
15027 MaskVec[i] = i * 2;
15028 SDValue V = DAG.getVectorShuffle(NVT, DL,
15029 DAG.getNode(ISD::BITCAST, DL, NVT, In),
15030 DAG.getUNDEF(NVT), &MaskVec[0]);
15031 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V,
15032 DAG.getIntPtrConstant(0));
15035 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
15036 SelectionDAG &DAG) const {
15037 assert(!Op.getSimpleValueType().isVector());
15039 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
15040 /*IsSigned=*/ true, /*IsReplace=*/ false);
15041 SDValue FIST = Vals.first, StackSlot = Vals.second;
15042 // If FP_TO_INTHelper failed, the node is actually supposed to be Legal.
15043 if (!FIST.getNode()) return Op;
15045 if (StackSlot.getNode())
15046 // Load the result.
15047 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
15048 FIST, StackSlot, MachinePointerInfo(),
15049 false, false, false, 0);
15051 // The node is the result.
15055 SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op,
15056 SelectionDAG &DAG) const {
15057 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
15058 /*IsSigned=*/ false, /*IsReplace=*/ false);
15059 SDValue FIST = Vals.first, StackSlot = Vals.second;
15060 assert(FIST.getNode() && "Unexpected failure");
15062 if (StackSlot.getNode())
15063 // Load the result.
15064 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
15065 FIST, StackSlot, MachinePointerInfo(),
15066 false, false, false, 0);
15068 // The node is the result.
15072 static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) {
15074 MVT VT = Op.getSimpleValueType();
15075 SDValue In = Op.getOperand(0);
15076 MVT SVT = In.getSimpleValueType();
15078 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
15080 return DAG.getNode(X86ISD::VFPEXT, DL, VT,
15081 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
15082 In, DAG.getUNDEF(SVT)));
15085 /// The only differences between FABS and FNEG are the mask and the logic op.
15086 /// FNEG also has a folding opportunity for FNEG(FABS(x)).
15087 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
15088 assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&
15089 "Wrong opcode for lowering FABS or FNEG.");
15091 bool IsFABS = (Op.getOpcode() == ISD::FABS);
15093 // If this is a FABS and it has an FNEG user, bail out to fold the combination
15094 // into an FNABS. We'll lower the FABS after that if it is still in use.
15096 for (SDNode *User : Op->uses())
15097 if (User->getOpcode() == ISD::FNEG)
15100 SDValue Op0 = Op.getOperand(0);
15101 bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
15104 MVT VT = Op.getSimpleValueType();
15105 // Assume scalar op for initialization; update for vector if needed.
15106 // Note that there are no scalar bitwise logical SSE/AVX instructions, so we
15107 // generate a 16-byte vector constant and logic op even for the scalar case.
15108 // Using a 16-byte mask allows folding the load of the mask with
15109 // the logic op, so it can save (~4 bytes) on code size.
15111 unsigned NumElts = VT == MVT::f64 ? 2 : 4;
15112 // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOpt::Level to
15113 // decide if we should generate a 16-byte constant mask when we only need 4 or
15114 // 8 bytes for the scalar case.
15115 if (VT.isVector()) {
15116 EltVT = VT.getVectorElementType();
15117 NumElts = VT.getVectorNumElements();
15120 unsigned EltBits = EltVT.getSizeInBits();
15121 LLVMContext *Context = DAG.getContext();
15122 // For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
15124 IsFABS ? APInt::getSignedMaxValue(EltBits) : APInt::getSignBit(EltBits);
15125 Constant *C = ConstantInt::get(*Context, MaskElt);
15126 C = ConstantVector::getSplat(NumElts, C);
15127 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15128 SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy());
15129 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
15130 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
15131 MachinePointerInfo::getConstantPool(),
15132 false, false, false, Alignment);
15134 if (VT.isVector()) {
15135 // For a vector, cast operands to a vector type, perform the logic op,
15136 // and cast the result back to the original value type.
15137 MVT VecVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
15138 SDValue MaskCasted = DAG.getNode(ISD::BITCAST, dl, VecVT, Mask);
15139 SDValue Operand = IsFNABS ?
15140 DAG.getNode(ISD::BITCAST, dl, VecVT, Op0.getOperand(0)) :
15141 DAG.getNode(ISD::BITCAST, dl, VecVT, Op0);
15142 unsigned BitOp = IsFABS ? ISD::AND : IsFNABS ? ISD::OR : ISD::XOR;
15143 return DAG.getNode(ISD::BITCAST, dl, VT,
15144 DAG.getNode(BitOp, dl, VecVT, Operand, MaskCasted));
15147 // If not vector, then scalar.
15148 unsigned BitOp = IsFABS ? X86ISD::FAND : IsFNABS ? X86ISD::FOR : X86ISD::FXOR;
15149 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
15150 return DAG.getNode(BitOp, dl, VT, Operand, Mask);
15153 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
15154 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15155 LLVMContext *Context = DAG.getContext();
15156 SDValue Op0 = Op.getOperand(0);
15157 SDValue Op1 = Op.getOperand(1);
15159 MVT VT = Op.getSimpleValueType();
15160 MVT SrcVT = Op1.getSimpleValueType();
15162 // If second operand is smaller, extend it first.
15163 if (SrcVT.bitsLT(VT)) {
15164 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1);
15167 // And if it is bigger, shrink it first.
15168 if (SrcVT.bitsGT(VT)) {
15169 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
15173 // At this point the operands and the result should have the same
15174 // type, and that won't be f80 since that is not custom lowered.
15176 const fltSemantics &Sem =
15177 VT == MVT::f64 ? APFloat::IEEEdouble : APFloat::IEEEsingle;
15178 const unsigned SizeInBits = VT.getSizeInBits();
15180 SmallVector<Constant *, 4> CV(
15181 VT == MVT::f64 ? 2 : 4,
15182 ConstantFP::get(*Context, APFloat(Sem, APInt(SizeInBits, 0))));
15184 // First, clear all bits but the sign bit from the second operand (sign).
15185 CV[0] = ConstantFP::get(*Context,
15186 APFloat(Sem, APInt::getHighBitsSet(SizeInBits, 1)));
15187 Constant *C = ConstantVector::get(CV);
15188 SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
15189 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx,
15190 MachinePointerInfo::getConstantPool(),
15191 false, false, false, 16);
15192 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
15194 // Next, clear the sign bit from the first operand (magnitude).
15195 // If it's a constant, we can clear it here.
15196 if (ConstantFPSDNode *Op0CN = dyn_cast<ConstantFPSDNode>(Op0)) {
15197 APFloat APF = Op0CN->getValueAPF();
15198 // If the magnitude is a positive zero, the sign bit alone is enough.
15199 if (APF.isPosZero())
15202 CV[0] = ConstantFP::get(*Context, APF);
15204 CV[0] = ConstantFP::get(
15206 APFloat(Sem, APInt::getLowBitsSet(SizeInBits, SizeInBits - 1)));
15208 C = ConstantVector::get(CV);
15209 CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
15210 SDValue Val = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
15211 MachinePointerInfo::getConstantPool(),
15212 false, false, false, 16);
15213 // If the magnitude operand wasn't a constant, we need to AND out the sign.
15214 if (!isa<ConstantFPSDNode>(Op0))
15215 Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Val);
15217 // OR the magnitude value with the sign bit.
15218 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
15221 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
15222 SDValue N0 = Op.getOperand(0);
15224 MVT VT = Op.getSimpleValueType();
15226 // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1).
15227 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
15228 DAG.getConstant(1, VT));
15229 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT));
15232 // Check whether an OR'd tree is PTEST-able.
15233 static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget *Subtarget,
15234 SelectionDAG &DAG) {
15235 assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
15237 if (!Subtarget->hasSSE41())
15240 if (!Op->hasOneUse())
15243 SDNode *N = Op.getNode();
15246 SmallVector<SDValue, 8> Opnds;
15247 DenseMap<SDValue, unsigned> VecInMap;
15248 SmallVector<SDValue, 8> VecIns;
15249 EVT VT = MVT::Other;
15251 // Recognize a special case where a vector is casted into wide integer to
15253 Opnds.push_back(N->getOperand(0));
15254 Opnds.push_back(N->getOperand(1));
15256 for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
15257 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
15258 // BFS traverse all OR'd operands.
15259 if (I->getOpcode() == ISD::OR) {
15260 Opnds.push_back(I->getOperand(0));
15261 Opnds.push_back(I->getOperand(1));
15262 // Re-evaluate the number of nodes to be traversed.
15263 e += 2; // 2 more nodes (LHS and RHS) are pushed.
15267 // Quit if a non-EXTRACT_VECTOR_ELT
15268 if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
15271 // Quit if without a constant index.
15272 SDValue Idx = I->getOperand(1);
15273 if (!isa<ConstantSDNode>(Idx))
15276 SDValue ExtractedFromVec = I->getOperand(0);
15277 DenseMap<SDValue, unsigned>::iterator M = VecInMap.find(ExtractedFromVec);
15278 if (M == VecInMap.end()) {
15279 VT = ExtractedFromVec.getValueType();
15280 // Quit if not 128/256-bit vector.
15281 if (!VT.is128BitVector() && !VT.is256BitVector())
15283 // Quit if not the same type.
15284 if (VecInMap.begin() != VecInMap.end() &&
15285 VT != VecInMap.begin()->first.getValueType())
15287 M = VecInMap.insert(std::make_pair(ExtractedFromVec, 0)).first;
15288 VecIns.push_back(ExtractedFromVec);
15290 M->second |= 1U << cast<ConstantSDNode>(Idx)->getZExtValue();
15293 assert((VT.is128BitVector() || VT.is256BitVector()) &&
15294 "Not extracted from 128-/256-bit vector.");
15296 unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U;
15298 for (DenseMap<SDValue, unsigned>::const_iterator
15299 I = VecInMap.begin(), E = VecInMap.end(); I != E; ++I) {
15300 // Quit if not all elements are used.
15301 if (I->second != FullMask)
15305 EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
15307 // Cast all vectors into TestVT for PTEST.
15308 for (unsigned i = 0, e = VecIns.size(); i < e; ++i)
15309 VecIns[i] = DAG.getNode(ISD::BITCAST, DL, TestVT, VecIns[i]);
15311 // If more than one full vectors are evaluated, OR them first before PTEST.
15312 for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) {
15313 // Each iteration will OR 2 nodes and append the result until there is only
15314 // 1 node left, i.e. the final OR'd value of all vectors.
15315 SDValue LHS = VecIns[Slot];
15316 SDValue RHS = VecIns[Slot + 1];
15317 VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
15320 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
15321 VecIns.back(), VecIns.back());
15324 /// \brief return true if \c Op has a use that doesn't just read flags.
15325 static bool hasNonFlagsUse(SDValue Op) {
15326 for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
15328 SDNode *User = *UI;
15329 unsigned UOpNo = UI.getOperandNo();
15330 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
15331 // Look pass truncate.
15332 UOpNo = User->use_begin().getOperandNo();
15333 User = *User->use_begin();
15336 if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
15337 !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
15343 /// Emit nodes that will be selected as "test Op0,Op0", or something
15345 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
15346 SelectionDAG &DAG) const {
15347 if (Op.getValueType() == MVT::i1) {
15348 SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op);
15349 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp,
15350 DAG.getConstant(0, MVT::i8));
15352 // CF and OF aren't always set the way we want. Determine which
15353 // of these we need.
15354 bool NeedCF = false;
15355 bool NeedOF = false;
15358 case X86::COND_A: case X86::COND_AE:
15359 case X86::COND_B: case X86::COND_BE:
15362 case X86::COND_G: case X86::COND_GE:
15363 case X86::COND_L: case X86::COND_LE:
15364 case X86::COND_O: case X86::COND_NO: {
15365 // Check if we really need to set the
15366 // Overflow flag. If NoSignedWrap is present
15367 // that is not actually needed.
15368 switch (Op->getOpcode()) {
15373 const BinaryWithFlagsSDNode *BinNode =
15374 cast<BinaryWithFlagsSDNode>(Op.getNode());
15375 if (BinNode->hasNoSignedWrap())
15385 // See if we can use the EFLAGS value from the operand instead of
15386 // doing a separate TEST. TEST always sets OF and CF to 0, so unless
15387 // we prove that the arithmetic won't overflow, we can't use OF or CF.
15388 if (Op.getResNo() != 0 || NeedOF || NeedCF) {
15389 // Emit a CMP with 0, which is the TEST pattern.
15390 //if (Op.getValueType() == MVT::i1)
15391 // return DAG.getNode(X86ISD::CMP, dl, MVT::i1, Op,
15392 // DAG.getConstant(0, MVT::i1));
15393 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
15394 DAG.getConstant(0, Op.getValueType()));
15396 unsigned Opcode = 0;
15397 unsigned NumOperands = 0;
15399 // Truncate operations may prevent the merge of the SETCC instruction
15400 // and the arithmetic instruction before it. Attempt to truncate the operands
15401 // of the arithmetic instruction and use a reduced bit-width instruction.
15402 bool NeedTruncation = false;
15403 SDValue ArithOp = Op;
15404 if (Op->getOpcode() == ISD::TRUNCATE && Op->hasOneUse()) {
15405 SDValue Arith = Op->getOperand(0);
15406 // Both the trunc and the arithmetic op need to have one user each.
15407 if (Arith->hasOneUse())
15408 switch (Arith.getOpcode()) {
15415 NeedTruncation = true;
15421 // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
15422 // which may be the result of a CAST. We use the variable 'Op', which is the
15423 // non-casted variable when we check for possible users.
15424 switch (ArithOp.getOpcode()) {
15426 // Due to an isel shortcoming, be conservative if this add is likely to be
15427 // selected as part of a load-modify-store instruction. When the root node
15428 // in a match is a store, isel doesn't know how to remap non-chain non-flag
15429 // uses of other nodes in the match, such as the ADD in this case. This
15430 // leads to the ADD being left around and reselected, with the result being
15431 // two adds in the output. Alas, even if none our users are stores, that
15432 // doesn't prove we're O.K. Ergo, if we have any parents that aren't
15433 // CopyToReg or SETCC, eschew INC/DEC. A better fix seems to require
15434 // climbing the DAG back to the root, and it doesn't seem to be worth the
15436 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
15437 UE = Op.getNode()->use_end(); UI != UE; ++UI)
15438 if (UI->getOpcode() != ISD::CopyToReg &&
15439 UI->getOpcode() != ISD::SETCC &&
15440 UI->getOpcode() != ISD::STORE)
15443 if (ConstantSDNode *C =
15444 dyn_cast<ConstantSDNode>(ArithOp.getNode()->getOperand(1))) {
15445 // An add of one will be selected as an INC.
15446 if (C->getAPIntValue() == 1 && !Subtarget->slowIncDec()) {
15447 Opcode = X86ISD::INC;
15452 // An add of negative one (subtract of one) will be selected as a DEC.
15453 if (C->getAPIntValue().isAllOnesValue() && !Subtarget->slowIncDec()) {
15454 Opcode = X86ISD::DEC;
15460 // Otherwise use a regular EFLAGS-setting add.
15461 Opcode = X86ISD::ADD;
15466 // If we have a constant logical shift that's only used in a comparison
15467 // against zero turn it into an equivalent AND. This allows turning it into
15468 // a TEST instruction later.
15469 if ((X86CC == X86::COND_E || X86CC == X86::COND_NE) && Op->hasOneUse() &&
15470 isa<ConstantSDNode>(Op->getOperand(1)) && !hasNonFlagsUse(Op)) {
15471 EVT VT = Op.getValueType();
15472 unsigned BitWidth = VT.getSizeInBits();
15473 unsigned ShAmt = Op->getConstantOperandVal(1);
15474 if (ShAmt >= BitWidth) // Avoid undefined shifts.
15476 APInt Mask = ArithOp.getOpcode() == ISD::SRL
15477 ? APInt::getHighBitsSet(BitWidth, BitWidth - ShAmt)
15478 : APInt::getLowBitsSet(BitWidth, BitWidth - ShAmt);
15479 if (!Mask.isSignedIntN(32)) // Avoid large immediates.
15481 SDValue New = DAG.getNode(ISD::AND, dl, VT, Op->getOperand(0),
15482 DAG.getConstant(Mask, VT));
15483 DAG.ReplaceAllUsesWith(Op, New);
15489 // If the primary and result isn't used, don't bother using X86ISD::AND,
15490 // because a TEST instruction will be better.
15491 if (!hasNonFlagsUse(Op))
15497 // Due to the ISEL shortcoming noted above, be conservative if this op is
15498 // likely to be selected as part of a load-modify-store instruction.
15499 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
15500 UE = Op.getNode()->use_end(); UI != UE; ++UI)
15501 if (UI->getOpcode() == ISD::STORE)
15504 // Otherwise use a regular EFLAGS-setting instruction.
15505 switch (ArithOp.getOpcode()) {
15506 default: llvm_unreachable("unexpected operator!");
15507 case ISD::SUB: Opcode = X86ISD::SUB; break;
15508 case ISD::XOR: Opcode = X86ISD::XOR; break;
15509 case ISD::AND: Opcode = X86ISD::AND; break;
15511 if (!NeedTruncation && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
15512 SDValue EFLAGS = LowerVectorAllZeroTest(Op, Subtarget, DAG);
15513 if (EFLAGS.getNode())
15516 Opcode = X86ISD::OR;
15530 return SDValue(Op.getNode(), 1);
15536 // If we found that truncation is beneficial, perform the truncation and
15538 if (NeedTruncation) {
15539 EVT VT = Op.getValueType();
15540 SDValue WideVal = Op->getOperand(0);
15541 EVT WideVT = WideVal.getValueType();
15542 unsigned ConvertedOp = 0;
15543 // Use a target machine opcode to prevent further DAGCombine
15544 // optimizations that may separate the arithmetic operations
15545 // from the setcc node.
15546 switch (WideVal.getOpcode()) {
15548 case ISD::ADD: ConvertedOp = X86ISD::ADD; break;
15549 case ISD::SUB: ConvertedOp = X86ISD::SUB; break;
15550 case ISD::AND: ConvertedOp = X86ISD::AND; break;
15551 case ISD::OR: ConvertedOp = X86ISD::OR; break;
15552 case ISD::XOR: ConvertedOp = X86ISD::XOR; break;
15556 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15557 if (TLI.isOperationLegal(WideVal.getOpcode(), WideVT)) {
15558 SDValue V0 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(0));
15559 SDValue V1 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(1));
15560 Op = DAG.getNode(ConvertedOp, dl, VT, V0, V1);
15566 // Emit a CMP with 0, which is the TEST pattern.
15567 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
15568 DAG.getConstant(0, Op.getValueType()));
15570 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
15571 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
15573 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
15574 DAG.ReplaceAllUsesWith(Op, New);
15575 return SDValue(New.getNode(), 1);
15578 /// Emit nodes that will be selected as "cmp Op0,Op1", or something
15580 SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
15581 SDLoc dl, SelectionDAG &DAG) const {
15582 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) {
15583 if (C->getAPIntValue() == 0)
15584 return EmitTest(Op0, X86CC, dl, DAG);
15586 if (Op0.getValueType() == MVT::i1)
15587 llvm_unreachable("Unexpected comparison operation for MVT::i1 operands");
15590 if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 ||
15591 Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) {
15592 // Do the comparison at i32 if it's smaller, besides the Atom case.
15593 // This avoids subregister aliasing issues. Keep the smaller reference
15594 // if we're optimizing for size, however, as that'll allow better folding
15595 // of memory operations.
15596 if (Op0.getValueType() != MVT::i32 && Op0.getValueType() != MVT::i64 &&
15597 !DAG.getMachineFunction().getFunction()->hasFnAttribute(
15598 Attribute::MinSize) &&
15599 !Subtarget->isAtom()) {
15600 unsigned ExtendOp =
15601 isX86CCUnsigned(X86CC) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
15602 Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0);
15603 Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1);
15605 // Use SUB instead of CMP to enable CSE between SUB and CMP.
15606 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
15607 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs,
15609 return SDValue(Sub.getNode(), 1);
15611 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
15614 /// Convert a comparison if required by the subtarget.
15615 SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
15616 SelectionDAG &DAG) const {
15617 // If the subtarget does not support the FUCOMI instruction, floating-point
15618 // comparisons have to be converted.
15619 if (Subtarget->hasCMov() ||
15620 Cmp.getOpcode() != X86ISD::CMP ||
15621 !Cmp.getOperand(0).getValueType().isFloatingPoint() ||
15622 !Cmp.getOperand(1).getValueType().isFloatingPoint())
15625 // The instruction selector will select an FUCOM instruction instead of
15626 // FUCOMI, which writes the comparison result to FPSW instead of EFLAGS. Hence
15627 // build an SDNode sequence that transfers the result from FPSW into EFLAGS:
15628 // (X86sahf (trunc (srl (X86fp_stsw (trunc (X86cmp ...)), 8))))
15630 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp);
15631 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
15632 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW,
15633 DAG.getConstant(8, MVT::i8));
15634 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl);
15635 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
15638 /// The minimum architected relative accuracy is 2^-12. We need one
15639 /// Newton-Raphson step to have a good float result (24 bits of precision).
15640 SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op,
15641 DAGCombinerInfo &DCI,
15642 unsigned &RefinementSteps,
15643 bool &UseOneConstNR) const {
15644 // FIXME: We should use instruction latency models to calculate the cost of
15645 // each potential sequence, but this is very hard to do reliably because
15646 // at least Intel's Core* chips have variable timing based on the number of
15647 // significant digits in the divisor and/or sqrt operand.
15648 if (!Subtarget->useSqrtEst())
15651 EVT VT = Op.getValueType();
15653 // SSE1 has rsqrtss and rsqrtps.
15654 // TODO: Add support for AVX512 (v16f32).
15655 // It is likely not profitable to do this for f64 because a double-precision
15656 // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
15657 // instructions: convert to single, rsqrtss, convert back to double, refine
15658 // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
15659 // along with FMA, this could be a throughput win.
15660 if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||
15661 (Subtarget->hasAVX() && VT == MVT::v8f32)) {
15662 RefinementSteps = 1;
15663 UseOneConstNR = false;
15664 return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
15669 /// The minimum architected relative accuracy is 2^-12. We need one
15670 /// Newton-Raphson step to have a good float result (24 bits of precision).
15671 SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
15672 DAGCombinerInfo &DCI,
15673 unsigned &RefinementSteps) const {
15674 // FIXME: We should use instruction latency models to calculate the cost of
15675 // each potential sequence, but this is very hard to do reliably because
15676 // at least Intel's Core* chips have variable timing based on the number of
15677 // significant digits in the divisor.
15678 if (!Subtarget->useReciprocalEst())
15681 EVT VT = Op.getValueType();
15683 // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
15684 // TODO: Add support for AVX512 (v16f32).
15685 // It is likely not profitable to do this for f64 because a double-precision
15686 // reciprocal estimate with refinement on x86 prior to FMA requires
15687 // 15 instructions: convert to single, rcpss, convert back to double, refine
15688 // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
15689 // along with FMA, this could be a throughput win.
15690 if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||
15691 (Subtarget->hasAVX() && VT == MVT::v8f32)) {
15692 RefinementSteps = ReciprocalEstimateRefinementSteps;
15693 return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
15698 static bool isAllOnes(SDValue V) {
15699 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
15700 return C && C->isAllOnesValue();
15703 /// LowerToBT - Result of 'and' is compared against zero. Turn it into a BT node
15704 /// if it's possible.
15705 SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
15706 SDLoc dl, SelectionDAG &DAG) const {
15707 SDValue Op0 = And.getOperand(0);
15708 SDValue Op1 = And.getOperand(1);
15709 if (Op0.getOpcode() == ISD::TRUNCATE)
15710 Op0 = Op0.getOperand(0);
15711 if (Op1.getOpcode() == ISD::TRUNCATE)
15712 Op1 = Op1.getOperand(0);
15715 if (Op1.getOpcode() == ISD::SHL)
15716 std::swap(Op0, Op1);
15717 if (Op0.getOpcode() == ISD::SHL) {
15718 if (ConstantSDNode *And00C = dyn_cast<ConstantSDNode>(Op0.getOperand(0)))
15719 if (And00C->getZExtValue() == 1) {
15720 // If we looked past a truncate, check that it's only truncating away
15722 unsigned BitWidth = Op0.getValueSizeInBits();
15723 unsigned AndBitWidth = And.getValueSizeInBits();
15724 if (BitWidth > AndBitWidth) {
15726 DAG.computeKnownBits(Op0, Zeros, Ones);
15727 if (Zeros.countLeadingOnes() < BitWidth - AndBitWidth)
15731 RHS = Op0.getOperand(1);
15733 } else if (Op1.getOpcode() == ISD::Constant) {
15734 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
15735 uint64_t AndRHSVal = AndRHS->getZExtValue();
15736 SDValue AndLHS = Op0;
15738 if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
15739 LHS = AndLHS.getOperand(0);
15740 RHS = AndLHS.getOperand(1);
15743 // Use BT if the immediate can't be encoded in a TEST instruction.
15744 if (!isUInt<32>(AndRHSVal) && isPowerOf2_64(AndRHSVal)) {
15746 RHS = DAG.getConstant(Log2_64_Ceil(AndRHSVal), LHS.getValueType());
15750 if (LHS.getNode()) {
15751 // If LHS is i8, promote it to i32 with any_extend. There is no i8 BT
15752 // instruction. Since the shift amount is in-range-or-undefined, we know
15753 // that doing a bittest on the i32 value is ok. We extend to i32 because
15754 // the encoding for the i16 version is larger than the i32 version.
15755 // Also promote i16 to i32 for performance / code size reason.
15756 if (LHS.getValueType() == MVT::i8 ||
15757 LHS.getValueType() == MVT::i16)
15758 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
15760 // If the operand types disagree, extend the shift amount to match. Since
15761 // BT ignores high bits (like shifts) we can use anyextend.
15762 if (LHS.getValueType() != RHS.getValueType())
15763 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS);
15765 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
15766 X86::CondCode Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
15767 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
15768 DAG.getConstant(Cond, MVT::i8), BT);
15774 /// \brief - Turns an ISD::CondCode into a value suitable for SSE floating point
15776 static int translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
15781 // SSE Condition code mapping:
15790 switch (SetCCOpcode) {
15791 default: llvm_unreachable("Unexpected SETCC condition");
15793 case ISD::SETEQ: SSECC = 0; break;
15795 case ISD::SETGT: Swap = true; // Fallthrough
15797 case ISD::SETOLT: SSECC = 1; break;
15799 case ISD::SETGE: Swap = true; // Fallthrough
15801 case ISD::SETOLE: SSECC = 2; break;
15802 case ISD::SETUO: SSECC = 3; break;
15804 case ISD::SETNE: SSECC = 4; break;
15805 case ISD::SETULE: Swap = true; // Fallthrough
15806 case ISD::SETUGE: SSECC = 5; break;
15807 case ISD::SETULT: Swap = true; // Fallthrough
15808 case ISD::SETUGT: SSECC = 6; break;
15809 case ISD::SETO: SSECC = 7; break;
15811 case ISD::SETONE: SSECC = 8; break;
15814 std::swap(Op0, Op1);
15819 // Lower256IntVSETCC - Break a VSETCC 256-bit integer VSETCC into two new 128
15820 // ones, and then concatenate the result back.
15821 static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
15822 MVT VT = Op.getSimpleValueType();
15824 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&
15825 "Unsupported value type for operation");
15827 unsigned NumElems = VT.getVectorNumElements();
15829 SDValue CC = Op.getOperand(2);
15831 // Extract the LHS vectors
15832 SDValue LHS = Op.getOperand(0);
15833 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
15834 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
15836 // Extract the RHS vectors
15837 SDValue RHS = Op.getOperand(1);
15838 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl);
15839 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl);
15841 // Issue the operation on the smaller types and concatenate the result back
15842 MVT EltVT = VT.getVectorElementType();
15843 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
15844 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
15845 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
15846 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
15849 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG,
15850 const X86Subtarget *Subtarget) {
15851 SDValue Op0 = Op.getOperand(0);
15852 SDValue Op1 = Op.getOperand(1);
15853 SDValue CC = Op.getOperand(2);
15854 MVT VT = Op.getSimpleValueType();
15857 assert(Op0.getValueType().getVectorElementType().getSizeInBits() >= 8 &&
15858 Op.getValueType().getScalarType() == MVT::i1 &&
15859 "Cannot set masked compare for this operation");
15861 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
15863 bool Unsigned = false;
15866 switch (SetCCOpcode) {
15867 default: llvm_unreachable("Unexpected SETCC condition");
15868 case ISD::SETNE: SSECC = 4; break;
15869 case ISD::SETEQ: Opc = X86ISD::PCMPEQM; break;
15870 case ISD::SETUGT: SSECC = 6; Unsigned = true; break;
15871 case ISD::SETLT: Swap = true; //fall-through
15872 case ISD::SETGT: Opc = X86ISD::PCMPGTM; break;
15873 case ISD::SETULT: SSECC = 1; Unsigned = true; break;
15874 case ISD::SETUGE: SSECC = 5; Unsigned = true; break; //NLT
15875 case ISD::SETGE: Swap = true; SSECC = 2; break; // LE + swap
15876 case ISD::SETULE: Unsigned = true; //fall-through
15877 case ISD::SETLE: SSECC = 2; break;
15881 std::swap(Op0, Op1);
15883 return DAG.getNode(Opc, dl, VT, Op0, Op1);
15884 Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM;
15885 return DAG.getNode(Opc, dl, VT, Op0, Op1,
15886 DAG.getConstant(SSECC, MVT::i8));
15889 /// \brief Try to turn a VSETULT into a VSETULE by modifying its second
15890 /// operand \p Op1. If non-trivial (for example because it's not constant)
15891 /// return an empty value.
15892 static SDValue ChangeVSETULTtoVSETULE(SDLoc dl, SDValue Op1, SelectionDAG &DAG)
15894 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op1.getNode());
15898 MVT VT = Op1.getSimpleValueType();
15899 MVT EVT = VT.getVectorElementType();
15900 unsigned n = VT.getVectorNumElements();
15901 SmallVector<SDValue, 8> ULTOp1;
15903 for (unsigned i = 0; i < n; ++i) {
15904 ConstantSDNode *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
15905 if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT)
15908 // Avoid underflow.
15909 APInt Val = Elt->getAPIntValue();
15913 ULTOp1.push_back(DAG.getConstant(Val - 1, EVT));
15916 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, ULTOp1);
15919 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
15920 SelectionDAG &DAG) {
15921 SDValue Op0 = Op.getOperand(0);
15922 SDValue Op1 = Op.getOperand(1);
15923 SDValue CC = Op.getOperand(2);
15924 MVT VT = Op.getSimpleValueType();
15925 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
15926 bool isFP = Op.getOperand(1).getSimpleValueType().isFloatingPoint();
15931 MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
15932 assert(EltVT == MVT::f32 || EltVT == MVT::f64);
15935 unsigned SSECC = translateX86FSETCC(SetCCOpcode, Op0, Op1);
15936 unsigned Opc = X86ISD::CMPP;
15937 if (Subtarget->hasAVX512() && VT.getVectorElementType() == MVT::i1) {
15938 assert(VT.getVectorNumElements() <= 16);
15939 Opc = X86ISD::CMPM;
15941 // In the two special cases we can't handle, emit two comparisons.
15944 unsigned CombineOpc;
15945 if (SetCCOpcode == ISD::SETUEQ) {
15946 CC0 = 3; CC1 = 0; CombineOpc = ISD::OR;
15948 assert(SetCCOpcode == ISD::SETONE);
15949 CC0 = 7; CC1 = 4; CombineOpc = ISD::AND;
15952 SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
15953 DAG.getConstant(CC0, MVT::i8));
15954 SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
15955 DAG.getConstant(CC1, MVT::i8));
15956 return DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
15958 // Handle all other FP comparisons here.
15959 return DAG.getNode(Opc, dl, VT, Op0, Op1,
15960 DAG.getConstant(SSECC, MVT::i8));
15963 // Break 256-bit integer vector compare into smaller ones.
15964 if (VT.is256BitVector() && !Subtarget->hasInt256())
15965 return Lower256IntVSETCC(Op, DAG);
15967 bool MaskResult = (VT.getVectorElementType() == MVT::i1);
15968 EVT OpVT = Op1.getValueType();
15969 if (Subtarget->hasAVX512()) {
15970 if (Op1.getValueType().is512BitVector() ||
15971 (Subtarget->hasBWI() && Subtarget->hasVLX()) ||
15972 (MaskResult && OpVT.getVectorElementType().getSizeInBits() >= 32))
15973 return LowerIntVSETCC_AVX512(Op, DAG, Subtarget);
15975 // In AVX-512 architecture setcc returns mask with i1 elements,
15976 // But there is no compare instruction for i8 and i16 elements in KNL.
15977 // We are not talking about 512-bit operands in this case, these
15978 // types are illegal.
15980 (OpVT.getVectorElementType().getSizeInBits() < 32 &&
15981 OpVT.getVectorElementType().getSizeInBits() >= 8))
15982 return DAG.getNode(ISD::TRUNCATE, dl, VT,
15983 DAG.getNode(ISD::SETCC, dl, OpVT, Op0, Op1, CC));
15986 // We are handling one of the integer comparisons here. Since SSE only has
15987 // GT and EQ comparisons for integer, swapping operands and multiple
15988 // operations may be required for some comparisons.
15990 bool Swap = false, Invert = false, FlipSigns = false, MinMax = false;
15991 bool Subus = false;
15993 switch (SetCCOpcode) {
15994 default: llvm_unreachable("Unexpected SETCC condition");
15995 case ISD::SETNE: Invert = true;
15996 case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break;
15997 case ISD::SETLT: Swap = true;
15998 case ISD::SETGT: Opc = X86ISD::PCMPGT; break;
15999 case ISD::SETGE: Swap = true;
16000 case ISD::SETLE: Opc = X86ISD::PCMPGT;
16001 Invert = true; break;
16002 case ISD::SETULT: Swap = true;
16003 case ISD::SETUGT: Opc = X86ISD::PCMPGT;
16004 FlipSigns = true; break;
16005 case ISD::SETUGE: Swap = true;
16006 case ISD::SETULE: Opc = X86ISD::PCMPGT;
16007 FlipSigns = true; Invert = true; break;
16010 // Special case: Use min/max operations for SETULE/SETUGE
16011 MVT VET = VT.getVectorElementType();
16013 (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32))
16014 || (Subtarget->hasSSE2() && (VET == MVT::i8));
16017 switch (SetCCOpcode) {
16019 case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break;
16020 case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break;
16023 if (MinMax) { Swap = false; Invert = false; FlipSigns = false; }
16026 bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16);
16027 if (!MinMax && hasSubus) {
16028 // As another special case, use PSUBUS[BW] when it's profitable. E.g. for
16030 // t = psubus Op0, Op1
16031 // pcmpeq t, <0..0>
16032 switch (SetCCOpcode) {
16034 case ISD::SETULT: {
16035 // If the comparison is against a constant we can turn this into a
16036 // setule. With psubus, setule does not require a swap. This is
16037 // beneficial because the constant in the register is no longer
16038 // destructed as the destination so it can be hoisted out of a loop.
16039 // Only do this pre-AVX since vpcmp* is no longer destructive.
16040 if (Subtarget->hasAVX())
16042 SDValue ULEOp1 = ChangeVSETULTtoVSETULE(dl, Op1, DAG);
16043 if (ULEOp1.getNode()) {
16045 Subus = true; Invert = false; Swap = false;
16049 // Psubus is better than flip-sign because it requires no inversion.
16050 case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break;
16051 case ISD::SETULE: Subus = true; Invert = false; Swap = false; break;
16055 Opc = X86ISD::SUBUS;
16061 std::swap(Op0, Op1);
16063 // Check that the operation in question is available (most are plain SSE2,
16064 // but PCMPGTQ and PCMPEQQ have different requirements).
16065 if (VT == MVT::v2i64) {
16066 if (Opc == X86ISD::PCMPGT && !Subtarget->hasSSE42()) {
16067 assert(Subtarget->hasSSE2() && "Don't know how to lower!");
16069 // First cast everything to the right type.
16070 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
16071 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
16073 // Since SSE has no unsigned integer comparisons, we need to flip the sign
16074 // bits of the inputs before performing those operations. The lower
16075 // compare is always unsigned.
16078 SB = DAG.getConstant(0x80000000U, MVT::v4i32);
16080 SDValue Sign = DAG.getConstant(0x80000000U, MVT::i32);
16081 SDValue Zero = DAG.getConstant(0x00000000U, MVT::i32);
16082 SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
16083 Sign, Zero, Sign, Zero);
16085 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB);
16086 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB);
16088 // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
16089 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
16090 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
16092 // Create masks for only the low parts/high parts of the 64 bit integers.
16093 static const int MaskHi[] = { 1, 1, 3, 3 };
16094 static const int MaskLo[] = { 0, 0, 2, 2 };
16095 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
16096 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
16097 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
16099 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
16100 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
16103 Result = DAG.getNOT(dl, Result, MVT::v4i32);
16105 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
16108 if (Opc == X86ISD::PCMPEQ && !Subtarget->hasSSE41()) {
16109 // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
16110 // pcmpeqd + pshufd + pand.
16111 assert(Subtarget->hasSSE2() && !FlipSigns && "Don't know how to lower!");
16113 // First cast everything to the right type.
16114 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
16115 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
16118 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
16120 // Make sure the lower and upper halves are both all-ones.
16121 static const int Mask[] = { 1, 0, 3, 2 };
16122 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
16123 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
16126 Result = DAG.getNOT(dl, Result, MVT::v4i32);
16128 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
16132 // Since SSE has no unsigned integer comparisons, we need to flip the sign
16133 // bits of the inputs before performing those operations.
16135 EVT EltVT = VT.getVectorElementType();
16136 SDValue SB = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), VT);
16137 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SB);
16138 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SB);
16141 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
16143 // If the logical-not of the result is required, perform that now.
16145 Result = DAG.getNOT(dl, Result, VT);
16148 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
16151 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
16152 getZeroVector(VT, Subtarget, DAG, dl));
16157 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
16159 MVT VT = Op.getSimpleValueType();
16161 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
16163 assert(((!Subtarget->hasAVX512() && VT == MVT::i8) || (VT == MVT::i1))
16164 && "SetCC type must be 8-bit or 1-bit integer");
16165 SDValue Op0 = Op.getOperand(0);
16166 SDValue Op1 = Op.getOperand(1);
16168 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
16170 // Optimize to BT if possible.
16171 // Lower (X & (1 << N)) == 0 to BT(X, N).
16172 // Lower ((X >>u N) & 1) != 0 to BT(X, N).
16173 // Lower ((X >>s N) & 1) != 0 to BT(X, N).
16174 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() &&
16175 Op1.getOpcode() == ISD::Constant &&
16176 cast<ConstantSDNode>(Op1)->isNullValue() &&
16177 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
16178 SDValue NewSetCC = LowerToBT(Op0, CC, dl, DAG);
16179 if (NewSetCC.getNode()) {
16181 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewSetCC);
16186 // Look for X == 0, X == 1, X != 0, or X != 1. We can simplify some forms of
16188 if (Op1.getOpcode() == ISD::Constant &&
16189 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1 ||
16190 cast<ConstantSDNode>(Op1)->isNullValue()) &&
16191 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
16193 // If the input is a setcc, then reuse the input setcc or use a new one with
16194 // the inverted condition.
16195 if (Op0.getOpcode() == X86ISD::SETCC) {
16196 X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
16197 bool Invert = (CC == ISD::SETNE) ^
16198 cast<ConstantSDNode>(Op1)->isNullValue();
16202 CCode = X86::GetOppositeBranchCondition(CCode);
16203 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
16204 DAG.getConstant(CCode, MVT::i8),
16205 Op0.getOperand(1));
16207 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC);
16211 if ((Op0.getValueType() == MVT::i1) && (Op1.getOpcode() == ISD::Constant) &&
16212 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1) &&
16213 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
16215 ISD::CondCode NewCC = ISD::getSetCCInverse(CC, true);
16216 return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, MVT::i1), NewCC);
16219 bool isFP = Op1.getSimpleValueType().isFloatingPoint();
16220 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG);
16221 if (X86CC == X86::COND_INVALID)
16224 SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, dl, DAG);
16225 EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
16226 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
16227 DAG.getConstant(X86CC, MVT::i8), EFLAGS);
16229 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC);
16233 // isX86LogicalCmp - Return true if opcode is a X86 logical comparison.
16234 static bool isX86LogicalCmp(SDValue Op) {
16235 unsigned Opc = Op.getNode()->getOpcode();
16236 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
16237 Opc == X86ISD::SAHF)
16239 if (Op.getResNo() == 1 &&
16240 (Opc == X86ISD::ADD ||
16241 Opc == X86ISD::SUB ||
16242 Opc == X86ISD::ADC ||
16243 Opc == X86ISD::SBB ||
16244 Opc == X86ISD::SMUL ||
16245 Opc == X86ISD::UMUL ||
16246 Opc == X86ISD::INC ||
16247 Opc == X86ISD::DEC ||
16248 Opc == X86ISD::OR ||
16249 Opc == X86ISD::XOR ||
16250 Opc == X86ISD::AND))
16253 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
16259 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
16260 if (V.getOpcode() != ISD::TRUNCATE)
16263 SDValue VOp0 = V.getOperand(0);
16264 unsigned InBits = VOp0.getValueSizeInBits();
16265 unsigned Bits = V.getValueSizeInBits();
16266 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
16269 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
16270 bool addTest = true;
16271 SDValue Cond = Op.getOperand(0);
16272 SDValue Op1 = Op.getOperand(1);
16273 SDValue Op2 = Op.getOperand(2);
16275 EVT VT = Op1.getValueType();
16278 // Lower fp selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
16279 // are available. Otherwise fp cmovs get lowered into a less efficient branch
16280 // sequence later on.
16281 if (Cond.getOpcode() == ISD::SETCC &&
16282 ((Subtarget->hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) ||
16283 (Subtarget->hasSSE1() && VT == MVT::f32)) &&
16284 VT == Cond.getOperand(0).getValueType() && Cond->hasOneUse()) {
16285 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
16286 int SSECC = translateX86FSETCC(
16287 cast<CondCodeSDNode>(Cond.getOperand(2))->get(), CondOp0, CondOp1);
16290 if (Subtarget->hasAVX512()) {
16291 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1,
16292 DAG.getConstant(SSECC, MVT::i8));
16293 return DAG.getNode(X86ISD::SELECT, DL, VT, Cmp, Op1, Op2);
16295 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
16296 DAG.getConstant(SSECC, MVT::i8));
16297 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
16298 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
16299 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
16303 if (Cond.getOpcode() == ISD::SETCC) {
16304 SDValue NewCond = LowerSETCC(Cond, DAG);
16305 if (NewCond.getNode())
16309 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y
16310 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y
16311 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y
16312 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y
16313 if (Cond.getOpcode() == X86ISD::SETCC &&
16314 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
16315 isZero(Cond.getOperand(1).getOperand(1))) {
16316 SDValue Cmp = Cond.getOperand(1);
16318 unsigned CondCode =cast<ConstantSDNode>(Cond.getOperand(0))->getZExtValue();
16320 if ((isAllOnes(Op1) || isAllOnes(Op2)) &&
16321 (CondCode == X86::COND_E || CondCode == X86::COND_NE)) {
16322 SDValue Y = isAllOnes(Op2) ? Op1 : Op2;
16324 SDValue CmpOp0 = Cmp.getOperand(0);
16325 // Apply further optimizations for special cases
16326 // (select (x != 0), -1, 0) -> neg & sbb
16327 // (select (x == 0), 0, -1) -> neg & sbb
16328 if (ConstantSDNode *YC = dyn_cast<ConstantSDNode>(Y))
16329 if (YC->isNullValue() &&
16330 (isAllOnes(Op1) == (CondCode == X86::COND_NE))) {
16331 SDVTList VTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);
16332 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, VTs,
16333 DAG.getConstant(0, CmpOp0.getValueType()),
16335 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
16336 DAG.getConstant(X86::COND_B, MVT::i8),
16337 SDValue(Neg.getNode(), 1));
16341 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
16342 CmpOp0, DAG.getConstant(1, CmpOp0.getValueType()));
16343 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
16345 SDValue Res = // Res = 0 or -1.
16346 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
16347 DAG.getConstant(X86::COND_B, MVT::i8), Cmp);
16349 if (isAllOnes(Op1) != (CondCode == X86::COND_E))
16350 Res = DAG.getNOT(DL, Res, Res.getValueType());
16352 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Op2);
16353 if (!N2C || !N2C->isNullValue())
16354 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y);
16359 // Look past (and (setcc_carry (cmp ...)), 1).
16360 if (Cond.getOpcode() == ISD::AND &&
16361 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
16362 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
16363 if (C && C->getAPIntValue() == 1)
16364 Cond = Cond.getOperand(0);
16367 // If condition flag is set by a X86ISD::CMP, then use it as the condition
16368 // setting operand in place of the X86ISD::SETCC.
16369 unsigned CondOpcode = Cond.getOpcode();
16370 if (CondOpcode == X86ISD::SETCC ||
16371 CondOpcode == X86ISD::SETCC_CARRY) {
16372 CC = Cond.getOperand(0);
16374 SDValue Cmp = Cond.getOperand(1);
16375 unsigned Opc = Cmp.getOpcode();
16376 MVT VT = Op.getSimpleValueType();
16378 bool IllegalFPCMov = false;
16379 if (VT.isFloatingPoint() && !VT.isVector() &&
16380 !isScalarFPTypeInSSEReg(VT)) // FPStack?
16381 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
16383 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
16384 Opc == X86ISD::BT) { // FIXME
16388 } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
16389 CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
16390 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) &&
16391 Cond.getOperand(0).getValueType() != MVT::i8)) {
16392 SDValue LHS = Cond.getOperand(0);
16393 SDValue RHS = Cond.getOperand(1);
16394 unsigned X86Opcode;
16397 switch (CondOpcode) {
16398 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
16399 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
16400 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
16401 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
16402 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
16403 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
16404 default: llvm_unreachable("unexpected overflowing operator");
16406 if (CondOpcode == ISD::UMULO)
16407 VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(),
16410 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
16412 SDValue X86Op = DAG.getNode(X86Opcode, DL, VTs, LHS, RHS);
16414 if (CondOpcode == ISD::UMULO)
16415 Cond = X86Op.getValue(2);
16417 Cond = X86Op.getValue(1);
16419 CC = DAG.getConstant(X86Cond, MVT::i8);
16424 // Look pass the truncate if the high bits are known zero.
16425 if (isTruncWithZeroHighBitsInput(Cond, DAG))
16426 Cond = Cond.getOperand(0);
16428 // We know the result of AND is compared against zero. Try to match
16430 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
16431 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, DL, DAG);
16432 if (NewSetCC.getNode()) {
16433 CC = NewSetCC.getOperand(0);
16434 Cond = NewSetCC.getOperand(1);
16441 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
16442 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG);
16445 // a < b ? -1 : 0 -> RES = ~setcc_carry
16446 // a < b ? 0 : -1 -> RES = setcc_carry
16447 // a >= b ? -1 : 0 -> RES = setcc_carry
16448 // a >= b ? 0 : -1 -> RES = ~setcc_carry
16449 if (Cond.getOpcode() == X86ISD::SUB) {
16450 Cond = ConvertCmpIfNecessary(Cond, DAG);
16451 unsigned CondCode = cast<ConstantSDNode>(CC)->getZExtValue();
16453 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
16454 (isAllOnes(Op1) || isAllOnes(Op2)) && (isZero(Op1) || isZero(Op2))) {
16455 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
16456 DAG.getConstant(X86::COND_B, MVT::i8), Cond);
16457 if (isAllOnes(Op1) != (CondCode == X86::COND_B))
16458 return DAG.getNOT(DL, Res, Res.getValueType());
16463 // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
16464 // widen the cmov and push the truncate through. This avoids introducing a new
16465 // branch during isel and doesn't add any extensions.
16466 if (Op.getValueType() == MVT::i8 &&
16467 Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
16468 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
16469 if (T1.getValueType() == T2.getValueType() &&
16470 // Blacklist CopyFromReg to avoid partial register stalls.
16471 T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
16472 SDVTList VTs = DAG.getVTList(T1.getValueType(), MVT::Glue);
16473 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VTs, T2, T1, CC, Cond);
16474 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
16478 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
16479 // condition is true.
16480 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
16481 SDValue Ops[] = { Op2, Op1, CC, Cond };
16482 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops);
16485 static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op, const X86Subtarget *Subtarget,
16486 SelectionDAG &DAG) {
16487 MVT VT = Op->getSimpleValueType(0);
16488 SDValue In = Op->getOperand(0);
16489 MVT InVT = In.getSimpleValueType();
16490 MVT VTElt = VT.getVectorElementType();
16491 MVT InVTElt = InVT.getVectorElementType();
16495 if ((InVTElt == MVT::i1) &&
16496 (((Subtarget->hasBWI() && Subtarget->hasVLX() &&
16497 VT.getSizeInBits() <= 256 && VTElt.getSizeInBits() <= 16)) ||
16499 ((Subtarget->hasBWI() && VT.is512BitVector() &&
16500 VTElt.getSizeInBits() <= 16)) ||
16502 ((Subtarget->hasDQI() && Subtarget->hasVLX() &&
16503 VT.getSizeInBits() <= 256 && VTElt.getSizeInBits() >= 32)) ||
16505 ((Subtarget->hasDQI() && VT.is512BitVector() &&
16506 VTElt.getSizeInBits() >= 32))))
16507 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16509 unsigned int NumElts = VT.getVectorNumElements();
16511 if (NumElts != 8 && NumElts != 16)
16514 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) {
16515 if (In.getOpcode() == X86ISD::VSEXT || In.getOpcode() == X86ISD::VZEXT)
16516 return DAG.getNode(In.getOpcode(), dl, VT, In.getOperand(0));
16517 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16520 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16521 assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
16523 MVT ExtVT = (NumElts == 8) ? MVT::v8i64 : MVT::v16i32;
16524 Constant *C = ConstantInt::get(*DAG.getContext(),
16525 APInt::getAllOnesValue(ExtVT.getScalarType().getSizeInBits()));
16527 SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
16528 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
16529 SDValue Ld = DAG.getLoad(ExtVT.getScalarType(), dl, DAG.getEntryNode(), CP,
16530 MachinePointerInfo::getConstantPool(),
16531 false, false, false, Alignment);
16532 SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, dl, ExtVT, In, Ld);
16533 if (VT.is512BitVector())
16535 return DAG.getNode(X86ISD::VTRUNC, dl, VT, Brcst);
16538 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
16539 SelectionDAG &DAG) {
16540 MVT VT = Op->getSimpleValueType(0);
16541 SDValue In = Op->getOperand(0);
16542 MVT InVT = In.getSimpleValueType();
16545 if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1)
16546 return LowerSIGN_EXTEND_AVX512(Op, Subtarget, DAG);
16548 if ((VT != MVT::v4i64 || InVT != MVT::v4i32) &&
16549 (VT != MVT::v8i32 || InVT != MVT::v8i16) &&
16550 (VT != MVT::v16i16 || InVT != MVT::v16i8))
16553 if (Subtarget->hasInt256())
16554 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16556 // Optimize vectors in AVX mode
16557 // Sign extend v8i16 to v8i32 and
16560 // Divide input vector into two parts
16561 // for v4i32 the shuffle mask will be { 0, 1, -1, -1} {2, 3, -1, -1}
16562 // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
16563 // concat the vectors to original VT
16565 unsigned NumElems = InVT.getVectorNumElements();
16566 SDValue Undef = DAG.getUNDEF(InVT);
16568 SmallVector<int,8> ShufMask1(NumElems, -1);
16569 for (unsigned i = 0; i != NumElems/2; ++i)
16572 SDValue OpLo = DAG.getVectorShuffle(InVT, dl, In, Undef, &ShufMask1[0]);
16574 SmallVector<int,8> ShufMask2(NumElems, -1);
16575 for (unsigned i = 0; i != NumElems/2; ++i)
16576 ShufMask2[i] = i + NumElems/2;
16578 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, Undef, &ShufMask2[0]);
16580 MVT HalfVT = MVT::getVectorVT(VT.getScalarType(),
16581 VT.getVectorNumElements()/2);
16583 OpLo = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpLo);
16584 OpHi = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpHi);
16586 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
16589 // Lower vector extended loads using a shuffle. If SSSE3 is not available we
16590 // may emit an illegal shuffle but the expansion is still better than scalar
16591 // code. We generate X86ISD::VSEXT for SEXTLOADs if it's available, otherwise
16592 // we'll emit a shuffle and a arithmetic shift.
16593 // FIXME: Is the expansion actually better than scalar code? It doesn't seem so.
16594 // TODO: It is possible to support ZExt by zeroing the undef values during
16595 // the shuffle phase or after the shuffle.
16596 static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
16597 SelectionDAG &DAG) {
16598 MVT RegVT = Op.getSimpleValueType();
16599 assert(RegVT.isVector() && "We only custom lower vector sext loads.");
16600 assert(RegVT.isInteger() &&
16601 "We only custom lower integer vector sext loads.");
16603 // Nothing useful we can do without SSE2 shuffles.
16604 assert(Subtarget->hasSSE2() && "We only custom lower sext loads with SSE2.");
16606 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
16608 EVT MemVT = Ld->getMemoryVT();
16609 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16610 unsigned RegSz = RegVT.getSizeInBits();
16612 ISD::LoadExtType Ext = Ld->getExtensionType();
16614 assert((Ext == ISD::EXTLOAD || Ext == ISD::SEXTLOAD)
16615 && "Only anyext and sext are currently implemented.");
16616 assert(MemVT != RegVT && "Cannot extend to the same type");
16617 assert(MemVT.isVector() && "Must load a vector from memory");
16619 unsigned NumElems = RegVT.getVectorNumElements();
16620 unsigned MemSz = MemVT.getSizeInBits();
16621 assert(RegSz > MemSz && "Register size must be greater than the mem size");
16623 if (Ext == ISD::SEXTLOAD && RegSz == 256 && !Subtarget->hasInt256()) {
16624 // The only way in which we have a legal 256-bit vector result but not the
16625 // integer 256-bit operations needed to directly lower a sextload is if we
16626 // have AVX1 but not AVX2. In that case, we can always emit a sextload to
16627 // a 128-bit vector and a normal sign_extend to 256-bits that should get
16628 // correctly legalized. We do this late to allow the canonical form of
16629 // sextload to persist throughout the rest of the DAG combiner -- it wants
16630 // to fold together any extensions it can, and so will fuse a sign_extend
16631 // of an sextload into a sextload targeting a wider value.
16633 if (MemSz == 128) {
16634 // Just switch this to a normal load.
16635 assert(TLI.isTypeLegal(MemVT) && "If the memory type is a 128-bit type, "
16636 "it must be a legal 128-bit vector "
16638 Load = DAG.getLoad(MemVT, dl, Ld->getChain(), Ld->getBasePtr(),
16639 Ld->getPointerInfo(), Ld->isVolatile(), Ld->isNonTemporal(),
16640 Ld->isInvariant(), Ld->getAlignment());
16642 assert(MemSz < 128 &&
16643 "Can't extend a type wider than 128 bits to a 256 bit vector!");
16644 // Do an sext load to a 128-bit vector type. We want to use the same
16645 // number of elements, but elements half as wide. This will end up being
16646 // recursively lowered by this routine, but will succeed as we definitely
16647 // have all the necessary features if we're using AVX1.
16649 EVT::getIntegerVT(*DAG.getContext(), RegVT.getScalarSizeInBits() / 2);
16650 EVT HalfVecVT = EVT::getVectorVT(*DAG.getContext(), HalfEltVT, NumElems);
16652 DAG.getExtLoad(Ext, dl, HalfVecVT, Ld->getChain(), Ld->getBasePtr(),
16653 Ld->getPointerInfo(), MemVT, Ld->isVolatile(),
16654 Ld->isNonTemporal(), Ld->isInvariant(),
16655 Ld->getAlignment());
16658 // Replace chain users with the new chain.
16659 assert(Load->getNumValues() == 2 && "Loads must carry a chain!");
16660 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
16662 // Finally, do a normal sign-extend to the desired register.
16663 return DAG.getSExtOrTrunc(Load, dl, RegVT);
16666 // All sizes must be a power of two.
16667 assert(isPowerOf2_32(RegSz * MemSz * NumElems) &&
16668 "Non-power-of-two elements are not custom lowered!");
16670 // Attempt to load the original value using scalar loads.
16671 // Find the largest scalar type that divides the total loaded size.
16672 MVT SclrLoadTy = MVT::i8;
16673 for (MVT Tp : MVT::integer_valuetypes()) {
16674 if (TLI.isTypeLegal(Tp) && ((MemSz % Tp.getSizeInBits()) == 0)) {
16679 // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
16680 if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 &&
16682 SclrLoadTy = MVT::f64;
16684 // Calculate the number of scalar loads that we need to perform
16685 // in order to load our vector from memory.
16686 unsigned NumLoads = MemSz / SclrLoadTy.getSizeInBits();
16688 assert((Ext != ISD::SEXTLOAD || NumLoads == 1) &&
16689 "Can only lower sext loads with a single scalar load!");
16691 unsigned loadRegZize = RegSz;
16692 if (Ext == ISD::SEXTLOAD && RegSz == 256)
16695 // Represent our vector as a sequence of elements which are the
16696 // largest scalar that we can load.
16697 EVT LoadUnitVecVT = EVT::getVectorVT(
16698 *DAG.getContext(), SclrLoadTy, loadRegZize / SclrLoadTy.getSizeInBits());
16700 // Represent the data using the same element type that is stored in
16701 // memory. In practice, we ''widen'' MemVT.
16703 EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
16704 loadRegZize / MemVT.getScalarType().getSizeInBits());
16706 assert(WideVecVT.getSizeInBits() == LoadUnitVecVT.getSizeInBits() &&
16707 "Invalid vector type");
16709 // We can't shuffle using an illegal type.
16710 assert(TLI.isTypeLegal(WideVecVT) &&
16711 "We only lower types that form legal widened vector types");
16713 SmallVector<SDValue, 8> Chains;
16714 SDValue Ptr = Ld->getBasePtr();
16715 SDValue Increment =
16716 DAG.getConstant(SclrLoadTy.getSizeInBits() / 8, TLI.getPointerTy());
16717 SDValue Res = DAG.getUNDEF(LoadUnitVecVT);
16719 for (unsigned i = 0; i < NumLoads; ++i) {
16720 // Perform a single load.
16721 SDValue ScalarLoad =
16722 DAG.getLoad(SclrLoadTy, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
16723 Ld->isVolatile(), Ld->isNonTemporal(), Ld->isInvariant(),
16724 Ld->getAlignment());
16725 Chains.push_back(ScalarLoad.getValue(1));
16726 // Create the first element type using SCALAR_TO_VECTOR in order to avoid
16727 // another round of DAGCombining.
16729 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoadUnitVecVT, ScalarLoad);
16731 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, LoadUnitVecVT, Res,
16732 ScalarLoad, DAG.getIntPtrConstant(i));
16734 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
16737 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
16739 // Bitcast the loaded value to a vector of the original element type, in
16740 // the size of the target vector type.
16741 SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Res);
16742 unsigned SizeRatio = RegSz / MemSz;
16744 if (Ext == ISD::SEXTLOAD) {
16745 // If we have SSE4.1, we can directly emit a VSEXT node.
16746 if (Subtarget->hasSSE41()) {
16747 SDValue Sext = DAG.getNode(X86ISD::VSEXT, dl, RegVT, SlicedVec);
16748 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
16752 // Otherwise we'll shuffle the small elements in the high bits of the
16753 // larger type and perform an arithmetic shift. If the shift is not legal
16754 // it's better to scalarize.
16755 assert(TLI.isOperationLegalOrCustom(ISD::SRA, RegVT) &&
16756 "We can't implement a sext load without an arithmetic right shift!");
16758 // Redistribute the loaded elements into the different locations.
16759 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
16760 for (unsigned i = 0; i != NumElems; ++i)
16761 ShuffleVec[i * SizeRatio + SizeRatio - 1] = i;
16763 SDValue Shuff = DAG.getVectorShuffle(
16764 WideVecVT, dl, SlicedVec, DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
16766 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
16768 // Build the arithmetic shift.
16769 unsigned Amt = RegVT.getVectorElementType().getSizeInBits() -
16770 MemVT.getVectorElementType().getSizeInBits();
16772 DAG.getNode(ISD::SRA, dl, RegVT, Shuff, DAG.getConstant(Amt, RegVT));
16774 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
16778 // Redistribute the loaded elements into the different locations.
16779 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
16780 for (unsigned i = 0; i != NumElems; ++i)
16781 ShuffleVec[i * SizeRatio] = i;
16783 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
16784 DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
16786 // Bitcast to the requested type.
16787 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
16788 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
16792 // isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or
16793 // ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart
16794 // from the AND / OR.
16795 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
16796 Opc = Op.getOpcode();
16797 if (Opc != ISD::OR && Opc != ISD::AND)
16799 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16800 Op.getOperand(0).hasOneUse() &&
16801 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
16802 Op.getOperand(1).hasOneUse());
16805 // isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and
16806 // 1 and that the SETCC node has a single use.
16807 static bool isXor1OfSetCC(SDValue Op) {
16808 if (Op.getOpcode() != ISD::XOR)
16810 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
16811 if (N1C && N1C->getAPIntValue() == 1) {
16812 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16813 Op.getOperand(0).hasOneUse();
16818 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
16819 bool addTest = true;
16820 SDValue Chain = Op.getOperand(0);
16821 SDValue Cond = Op.getOperand(1);
16822 SDValue Dest = Op.getOperand(2);
16825 bool Inverted = false;
16827 if (Cond.getOpcode() == ISD::SETCC) {
16828 // Check for setcc([su]{add,sub,mul}o == 0).
16829 if (cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
16830 isa<ConstantSDNode>(Cond.getOperand(1)) &&
16831 cast<ConstantSDNode>(Cond.getOperand(1))->isNullValue() &&
16832 Cond.getOperand(0).getResNo() == 1 &&
16833 (Cond.getOperand(0).getOpcode() == ISD::SADDO ||
16834 Cond.getOperand(0).getOpcode() == ISD::UADDO ||
16835 Cond.getOperand(0).getOpcode() == ISD::SSUBO ||
16836 Cond.getOperand(0).getOpcode() == ISD::USUBO ||
16837 Cond.getOperand(0).getOpcode() == ISD::SMULO ||
16838 Cond.getOperand(0).getOpcode() == ISD::UMULO)) {
16840 Cond = Cond.getOperand(0);
16842 SDValue NewCond = LowerSETCC(Cond, DAG);
16843 if (NewCond.getNode())
16848 // FIXME: LowerXALUO doesn't handle these!!
16849 else if (Cond.getOpcode() == X86ISD::ADD ||
16850 Cond.getOpcode() == X86ISD::SUB ||
16851 Cond.getOpcode() == X86ISD::SMUL ||
16852 Cond.getOpcode() == X86ISD::UMUL)
16853 Cond = LowerXALUO(Cond, DAG);
16856 // Look pass (and (setcc_carry (cmp ...)), 1).
16857 if (Cond.getOpcode() == ISD::AND &&
16858 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
16859 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
16860 if (C && C->getAPIntValue() == 1)
16861 Cond = Cond.getOperand(0);
16864 // If condition flag is set by a X86ISD::CMP, then use it as the condition
16865 // setting operand in place of the X86ISD::SETCC.
16866 unsigned CondOpcode = Cond.getOpcode();
16867 if (CondOpcode == X86ISD::SETCC ||
16868 CondOpcode == X86ISD::SETCC_CARRY) {
16869 CC = Cond.getOperand(0);
16871 SDValue Cmp = Cond.getOperand(1);
16872 unsigned Opc = Cmp.getOpcode();
16873 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp??
16874 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
16878 switch (cast<ConstantSDNode>(CC)->getZExtValue()) {
16882 // These can only come from an arithmetic instruction with overflow,
16883 // e.g. SADDO, UADDO.
16884 Cond = Cond.getNode()->getOperand(1);
16890 CondOpcode = Cond.getOpcode();
16891 if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
16892 CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
16893 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) &&
16894 Cond.getOperand(0).getValueType() != MVT::i8)) {
16895 SDValue LHS = Cond.getOperand(0);
16896 SDValue RHS = Cond.getOperand(1);
16897 unsigned X86Opcode;
16900 // Keep this in sync with LowerXALUO, otherwise we might create redundant
16901 // instructions that can't be removed afterwards (i.e. X86ISD::ADD and
16903 switch (CondOpcode) {
16904 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
16906 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
16908 X86Opcode = X86ISD::INC; X86Cond = X86::COND_O;
16911 X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
16912 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
16914 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
16916 X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O;
16919 X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
16920 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
16921 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
16922 default: llvm_unreachable("unexpected overflowing operator");
16925 X86Cond = X86::GetOppositeBranchCondition((X86::CondCode)X86Cond);
16926 if (CondOpcode == ISD::UMULO)
16927 VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(),
16930 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
16932 SDValue X86Op = DAG.getNode(X86Opcode, dl, VTs, LHS, RHS);
16934 if (CondOpcode == ISD::UMULO)
16935 Cond = X86Op.getValue(2);
16937 Cond = X86Op.getValue(1);
16939 CC = DAG.getConstant(X86Cond, MVT::i8);
16943 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) {
16944 SDValue Cmp = Cond.getOperand(0).getOperand(1);
16945 if (CondOpc == ISD::OR) {
16946 // Also, recognize the pattern generated by an FCMP_UNE. We can emit
16947 // two branches instead of an explicit OR instruction with a
16949 if (Cmp == Cond.getOperand(1).getOperand(1) &&
16950 isX86LogicalCmp(Cmp)) {
16951 CC = Cond.getOperand(0).getOperand(0);
16952 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16953 Chain, Dest, CC, Cmp);
16954 CC = Cond.getOperand(1).getOperand(0);
16958 } else { // ISD::AND
16959 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit
16960 // two branches instead of an explicit AND instruction with a
16961 // separate test. However, we only do this if this block doesn't
16962 // have a fall-through edge, because this requires an explicit
16963 // jmp when the condition is false.
16964 if (Cmp == Cond.getOperand(1).getOperand(1) &&
16965 isX86LogicalCmp(Cmp) &&
16966 Op.getNode()->hasOneUse()) {
16967 X86::CondCode CCode =
16968 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
16969 CCode = X86::GetOppositeBranchCondition(CCode);
16970 CC = DAG.getConstant(CCode, MVT::i8);
16971 SDNode *User = *Op.getNode()->use_begin();
16972 // Look for an unconditional branch following this conditional branch.
16973 // We need this because we need to reverse the successors in order
16974 // to implement FCMP_OEQ.
16975 if (User->getOpcode() == ISD::BR) {
16976 SDValue FalseBB = User->getOperand(1);
16978 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
16979 assert(NewBR == User);
16983 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16984 Chain, Dest, CC, Cmp);
16985 X86::CondCode CCode =
16986 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
16987 CCode = X86::GetOppositeBranchCondition(CCode);
16988 CC = DAG.getConstant(CCode, MVT::i8);
16994 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) {
16995 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition.
16996 // It should be transformed during dag combiner except when the condition
16997 // is set by a arithmetics with overflow node.
16998 X86::CondCode CCode =
16999 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
17000 CCode = X86::GetOppositeBranchCondition(CCode);
17001 CC = DAG.getConstant(CCode, MVT::i8);
17002 Cond = Cond.getOperand(0).getOperand(1);
17004 } else if (Cond.getOpcode() == ISD::SETCC &&
17005 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETOEQ) {
17006 // For FCMP_OEQ, we can emit
17007 // two branches instead of an explicit AND instruction with a
17008 // separate test. However, we only do this if this block doesn't
17009 // have a fall-through edge, because this requires an explicit
17010 // jmp when the condition is false.
17011 if (Op.getNode()->hasOneUse()) {
17012 SDNode *User = *Op.getNode()->use_begin();
17013 // Look for an unconditional branch following this conditional branch.
17014 // We need this because we need to reverse the successors in order
17015 // to implement FCMP_OEQ.
17016 if (User->getOpcode() == ISD::BR) {
17017 SDValue FalseBB = User->getOperand(1);
17019 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
17020 assert(NewBR == User);
17024 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
17025 Cond.getOperand(0), Cond.getOperand(1));
17026 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
17027 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
17028 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17029 Chain, Dest, CC, Cmp);
17030 CC = DAG.getConstant(X86::COND_P, MVT::i8);
17035 } else if (Cond.getOpcode() == ISD::SETCC &&
17036 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) {
17037 // For FCMP_UNE, we can emit
17038 // two branches instead of an explicit AND instruction with a
17039 // separate test. However, we only do this if this block doesn't
17040 // have a fall-through edge, because this requires an explicit
17041 // jmp when the condition is false.
17042 if (Op.getNode()->hasOneUse()) {
17043 SDNode *User = *Op.getNode()->use_begin();
17044 // Look for an unconditional branch following this conditional branch.
17045 // We need this because we need to reverse the successors in order
17046 // to implement FCMP_UNE.
17047 if (User->getOpcode() == ISD::BR) {
17048 SDValue FalseBB = User->getOperand(1);
17050 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
17051 assert(NewBR == User);
17054 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
17055 Cond.getOperand(0), Cond.getOperand(1));
17056 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
17057 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
17058 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17059 Chain, Dest, CC, Cmp);
17060 CC = DAG.getConstant(X86::COND_NP, MVT::i8);
17070 // Look pass the truncate if the high bits are known zero.
17071 if (isTruncWithZeroHighBitsInput(Cond, DAG))
17072 Cond = Cond.getOperand(0);
17074 // We know the result of AND is compared against zero. Try to match
17076 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
17077 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG);
17078 if (NewSetCC.getNode()) {
17079 CC = NewSetCC.getOperand(0);
17080 Cond = NewSetCC.getOperand(1);
17087 X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE;
17088 CC = DAG.getConstant(X86Cond, MVT::i8);
17089 Cond = EmitTest(Cond, X86Cond, dl, DAG);
17091 Cond = ConvertCmpIfNecessary(Cond, DAG);
17092 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17093 Chain, Dest, CC, Cond);
17096 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
17097 // Calls to _alloca are needed to probe the stack when allocating more than 4k
17098 // bytes in one go. Touching the stack at 4K increments is necessary to ensure
17099 // that the guard pages used by the OS virtual memory manager are allocated in
17100 // correct sequence.
17102 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
17103 SelectionDAG &DAG) const {
17104 MachineFunction &MF = DAG.getMachineFunction();
17105 bool SplitStack = MF.shouldSplitStack();
17106 bool Lower = (Subtarget->isOSWindows() && !Subtarget->isTargetMachO()) ||
17111 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17112 SDNode* Node = Op.getNode();
17114 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
17115 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
17116 " not tell us which reg is the stack pointer!");
17117 EVT VT = Node->getValueType(0);
17118 SDValue Tmp1 = SDValue(Node, 0);
17119 SDValue Tmp2 = SDValue(Node, 1);
17120 SDValue Tmp3 = Node->getOperand(2);
17121 SDValue Chain = Tmp1.getOperand(0);
17123 // Chain the dynamic stack allocation so that it doesn't modify the stack
17124 // pointer when other instructions are using the stack.
17125 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true),
17128 SDValue Size = Tmp2.getOperand(1);
17129 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
17130 Chain = SP.getValue(1);
17131 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
17132 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
17133 unsigned StackAlign = TFI.getStackAlignment();
17134 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
17135 if (Align > StackAlign)
17136 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
17137 DAG.getConstant(-(uint64_t)Align, VT));
17138 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
17140 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true),
17141 DAG.getIntPtrConstant(0, true), SDValue(),
17144 SDValue Ops[2] = { Tmp1, Tmp2 };
17145 return DAG.getMergeValues(Ops, dl);
17149 SDValue Chain = Op.getOperand(0);
17150 SDValue Size = Op.getOperand(1);
17151 unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
17152 EVT VT = Op.getNode()->getValueType(0);
17154 bool Is64Bit = Subtarget->is64Bit();
17155 EVT SPTy = getPointerTy();
17158 MachineRegisterInfo &MRI = MF.getRegInfo();
17161 // The 64 bit implementation of segmented stacks needs to clobber both r10
17162 // r11. This makes it impossible to use it along with nested parameters.
17163 const Function *F = MF.getFunction();
17165 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
17167 if (I->hasNestAttr())
17168 report_fatal_error("Cannot use segmented stacks with functions that "
17169 "have nested arguments.");
17172 const TargetRegisterClass *AddrRegClass =
17173 getRegClassFor(getPointerTy());
17174 unsigned Vreg = MRI.createVirtualRegister(AddrRegClass);
17175 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
17176 SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
17177 DAG.getRegister(Vreg, SPTy));
17178 SDValue Ops1[2] = { Value, Chain };
17179 return DAG.getMergeValues(Ops1, dl);
17182 const unsigned Reg = (Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX);
17184 Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag);
17185 Flag = Chain.getValue(1);
17186 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
17188 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
17190 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
17191 unsigned SPReg = RegInfo->getStackRegister();
17192 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
17193 Chain = SP.getValue(1);
17196 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
17197 DAG.getConstant(-(uint64_t)Align, VT));
17198 Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
17201 SDValue Ops1[2] = { SP, Chain };
17202 return DAG.getMergeValues(Ops1, dl);
17206 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
17207 MachineFunction &MF = DAG.getMachineFunction();
17208 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
17210 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
17213 if (!Subtarget->is64Bit() || Subtarget->isTargetWin64()) {
17214 // vastart just stores the address of the VarArgsFrameIndex slot into the
17215 // memory location argument.
17216 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
17218 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
17219 MachinePointerInfo(SV), false, false, 0);
17223 // gp_offset (0 - 6 * 8)
17224 // fp_offset (48 - 48 + 8 * 16)
17225 // overflow_arg_area (point to parameters coming in memory).
17227 SmallVector<SDValue, 8> MemOps;
17228 SDValue FIN = Op.getOperand(1);
17230 SDValue Store = DAG.getStore(Op.getOperand(0), DL,
17231 DAG.getConstant(FuncInfo->getVarArgsGPOffset(),
17233 FIN, MachinePointerInfo(SV), false, false, 0);
17234 MemOps.push_back(Store);
17237 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
17238 FIN, DAG.getIntPtrConstant(4));
17239 Store = DAG.getStore(Op.getOperand(0), DL,
17240 DAG.getConstant(FuncInfo->getVarArgsFPOffset(),
17242 FIN, MachinePointerInfo(SV, 4), false, false, 0);
17243 MemOps.push_back(Store);
17245 // Store ptr to overflow_arg_area
17246 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
17247 FIN, DAG.getIntPtrConstant(4));
17248 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
17250 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN,
17251 MachinePointerInfo(SV, 8),
17253 MemOps.push_back(Store);
17255 // Store ptr to reg_save_area.
17256 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
17257 FIN, DAG.getIntPtrConstant(8));
17258 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
17260 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN,
17261 MachinePointerInfo(SV, 16), false, false, 0);
17262 MemOps.push_back(Store);
17263 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
17266 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
17267 assert(Subtarget->is64Bit() &&
17268 "LowerVAARG only handles 64-bit va_arg!");
17269 assert((Subtarget->isTargetLinux() ||
17270 Subtarget->isTargetDarwin()) &&
17271 "Unhandled target in LowerVAARG");
17272 assert(Op.getNode()->getNumOperands() == 4);
17273 SDValue Chain = Op.getOperand(0);
17274 SDValue SrcPtr = Op.getOperand(1);
17275 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
17276 unsigned Align = Op.getConstantOperandVal(3);
17279 EVT ArgVT = Op.getNode()->getValueType(0);
17280 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
17281 uint32_t ArgSize = getDataLayout()->getTypeAllocSize(ArgTy);
17284 // Decide which area this value should be read from.
17285 // TODO: Implement the AMD64 ABI in its entirety. This simple
17286 // selection mechanism works only for the basic types.
17287 if (ArgVT == MVT::f80) {
17288 llvm_unreachable("va_arg for f80 not yet implemented");
17289 } else if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
17290 ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
17291 } else if (ArgVT.isInteger() && ArgSize <= 32 /*bytes*/) {
17292 ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
17294 llvm_unreachable("Unhandled argument type in LowerVAARG");
17297 if (ArgMode == 2) {
17298 // Sanity Check: Make sure using fp_offset makes sense.
17299 assert(!DAG.getTarget().Options.UseSoftFloat &&
17300 !(DAG.getMachineFunction().getFunction()->hasFnAttribute(
17301 Attribute::NoImplicitFloat)) &&
17302 Subtarget->hasSSE1());
17305 // Insert VAARG_64 node into the DAG
17306 // VAARG_64 returns two values: Variable Argument Address, Chain
17307 SmallVector<SDValue, 11> InstOps;
17308 InstOps.push_back(Chain);
17309 InstOps.push_back(SrcPtr);
17310 InstOps.push_back(DAG.getConstant(ArgSize, MVT::i32));
17311 InstOps.push_back(DAG.getConstant(ArgMode, MVT::i8));
17312 InstOps.push_back(DAG.getConstant(Align, MVT::i32));
17313 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other);
17314 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl,
17315 VTs, InstOps, MVT::i64,
17316 MachinePointerInfo(SV),
17318 /*Volatile=*/false,
17320 /*WriteMem=*/true);
17321 Chain = VAARG.getValue(1);
17323 // Load the next argument and return it
17324 return DAG.getLoad(ArgVT, dl,
17327 MachinePointerInfo(),
17328 false, false, false, 0);
17331 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget,
17332 SelectionDAG &DAG) {
17333 // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
17334 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!");
17335 SDValue Chain = Op.getOperand(0);
17336 SDValue DstPtr = Op.getOperand(1);
17337 SDValue SrcPtr = Op.getOperand(2);
17338 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
17339 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
17342 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr,
17343 DAG.getIntPtrConstant(24), 8, /*isVolatile*/false,
17345 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
17348 // getTargetVShiftByConstNode - Handle vector element shifts where the shift
17349 // amount is a constant. Takes immediate version of shift as input.
17350 static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
17351 SDValue SrcOp, uint64_t ShiftAmt,
17352 SelectionDAG &DAG) {
17353 MVT ElementType = VT.getVectorElementType();
17355 // Fold this packed shift into its first operand if ShiftAmt is 0.
17359 // Check for ShiftAmt >= element width
17360 if (ShiftAmt >= ElementType.getSizeInBits()) {
17361 if (Opc == X86ISD::VSRAI)
17362 ShiftAmt = ElementType.getSizeInBits() - 1;
17364 return DAG.getConstant(0, VT);
17367 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
17368 && "Unknown target vector shift-by-constant node");
17370 // Fold this packed vector shift into a build vector if SrcOp is a
17371 // vector of Constants or UNDEFs, and SrcOp valuetype is the same as VT.
17372 if (VT == SrcOp.getSimpleValueType() &&
17373 ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
17374 SmallVector<SDValue, 8> Elts;
17375 unsigned NumElts = SrcOp->getNumOperands();
17376 ConstantSDNode *ND;
17379 default: llvm_unreachable(nullptr);
17380 case X86ISD::VSHLI:
17381 for (unsigned i=0; i!=NumElts; ++i) {
17382 SDValue CurrentOp = SrcOp->getOperand(i);
17383 if (CurrentOp->getOpcode() == ISD::UNDEF) {
17384 Elts.push_back(CurrentOp);
17387 ND = cast<ConstantSDNode>(CurrentOp);
17388 const APInt &C = ND->getAPIntValue();
17389 Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), ElementType));
17392 case X86ISD::VSRLI:
17393 for (unsigned i=0; i!=NumElts; ++i) {
17394 SDValue CurrentOp = SrcOp->getOperand(i);
17395 if (CurrentOp->getOpcode() == ISD::UNDEF) {
17396 Elts.push_back(CurrentOp);
17399 ND = cast<ConstantSDNode>(CurrentOp);
17400 const APInt &C = ND->getAPIntValue();
17401 Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), ElementType));
17404 case X86ISD::VSRAI:
17405 for (unsigned i=0; i!=NumElts; ++i) {
17406 SDValue CurrentOp = SrcOp->getOperand(i);
17407 if (CurrentOp->getOpcode() == ISD::UNDEF) {
17408 Elts.push_back(CurrentOp);
17411 ND = cast<ConstantSDNode>(CurrentOp);
17412 const APInt &C = ND->getAPIntValue();
17413 Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), ElementType));
17418 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
17421 return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8));
17424 // getTargetVShiftNode - Handle vector element shifts where the shift amount
17425 // may or may not be a constant. Takes immediate version of shift as input.
17426 static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT,
17427 SDValue SrcOp, SDValue ShAmt,
17428 SelectionDAG &DAG) {
17429 MVT SVT = ShAmt.getSimpleValueType();
17430 assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!");
17432 // Catch shift-by-constant.
17433 if (ConstantSDNode *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
17434 return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp,
17435 CShAmt->getZExtValue(), DAG);
17437 // Change opcode to non-immediate version
17439 default: llvm_unreachable("Unknown target vector shift node");
17440 case X86ISD::VSHLI: Opc = X86ISD::VSHL; break;
17441 case X86ISD::VSRLI: Opc = X86ISD::VSRL; break;
17442 case X86ISD::VSRAI: Opc = X86ISD::VSRA; break;
17445 const X86Subtarget &Subtarget =
17446 static_cast<const X86Subtarget &>(DAG.getSubtarget());
17447 if (Subtarget.hasSSE41() && ShAmt.getOpcode() == ISD::ZERO_EXTEND &&
17448 ShAmt.getOperand(0).getSimpleValueType() == MVT::i16) {
17449 // Let the shuffle legalizer expand this shift amount node.
17450 SDValue Op0 = ShAmt.getOperand(0);
17451 Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(Op0), MVT::v8i16, Op0);
17452 ShAmt = getShuffleVectorZeroOrUndef(Op0, 0, true, &Subtarget, DAG);
17454 // Need to build a vector containing shift amount.
17455 // SSE/AVX packed shifts only use the lower 64-bit of the shift count.
17456 SmallVector<SDValue, 4> ShOps;
17457 ShOps.push_back(ShAmt);
17458 if (SVT == MVT::i32) {
17459 ShOps.push_back(DAG.getConstant(0, SVT));
17460 ShOps.push_back(DAG.getUNDEF(SVT));
17462 ShOps.push_back(DAG.getUNDEF(SVT));
17464 MVT BVT = SVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64;
17465 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, BVT, ShOps);
17468 // The return type has to be a 128-bit type with the same element
17469 // type as the input type.
17470 MVT EltVT = VT.getVectorElementType();
17471 EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits());
17473 ShAmt = DAG.getNode(ISD::BITCAST, dl, ShVT, ShAmt);
17474 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
17477 /// \brief Return (and \p Op, \p Mask) for compare instructions or
17478 /// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
17479 /// necessary casting for \p Mask when lowering masking intrinsics.
17480 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
17481 SDValue PreservedSrc,
17482 const X86Subtarget *Subtarget,
17483 SelectionDAG &DAG) {
17484 EVT VT = Op.getValueType();
17485 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(),
17486 MVT::i1, VT.getVectorNumElements());
17487 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17488 Mask.getValueType().getSizeInBits());
17491 assert(MaskVT.isSimple() && "invalid mask type");
17493 if (isAllOnes(Mask))
17496 // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
17497 // are extracted by EXTRACT_SUBVECTOR.
17498 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
17499 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
17500 DAG.getIntPtrConstant(0));
17502 switch (Op.getOpcode()) {
17504 case X86ISD::PCMPEQM:
17505 case X86ISD::PCMPGTM:
17507 case X86ISD::CMPMU:
17508 return DAG.getNode(ISD::AND, dl, VT, Op, VMask);
17510 if (PreservedSrc.getOpcode() == ISD::UNDEF)
17511 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
17512 return DAG.getNode(ISD::VSELECT, dl, VT, VMask, Op, PreservedSrc);
17515 /// \brief Creates an SDNode for a predicated scalar operation.
17516 /// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
17517 /// The mask is comming as MVT::i8 and it should be truncated
17518 /// to MVT::i1 while lowering masking intrinsics.
17519 /// The main difference between ScalarMaskingNode and VectorMaskingNode is using
17520 /// "X86select" instead of "vselect". We just can't create the "vselect" node for
17521 /// a scalar instruction.
17522 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
17523 SDValue PreservedSrc,
17524 const X86Subtarget *Subtarget,
17525 SelectionDAG &DAG) {
17526 if (isAllOnes(Mask))
17529 EVT VT = Op.getValueType();
17531 // The mask should be of type MVT::i1
17532 SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask);
17534 if (PreservedSrc.getOpcode() == ISD::UNDEF)
17535 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
17536 return DAG.getNode(X86ISD::SELECT, dl, VT, IMask, Op, PreservedSrc);
17539 static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
17540 SelectionDAG &DAG) {
17542 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
17543 EVT VT = Op.getValueType();
17544 const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
17546 switch(IntrData->Type) {
17547 case INTR_TYPE_1OP:
17548 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1));
17549 case INTR_TYPE_2OP:
17550 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17552 case INTR_TYPE_3OP:
17553 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17554 Op.getOperand(2), Op.getOperand(3));
17555 case INTR_TYPE_1OP_MASK_RM: {
17556 SDValue Src = Op.getOperand(1);
17557 SDValue Src0 = Op.getOperand(2);
17558 SDValue Mask = Op.getOperand(3);
17559 SDValue RoundingMode = Op.getOperand(4);
17560 return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src,
17562 Mask, Src0, Subtarget, DAG);
17564 case INTR_TYPE_SCALAR_MASK_RM: {
17565 SDValue Src1 = Op.getOperand(1);
17566 SDValue Src2 = Op.getOperand(2);
17567 SDValue Src0 = Op.getOperand(3);
17568 SDValue Mask = Op.getOperand(4);
17569 SDValue RoundingMode = Op.getOperand(5);
17570 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2,
17572 Mask, Src0, Subtarget, DAG);
17574 case INTR_TYPE_2OP_MASK: {
17575 SDValue Mask = Op.getOperand(4);
17576 SDValue PassThru = Op.getOperand(3);
17577 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
17578 if (IntrWithRoundingModeOpcode != 0) {
17579 unsigned Round = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue();
17580 if (Round != X86::STATIC_ROUNDING::CUR_DIRECTION) {
17581 return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
17582 dl, Op.getValueType(),
17583 Op.getOperand(1), Op.getOperand(2),
17584 Op.getOperand(3), Op.getOperand(5)),
17585 Mask, PassThru, Subtarget, DAG);
17588 return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
17591 Mask, PassThru, Subtarget, DAG);
17593 case FMA_OP_MASK: {
17594 SDValue Src1 = Op.getOperand(1);
17595 SDValue Src2 = Op.getOperand(2);
17596 SDValue Src3 = Op.getOperand(3);
17597 SDValue Mask = Op.getOperand(4);
17598 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
17599 if (IntrWithRoundingModeOpcode != 0) {
17600 SDValue Rnd = Op.getOperand(5);
17601 if (cast<ConstantSDNode>(Rnd)->getZExtValue() !=
17602 X86::STATIC_ROUNDING::CUR_DIRECTION)
17603 return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
17604 dl, Op.getValueType(),
17605 Src1, Src2, Src3, Rnd),
17606 Mask, Src1, Subtarget, DAG);
17608 return getVectorMaskingNode(DAG.getNode(IntrData->Opc0,
17609 dl, Op.getValueType(),
17611 Mask, Src1, Subtarget, DAG);
17614 case CMP_MASK_CC: {
17615 // Comparison intrinsics with masks.
17616 // Example of transformation:
17617 // (i8 (int_x86_avx512_mask_pcmpeq_q_128
17618 // (v2i64 %a), (v2i64 %b), (i8 %mask))) ->
17620 // (v8i1 (insert_subvector undef,
17621 // (v2i1 (and (PCMPEQM %a, %b),
17622 // (extract_subvector
17623 // (v8i1 (bitcast %mask)), 0))), 0))))
17624 EVT VT = Op.getOperand(1).getValueType();
17625 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17626 VT.getVectorNumElements());
17627 SDValue Mask = Op.getOperand((IntrData->Type == CMP_MASK_CC) ? 4 : 3);
17628 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17629 Mask.getValueType().getSizeInBits());
17631 if (IntrData->Type == CMP_MASK_CC) {
17632 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
17633 Op.getOperand(2), Op.getOperand(3));
17635 assert(IntrData->Type == CMP_MASK && "Unexpected intrinsic type!");
17636 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
17639 SDValue CmpMask = getVectorMaskingNode(Cmp, Mask,
17640 DAG.getTargetConstant(0, MaskVT),
17642 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, BitcastVT,
17643 DAG.getUNDEF(BitcastVT), CmpMask,
17644 DAG.getIntPtrConstant(0));
17645 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
17647 case COMI: { // Comparison intrinsics
17648 ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
17649 SDValue LHS = Op.getOperand(1);
17650 SDValue RHS = Op.getOperand(2);
17651 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG);
17652 assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!");
17653 SDValue Cond = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
17654 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17655 DAG.getConstant(X86CC, MVT::i8), Cond);
17656 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17659 return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
17660 Op.getOperand(1), Op.getOperand(2), DAG);
17662 return getVectorMaskingNode(getTargetVShiftNode(IntrData->Opc0, dl,
17663 Op.getSimpleValueType(),
17665 Op.getOperand(2), DAG),
17666 Op.getOperand(4), Op.getOperand(3), Subtarget,
17668 case COMPRESS_EXPAND_IN_REG: {
17669 SDValue Mask = Op.getOperand(3);
17670 SDValue DataToCompress = Op.getOperand(1);
17671 SDValue PassThru = Op.getOperand(2);
17672 if (isAllOnes(Mask)) // return data as is
17673 return Op.getOperand(1);
17674 EVT VT = Op.getValueType();
17675 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17676 VT.getVectorNumElements());
17677 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17678 Mask.getValueType().getSizeInBits());
17680 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
17681 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
17682 DAG.getIntPtrConstant(0));
17684 return DAG.getNode(IntrData->Opc0, dl, VT, VMask, DataToCompress,
17688 SDValue Mask = Op.getOperand(3);
17689 EVT VT = Op.getValueType();
17690 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17691 VT.getVectorNumElements());
17692 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17693 Mask.getValueType().getSizeInBits());
17695 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
17696 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
17697 DAG.getIntPtrConstant(0));
17698 return DAG.getNode(IntrData->Opc0, dl, VT, VMask, Op.getOperand(1),
17707 default: return SDValue(); // Don't custom lower most intrinsics.
17709 case Intrinsic::x86_avx512_mask_valign_q_512:
17710 case Intrinsic::x86_avx512_mask_valign_d_512:
17711 // Vector source operands are swapped.
17712 return getVectorMaskingNode(DAG.getNode(X86ISD::VALIGN, dl,
17713 Op.getValueType(), Op.getOperand(2),
17716 Op.getOperand(5), Op.getOperand(4),
17719 // ptest and testp intrinsics. The intrinsic these come from are designed to
17720 // return an integer value, not just an instruction so lower it to the ptest
17721 // or testp pattern and a setcc for the result.
17722 case Intrinsic::x86_sse41_ptestz:
17723 case Intrinsic::x86_sse41_ptestc:
17724 case Intrinsic::x86_sse41_ptestnzc:
17725 case Intrinsic::x86_avx_ptestz_256:
17726 case Intrinsic::x86_avx_ptestc_256:
17727 case Intrinsic::x86_avx_ptestnzc_256:
17728 case Intrinsic::x86_avx_vtestz_ps:
17729 case Intrinsic::x86_avx_vtestc_ps:
17730 case Intrinsic::x86_avx_vtestnzc_ps:
17731 case Intrinsic::x86_avx_vtestz_pd:
17732 case Intrinsic::x86_avx_vtestc_pd:
17733 case Intrinsic::x86_avx_vtestnzc_pd:
17734 case Intrinsic::x86_avx_vtestz_ps_256:
17735 case Intrinsic::x86_avx_vtestc_ps_256:
17736 case Intrinsic::x86_avx_vtestnzc_ps_256:
17737 case Intrinsic::x86_avx_vtestz_pd_256:
17738 case Intrinsic::x86_avx_vtestc_pd_256:
17739 case Intrinsic::x86_avx_vtestnzc_pd_256: {
17740 bool IsTestPacked = false;
17743 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
17744 case Intrinsic::x86_avx_vtestz_ps:
17745 case Intrinsic::x86_avx_vtestz_pd:
17746 case Intrinsic::x86_avx_vtestz_ps_256:
17747 case Intrinsic::x86_avx_vtestz_pd_256:
17748 IsTestPacked = true; // Fallthrough
17749 case Intrinsic::x86_sse41_ptestz:
17750 case Intrinsic::x86_avx_ptestz_256:
17752 X86CC = X86::COND_E;
17754 case Intrinsic::x86_avx_vtestc_ps:
17755 case Intrinsic::x86_avx_vtestc_pd:
17756 case Intrinsic::x86_avx_vtestc_ps_256:
17757 case Intrinsic::x86_avx_vtestc_pd_256:
17758 IsTestPacked = true; // Fallthrough
17759 case Intrinsic::x86_sse41_ptestc:
17760 case Intrinsic::x86_avx_ptestc_256:
17762 X86CC = X86::COND_B;
17764 case Intrinsic::x86_avx_vtestnzc_ps:
17765 case Intrinsic::x86_avx_vtestnzc_pd:
17766 case Intrinsic::x86_avx_vtestnzc_ps_256:
17767 case Intrinsic::x86_avx_vtestnzc_pd_256:
17768 IsTestPacked = true; // Fallthrough
17769 case Intrinsic::x86_sse41_ptestnzc:
17770 case Intrinsic::x86_avx_ptestnzc_256:
17772 X86CC = X86::COND_A;
17776 SDValue LHS = Op.getOperand(1);
17777 SDValue RHS = Op.getOperand(2);
17778 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
17779 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
17780 SDValue CC = DAG.getConstant(X86CC, MVT::i8);
17781 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
17782 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17784 case Intrinsic::x86_avx512_kortestz_w:
17785 case Intrinsic::x86_avx512_kortestc_w: {
17786 unsigned X86CC = (IntNo == Intrinsic::x86_avx512_kortestz_w)? X86::COND_E: X86::COND_B;
17787 SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1));
17788 SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2));
17789 SDValue CC = DAG.getConstant(X86CC, MVT::i8);
17790 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
17791 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test);
17792 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17795 case Intrinsic::x86_sse42_pcmpistria128:
17796 case Intrinsic::x86_sse42_pcmpestria128:
17797 case Intrinsic::x86_sse42_pcmpistric128:
17798 case Intrinsic::x86_sse42_pcmpestric128:
17799 case Intrinsic::x86_sse42_pcmpistrio128:
17800 case Intrinsic::x86_sse42_pcmpestrio128:
17801 case Intrinsic::x86_sse42_pcmpistris128:
17802 case Intrinsic::x86_sse42_pcmpestris128:
17803 case Intrinsic::x86_sse42_pcmpistriz128:
17804 case Intrinsic::x86_sse42_pcmpestriz128: {
17808 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
17809 case Intrinsic::x86_sse42_pcmpistria128:
17810 Opcode = X86ISD::PCMPISTRI;
17811 X86CC = X86::COND_A;
17813 case Intrinsic::x86_sse42_pcmpestria128:
17814 Opcode = X86ISD::PCMPESTRI;
17815 X86CC = X86::COND_A;
17817 case Intrinsic::x86_sse42_pcmpistric128:
17818 Opcode = X86ISD::PCMPISTRI;
17819 X86CC = X86::COND_B;
17821 case Intrinsic::x86_sse42_pcmpestric128:
17822 Opcode = X86ISD::PCMPESTRI;
17823 X86CC = X86::COND_B;
17825 case Intrinsic::x86_sse42_pcmpistrio128:
17826 Opcode = X86ISD::PCMPISTRI;
17827 X86CC = X86::COND_O;
17829 case Intrinsic::x86_sse42_pcmpestrio128:
17830 Opcode = X86ISD::PCMPESTRI;
17831 X86CC = X86::COND_O;
17833 case Intrinsic::x86_sse42_pcmpistris128:
17834 Opcode = X86ISD::PCMPISTRI;
17835 X86CC = X86::COND_S;
17837 case Intrinsic::x86_sse42_pcmpestris128:
17838 Opcode = X86ISD::PCMPESTRI;
17839 X86CC = X86::COND_S;
17841 case Intrinsic::x86_sse42_pcmpistriz128:
17842 Opcode = X86ISD::PCMPISTRI;
17843 X86CC = X86::COND_E;
17845 case Intrinsic::x86_sse42_pcmpestriz128:
17846 Opcode = X86ISD::PCMPESTRI;
17847 X86CC = X86::COND_E;
17850 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
17851 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
17852 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps);
17853 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17854 DAG.getConstant(X86CC, MVT::i8),
17855 SDValue(PCMP.getNode(), 1));
17856 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17859 case Intrinsic::x86_sse42_pcmpistri128:
17860 case Intrinsic::x86_sse42_pcmpestri128: {
17862 if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
17863 Opcode = X86ISD::PCMPISTRI;
17865 Opcode = X86ISD::PCMPESTRI;
17867 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
17868 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
17869 return DAG.getNode(Opcode, dl, VTs, NewOps);
17874 static SDValue getGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
17875 SDValue Src, SDValue Mask, SDValue Base,
17876 SDValue Index, SDValue ScaleOp, SDValue Chain,
17877 const X86Subtarget * Subtarget) {
17879 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
17880 assert(C && "Invalid scale type");
17881 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
17882 EVT MaskVT = MVT::getVectorVT(MVT::i1,
17883 Index.getSimpleValueType().getVectorNumElements());
17885 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
17887 MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
17889 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
17890 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
17891 SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
17892 SDValue Segment = DAG.getRegister(0, MVT::i32);
17893 if (Src.getOpcode() == ISD::UNDEF)
17894 Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl);
17895 SDValue Ops[] = {Src, MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
17896 SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
17897 SDValue RetOps[] = { SDValue(Res, 0), SDValue(Res, 2) };
17898 return DAG.getMergeValues(RetOps, dl);
17901 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
17902 SDValue Src, SDValue Mask, SDValue Base,
17903 SDValue Index, SDValue ScaleOp, SDValue Chain) {
17905 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
17906 assert(C && "Invalid scale type");
17907 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
17908 SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
17909 SDValue Segment = DAG.getRegister(0, MVT::i32);
17910 EVT MaskVT = MVT::getVectorVT(MVT::i1,
17911 Index.getSimpleValueType().getVectorNumElements());
17913 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
17915 MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
17917 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
17918 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
17919 SDValue Ops[] = {Base, Scale, Index, Disp, Segment, MaskInReg, Src, Chain};
17920 SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
17921 return SDValue(Res, 1);
17924 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
17925 SDValue Mask, SDValue Base, SDValue Index,
17926 SDValue ScaleOp, SDValue Chain) {
17928 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
17929 assert(C && "Invalid scale type");
17930 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
17931 SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
17932 SDValue Segment = DAG.getRegister(0, MVT::i32);
17934 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
17936 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
17938 MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
17940 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
17941 //SDVTList VTs = DAG.getVTList(MVT::Other);
17942 SDValue Ops[] = {MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
17943 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
17944 return SDValue(Res, 0);
17947 // getReadPerformanceCounter - Handles the lowering of builtin intrinsics that
17948 // read performance monitor counters (x86_rdpmc).
17949 static void getReadPerformanceCounter(SDNode *N, SDLoc DL,
17950 SelectionDAG &DAG, const X86Subtarget *Subtarget,
17951 SmallVectorImpl<SDValue> &Results) {
17952 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
17953 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
17956 // The ECX register is used to select the index of the performance counter
17958 SDValue Chain = DAG.getCopyToReg(N->getOperand(0), DL, X86::ECX,
17960 SDValue rd = DAG.getNode(X86ISD::RDPMC_DAG, DL, Tys, Chain);
17962 // Reads the content of a 64-bit performance counter and returns it in the
17963 // registers EDX:EAX.
17964 if (Subtarget->is64Bit()) {
17965 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
17966 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
17969 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
17970 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
17973 Chain = HI.getValue(1);
17975 if (Subtarget->is64Bit()) {
17976 // The EAX register is loaded with the low-order 32 bits. The EDX register
17977 // is loaded with the supported high-order bits of the counter.
17978 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
17979 DAG.getConstant(32, MVT::i8));
17980 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
17981 Results.push_back(Chain);
17985 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
17986 SDValue Ops[] = { LO, HI };
17987 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
17988 Results.push_back(Pair);
17989 Results.push_back(Chain);
17992 // getReadTimeStampCounter - Handles the lowering of builtin intrinsics that
17993 // read the time stamp counter (x86_rdtsc and x86_rdtscp). This function is
17994 // also used to custom lower READCYCLECOUNTER nodes.
17995 static void getReadTimeStampCounter(SDNode *N, SDLoc DL, unsigned Opcode,
17996 SelectionDAG &DAG, const X86Subtarget *Subtarget,
17997 SmallVectorImpl<SDValue> &Results) {
17998 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
17999 SDValue rd = DAG.getNode(Opcode, DL, Tys, N->getOperand(0));
18002 // The processor's time-stamp counter (a 64-bit MSR) is stored into the
18003 // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
18004 // and the EAX register is loaded with the low-order 32 bits.
18005 if (Subtarget->is64Bit()) {
18006 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
18007 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
18010 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
18011 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
18014 SDValue Chain = HI.getValue(1);
18016 if (Opcode == X86ISD::RDTSCP_DAG) {
18017 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
18019 // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
18020 // the ECX register. Add 'ecx' explicitly to the chain.
18021 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32,
18023 // Explicitly store the content of ECX at the location passed in input
18024 // to the 'rdtscp' intrinsic.
18025 Chain = DAG.getStore(ecx.getValue(1), DL, ecx, N->getOperand(2),
18026 MachinePointerInfo(), false, false, 0);
18029 if (Subtarget->is64Bit()) {
18030 // The EDX register is loaded with the high-order 32 bits of the MSR, and
18031 // the EAX register is loaded with the low-order 32 bits.
18032 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
18033 DAG.getConstant(32, MVT::i8));
18034 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
18035 Results.push_back(Chain);
18039 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
18040 SDValue Ops[] = { LO, HI };
18041 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
18042 Results.push_back(Pair);
18043 Results.push_back(Chain);
18046 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
18047 SelectionDAG &DAG) {
18048 SmallVector<SDValue, 2> Results;
18050 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
18052 return DAG.getMergeValues(Results, DL);
18056 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
18057 SelectionDAG &DAG) {
18058 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
18060 const IntrinsicData* IntrData = getIntrinsicWithChain(IntNo);
18065 switch(IntrData->Type) {
18067 llvm_unreachable("Unknown Intrinsic Type");
18071 // Emit the node with the right value type.
18072 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other);
18073 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
18075 // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
18076 // Otherwise return the value from Rand, which is always 0, casted to i32.
18077 SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
18078 DAG.getConstant(1, Op->getValueType(1)),
18079 DAG.getConstant(X86::COND_B, MVT::i32),
18080 SDValue(Result.getNode(), 1) };
18081 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl,
18082 DAG.getVTList(Op->getValueType(1), MVT::Glue),
18085 // Return { result, isValid, chain }.
18086 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
18087 SDValue(Result.getNode(), 2));
18090 //gather(v1, mask, index, base, scale);
18091 SDValue Chain = Op.getOperand(0);
18092 SDValue Src = Op.getOperand(2);
18093 SDValue Base = Op.getOperand(3);
18094 SDValue Index = Op.getOperand(4);
18095 SDValue Mask = Op.getOperand(5);
18096 SDValue Scale = Op.getOperand(6);
18097 return getGatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain,
18101 //scatter(base, mask, index, v1, scale);
18102 SDValue Chain = Op.getOperand(0);
18103 SDValue Base = Op.getOperand(2);
18104 SDValue Mask = Op.getOperand(3);
18105 SDValue Index = Op.getOperand(4);
18106 SDValue Src = Op.getOperand(5);
18107 SDValue Scale = Op.getOperand(6);
18108 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain);
18111 SDValue Hint = Op.getOperand(6);
18113 if (dyn_cast<ConstantSDNode> (Hint) == nullptr ||
18114 (HintVal = dyn_cast<ConstantSDNode> (Hint)->getZExtValue()) > 1)
18115 llvm_unreachable("Wrong prefetch hint in intrinsic: should be 0 or 1");
18116 unsigned Opcode = (HintVal ? IntrData->Opc1 : IntrData->Opc0);
18117 SDValue Chain = Op.getOperand(0);
18118 SDValue Mask = Op.getOperand(2);
18119 SDValue Index = Op.getOperand(3);
18120 SDValue Base = Op.getOperand(4);
18121 SDValue Scale = Op.getOperand(5);
18122 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain);
18124 // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
18126 SmallVector<SDValue, 2> Results;
18127 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, Results);
18128 return DAG.getMergeValues(Results, dl);
18130 // Read Performance Monitoring Counters.
18132 SmallVector<SDValue, 2> Results;
18133 getReadPerformanceCounter(Op.getNode(), dl, DAG, Subtarget, Results);
18134 return DAG.getMergeValues(Results, dl);
18136 // XTEST intrinsics.
18138 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
18139 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
18140 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18141 DAG.getConstant(X86::COND_NE, MVT::i8),
18143 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
18144 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
18145 Ret, SDValue(InTrans.getNode(), 1));
18149 SmallVector<SDValue, 2> Results;
18150 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
18151 SDVTList VTs = DAG.getVTList(Op.getOperand(3)->getValueType(0), MVT::Other);
18152 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(2),
18153 DAG.getConstant(-1, MVT::i8));
18154 SDValue Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(3),
18155 Op.getOperand(4), GenCF.getValue(1));
18156 SDValue Store = DAG.getStore(Op.getOperand(0), dl, Res.getValue(0),
18157 Op.getOperand(5), MachinePointerInfo(),
18159 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18160 DAG.getConstant(X86::COND_B, MVT::i8),
18162 Results.push_back(SetCC);
18163 Results.push_back(Store);
18164 return DAG.getMergeValues(Results, dl);
18166 case COMPRESS_TO_MEM: {
18168 SDValue Mask = Op.getOperand(4);
18169 SDValue DataToCompress = Op.getOperand(3);
18170 SDValue Addr = Op.getOperand(2);
18171 SDValue Chain = Op.getOperand(0);
18173 if (isAllOnes(Mask)) // return just a store
18174 return DAG.getStore(Chain, dl, DataToCompress, Addr,
18175 MachinePointerInfo(), false, false, 0);
18177 EVT VT = DataToCompress.getValueType();
18178 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18179 VT.getVectorNumElements());
18180 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18181 Mask.getValueType().getSizeInBits());
18182 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
18183 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
18184 DAG.getIntPtrConstant(0));
18186 SDValue Compressed = DAG.getNode(IntrData->Opc0, dl, VT, VMask,
18187 DataToCompress, DAG.getUNDEF(VT));
18188 return DAG.getStore(Chain, dl, Compressed, Addr,
18189 MachinePointerInfo(), false, false, 0);
18191 case EXPAND_FROM_MEM: {
18193 SDValue Mask = Op.getOperand(4);
18194 SDValue PathThru = Op.getOperand(3);
18195 SDValue Addr = Op.getOperand(2);
18196 SDValue Chain = Op.getOperand(0);
18197 EVT VT = Op.getValueType();
18199 if (isAllOnes(Mask)) // return just a load
18200 return DAG.getLoad(VT, dl, Chain, Addr, MachinePointerInfo(), false, false,
18202 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18203 VT.getVectorNumElements());
18204 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18205 Mask.getValueType().getSizeInBits());
18206 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
18207 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
18208 DAG.getIntPtrConstant(0));
18210 SDValue DataToExpand = DAG.getLoad(VT, dl, Chain, Addr, MachinePointerInfo(),
18211 false, false, false, 0);
18213 SmallVector<SDValue, 2> Results;
18214 Results.push_back(DAG.getNode(IntrData->Opc0, dl, VT, VMask, DataToExpand,
18216 Results.push_back(Chain);
18217 return DAG.getMergeValues(Results, dl);
18222 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
18223 SelectionDAG &DAG) const {
18224 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
18225 MFI->setReturnAddressIsTaken(true);
18227 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
18230 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
18232 EVT PtrVT = getPointerTy();
18235 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
18236 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18237 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), PtrVT);
18238 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
18239 DAG.getNode(ISD::ADD, dl, PtrVT,
18240 FrameAddr, Offset),
18241 MachinePointerInfo(), false, false, false, 0);
18244 // Just load the return address.
18245 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
18246 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
18247 RetAddrFI, MachinePointerInfo(), false, false, false, 0);
18250 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
18251 MachineFunction &MF = DAG.getMachineFunction();
18252 MachineFrameInfo *MFI = MF.getFrameInfo();
18253 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
18254 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18255 EVT VT = Op.getValueType();
18257 MFI->setFrameAddressIsTaken(true);
18259 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
18260 // Depth > 0 makes no sense on targets which use Windows unwind codes. It
18261 // is not possible to crawl up the stack without looking at the unwind codes
18263 int FrameAddrIndex = FuncInfo->getFAIndex();
18264 if (!FrameAddrIndex) {
18265 // Set up a frame object for the return address.
18266 unsigned SlotSize = RegInfo->getSlotSize();
18267 FrameAddrIndex = MF.getFrameInfo()->CreateFixedObject(
18268 SlotSize, /*Offset=*/INT64_MIN, /*IsImmutable=*/false);
18269 FuncInfo->setFAIndex(FrameAddrIndex);
18271 return DAG.getFrameIndex(FrameAddrIndex, VT);
18274 unsigned FrameReg =
18275 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
18276 SDLoc dl(Op); // FIXME probably not meaningful
18277 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
18278 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
18279 (FrameReg == X86::EBP && VT == MVT::i32)) &&
18280 "Invalid Frame Register!");
18281 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
18283 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
18284 MachinePointerInfo(),
18285 false, false, false, 0);
18289 // FIXME? Maybe this could be a TableGen attribute on some registers and
18290 // this table could be generated automatically from RegInfo.
18291 unsigned X86TargetLowering::getRegisterByName(const char* RegName,
18293 unsigned Reg = StringSwitch<unsigned>(RegName)
18294 .Case("esp", X86::ESP)
18295 .Case("rsp", X86::RSP)
18299 report_fatal_error("Invalid register name global variable");
18302 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
18303 SelectionDAG &DAG) const {
18304 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18305 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize());
18308 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
18309 SDValue Chain = Op.getOperand(0);
18310 SDValue Offset = Op.getOperand(1);
18311 SDValue Handler = Op.getOperand(2);
18314 EVT PtrVT = getPointerTy();
18315 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18316 unsigned FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
18317 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
18318 (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
18319 "Invalid Frame Register!");
18320 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
18321 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
18323 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
18324 DAG.getIntPtrConstant(RegInfo->getSlotSize()));
18325 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
18326 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(),
18328 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
18330 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
18331 DAG.getRegister(StoreAddrReg, PtrVT));
18334 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
18335 SelectionDAG &DAG) const {
18337 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
18338 DAG.getVTList(MVT::i32, MVT::Other),
18339 Op.getOperand(0), Op.getOperand(1));
18342 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
18343 SelectionDAG &DAG) const {
18345 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
18346 Op.getOperand(0), Op.getOperand(1));
18349 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
18350 return Op.getOperand(0);
18353 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
18354 SelectionDAG &DAG) const {
18355 SDValue Root = Op.getOperand(0);
18356 SDValue Trmp = Op.getOperand(1); // trampoline
18357 SDValue FPtr = Op.getOperand(2); // nested function
18358 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
18361 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
18362 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
18364 if (Subtarget->is64Bit()) {
18365 SDValue OutChains[6];
18367 // Large code-model.
18368 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
18369 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
18371 const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
18372 const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
18374 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
18376 // Load the pointer to the nested function into R11.
18377 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
18378 SDValue Addr = Trmp;
18379 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
18380 Addr, MachinePointerInfo(TrmpAddr),
18383 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18384 DAG.getConstant(2, MVT::i64));
18385 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
18386 MachinePointerInfo(TrmpAddr, 2),
18389 // Load the 'nest' parameter value into R10.
18390 // R10 is specified in X86CallingConv.td
18391 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
18392 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18393 DAG.getConstant(10, MVT::i64));
18394 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
18395 Addr, MachinePointerInfo(TrmpAddr, 10),
18398 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18399 DAG.getConstant(12, MVT::i64));
18400 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
18401 MachinePointerInfo(TrmpAddr, 12),
18404 // Jump to the nested function.
18405 OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
18406 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18407 DAG.getConstant(20, MVT::i64));
18408 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
18409 Addr, MachinePointerInfo(TrmpAddr, 20),
18412 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
18413 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18414 DAG.getConstant(22, MVT::i64));
18415 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr,
18416 MachinePointerInfo(TrmpAddr, 22),
18419 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
18421 const Function *Func =
18422 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
18423 CallingConv::ID CC = Func->getCallingConv();
18428 llvm_unreachable("Unsupported calling convention");
18429 case CallingConv::C:
18430 case CallingConv::X86_StdCall: {
18431 // Pass 'nest' parameter in ECX.
18432 // Must be kept in sync with X86CallingConv.td
18433 NestReg = X86::ECX;
18435 // Check that ECX wasn't needed by an 'inreg' parameter.
18436 FunctionType *FTy = Func->getFunctionType();
18437 const AttributeSet &Attrs = Func->getAttributes();
18439 if (!Attrs.isEmpty() && !Func->isVarArg()) {
18440 unsigned InRegCount = 0;
18443 for (FunctionType::param_iterator I = FTy->param_begin(),
18444 E = FTy->param_end(); I != E; ++I, ++Idx)
18445 if (Attrs.hasAttribute(Idx, Attribute::InReg))
18446 // FIXME: should only count parameters that are lowered to integers.
18447 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32;
18449 if (InRegCount > 2) {
18450 report_fatal_error("Nest register in use - reduce number of inreg"
18456 case CallingConv::X86_FastCall:
18457 case CallingConv::X86_ThisCall:
18458 case CallingConv::Fast:
18459 // Pass 'nest' parameter in EAX.
18460 // Must be kept in sync with X86CallingConv.td
18461 NestReg = X86::EAX;
18465 SDValue OutChains[4];
18466 SDValue Addr, Disp;
18468 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18469 DAG.getConstant(10, MVT::i32));
18470 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
18472 // This is storing the opcode for MOV32ri.
18473 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
18474 const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
18475 OutChains[0] = DAG.getStore(Root, dl,
18476 DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
18477 Trmp, MachinePointerInfo(TrmpAddr),
18480 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18481 DAG.getConstant(1, MVT::i32));
18482 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
18483 MachinePointerInfo(TrmpAddr, 1),
18486 const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
18487 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18488 DAG.getConstant(5, MVT::i32));
18489 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr,
18490 MachinePointerInfo(TrmpAddr, 5),
18493 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18494 DAG.getConstant(6, MVT::i32));
18495 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
18496 MachinePointerInfo(TrmpAddr, 6),
18499 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
18503 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
18504 SelectionDAG &DAG) const {
18506 The rounding mode is in bits 11:10 of FPSR, and has the following
18508 00 Round to nearest
18513 FLT_ROUNDS, on the other hand, expects the following:
18520 To perform the conversion, we do:
18521 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3)
18524 MachineFunction &MF = DAG.getMachineFunction();
18525 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
18526 unsigned StackAlignment = TFI.getStackAlignment();
18527 MVT VT = Op.getSimpleValueType();
18530 // Save FP Control Word to stack slot
18531 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment, false);
18532 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
18534 MachineMemOperand *MMO =
18535 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
18536 MachineMemOperand::MOStore, 2, 2);
18538 SDValue Ops[] = { DAG.getEntryNode(), StackSlot };
18539 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
18540 DAG.getVTList(MVT::Other),
18541 Ops, MVT::i16, MMO);
18543 // Load FP Control Word from stack slot
18544 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot,
18545 MachinePointerInfo(), false, false, false, 0);
18547 // Transform as necessary
18549 DAG.getNode(ISD::SRL, DL, MVT::i16,
18550 DAG.getNode(ISD::AND, DL, MVT::i16,
18551 CWD, DAG.getConstant(0x800, MVT::i16)),
18552 DAG.getConstant(11, MVT::i8));
18554 DAG.getNode(ISD::SRL, DL, MVT::i16,
18555 DAG.getNode(ISD::AND, DL, MVT::i16,
18556 CWD, DAG.getConstant(0x400, MVT::i16)),
18557 DAG.getConstant(9, MVT::i8));
18560 DAG.getNode(ISD::AND, DL, MVT::i16,
18561 DAG.getNode(ISD::ADD, DL, MVT::i16,
18562 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2),
18563 DAG.getConstant(1, MVT::i16)),
18564 DAG.getConstant(3, MVT::i16));
18566 return DAG.getNode((VT.getSizeInBits() < 16 ?
18567 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
18570 static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) {
18571 MVT VT = Op.getSimpleValueType();
18573 unsigned NumBits = VT.getSizeInBits();
18576 Op = Op.getOperand(0);
18577 if (VT == MVT::i8) {
18578 // Zero extend to i32 since there is not an i8 bsr.
18580 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
18583 // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
18584 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
18585 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
18587 // If src is zero (i.e. bsr sets ZF), returns NumBits.
18590 DAG.getConstant(NumBits+NumBits-1, OpVT),
18591 DAG.getConstant(X86::COND_E, MVT::i8),
18594 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
18596 // Finally xor with NumBits-1.
18597 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
18600 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
18604 static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
18605 MVT VT = Op.getSimpleValueType();
18607 unsigned NumBits = VT.getSizeInBits();
18610 Op = Op.getOperand(0);
18611 if (VT == MVT::i8) {
18612 // Zero extend to i32 since there is not an i8 bsr.
18614 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
18617 // Issue a bsr (scan bits in reverse).
18618 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
18619 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
18621 // And xor with NumBits-1.
18622 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
18625 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
18629 static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
18630 MVT VT = Op.getSimpleValueType();
18631 unsigned NumBits = VT.getSizeInBits();
18633 Op = Op.getOperand(0);
18635 // Issue a bsf (scan bits forward) which also sets EFLAGS.
18636 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18637 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
18639 // If src is zero (i.e. bsf sets ZF), returns NumBits.
18642 DAG.getConstant(NumBits, VT),
18643 DAG.getConstant(X86::COND_E, MVT::i8),
18646 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
18649 // Lower256IntArith - Break a 256-bit integer operation into two new 128-bit
18650 // ones, and then concatenate the result back.
18651 static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
18652 MVT VT = Op.getSimpleValueType();
18654 assert(VT.is256BitVector() && VT.isInteger() &&
18655 "Unsupported value type for operation");
18657 unsigned NumElems = VT.getVectorNumElements();
18660 // Extract the LHS vectors
18661 SDValue LHS = Op.getOperand(0);
18662 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
18663 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
18665 // Extract the RHS vectors
18666 SDValue RHS = Op.getOperand(1);
18667 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl);
18668 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl);
18670 MVT EltVT = VT.getVectorElementType();
18671 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
18673 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
18674 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
18675 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
18678 static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
18679 assert(Op.getSimpleValueType().is256BitVector() &&
18680 Op.getSimpleValueType().isInteger() &&
18681 "Only handle AVX 256-bit vector integer operation");
18682 return Lower256IntArith(Op, DAG);
18685 static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) {
18686 assert(Op.getSimpleValueType().is256BitVector() &&
18687 Op.getSimpleValueType().isInteger() &&
18688 "Only handle AVX 256-bit vector integer operation");
18689 return Lower256IntArith(Op, DAG);
18692 static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
18693 SelectionDAG &DAG) {
18695 MVT VT = Op.getSimpleValueType();
18697 // Decompose 256-bit ops into smaller 128-bit ops.
18698 if (VT.is256BitVector() && !Subtarget->hasInt256())
18699 return Lower256IntArith(Op, DAG);
18701 SDValue A = Op.getOperand(0);
18702 SDValue B = Op.getOperand(1);
18704 // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
18705 if (VT == MVT::v4i32) {
18706 assert(Subtarget->hasSSE2() && !Subtarget->hasSSE41() &&
18707 "Should not custom lower when pmuldq is available!");
18709 // Extract the odd parts.
18710 static const int UnpackMask[] = { 1, -1, 3, -1 };
18711 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
18712 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
18714 // Multiply the even parts.
18715 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B);
18716 // Now multiply odd parts.
18717 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds);
18719 Evens = DAG.getNode(ISD::BITCAST, dl, VT, Evens);
18720 Odds = DAG.getNode(ISD::BITCAST, dl, VT, Odds);
18722 // Merge the two vectors back together with a shuffle. This expands into 2
18724 static const int ShufMask[] = { 0, 4, 2, 6 };
18725 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
18728 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
18729 "Only know how to lower V2I64/V4I64/V8I64 multiply");
18731 // Ahi = psrlqi(a, 32);
18732 // Bhi = psrlqi(b, 32);
18734 // AloBlo = pmuludq(a, b);
18735 // AloBhi = pmuludq(a, Bhi);
18736 // AhiBlo = pmuludq(Ahi, b);
18738 // AloBhi = psllqi(AloBhi, 32);
18739 // AhiBlo = psllqi(AhiBlo, 32);
18740 // return AloBlo + AloBhi + AhiBlo;
18742 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
18743 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
18745 // Bit cast to 32-bit vectors for MULUDQ
18746 EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 :
18747 (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32;
18748 A = DAG.getNode(ISD::BITCAST, dl, MulVT, A);
18749 B = DAG.getNode(ISD::BITCAST, dl, MulVT, B);
18750 Ahi = DAG.getNode(ISD::BITCAST, dl, MulVT, Ahi);
18751 Bhi = DAG.getNode(ISD::BITCAST, dl, MulVT, Bhi);
18753 SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
18754 SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
18755 SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
18757 AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
18758 AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
18760 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
18761 return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
18764 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
18765 assert(Subtarget->isTargetWin64() && "Unexpected target");
18766 EVT VT = Op.getValueType();
18767 assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
18768 "Unexpected return type for lowering");
18772 switch (Op->getOpcode()) {
18773 default: llvm_unreachable("Unexpected request for libcall!");
18774 case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
18775 case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
18776 case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
18777 case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
18778 case ISD::SDIVREM: isSigned = true; LC = RTLIB::SDIVREM_I128; break;
18779 case ISD::UDIVREM: isSigned = false; LC = RTLIB::UDIVREM_I128; break;
18783 SDValue InChain = DAG.getEntryNode();
18785 TargetLowering::ArgListTy Args;
18786 TargetLowering::ArgListEntry Entry;
18787 for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
18788 EVT ArgVT = Op->getOperand(i).getValueType();
18789 assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
18790 "Unexpected argument type for lowering");
18791 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
18792 Entry.Node = StackPtr;
18793 InChain = DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MachinePointerInfo(),
18795 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
18796 Entry.Ty = PointerType::get(ArgTy,0);
18797 Entry.isSExt = false;
18798 Entry.isZExt = false;
18799 Args.push_back(Entry);
18802 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
18805 TargetLowering::CallLoweringInfo CLI(DAG);
18806 CLI.setDebugLoc(dl).setChain(InChain)
18807 .setCallee(getLibcallCallingConv(LC),
18808 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()),
18809 Callee, std::move(Args), 0)
18810 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
18812 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
18813 return DAG.getNode(ISD::BITCAST, dl, VT, CallInfo.first);
18816 static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget *Subtarget,
18817 SelectionDAG &DAG) {
18818 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
18819 EVT VT = Op0.getValueType();
18822 assert((VT == MVT::v4i32 && Subtarget->hasSSE2()) ||
18823 (VT == MVT::v8i32 && Subtarget->hasInt256()));
18825 // PMULxD operations multiply each even value (starting at 0) of LHS with
18826 // the related value of RHS and produce a widen result.
18827 // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
18828 // => <2 x i64> <ae|cg>
18830 // In other word, to have all the results, we need to perform two PMULxD:
18831 // 1. one with the even values.
18832 // 2. one with the odd values.
18833 // To achieve #2, with need to place the odd values at an even position.
18835 // Place the odd value at an even position (basically, shift all values 1
18836 // step to the left):
18837 const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1};
18838 // <a|b|c|d> => <b|undef|d|undef>
18839 SDValue Odd0 = DAG.getVectorShuffle(VT, dl, Op0, Op0, Mask);
18840 // <e|f|g|h> => <f|undef|h|undef>
18841 SDValue Odd1 = DAG.getVectorShuffle(VT, dl, Op1, Op1, Mask);
18843 // Emit two multiplies, one for the lower 2 ints and one for the higher 2
18845 MVT MulVT = VT == MVT::v4i32 ? MVT::v2i64 : MVT::v4i64;
18846 bool IsSigned = Op->getOpcode() == ISD::SMUL_LOHI;
18848 (!IsSigned || !Subtarget->hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
18849 // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
18850 // => <2 x i64> <ae|cg>
18851 SDValue Mul1 = DAG.getNode(ISD::BITCAST, dl, VT,
18852 DAG.getNode(Opcode, dl, MulVT, Op0, Op1));
18853 // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
18854 // => <2 x i64> <bf|dh>
18855 SDValue Mul2 = DAG.getNode(ISD::BITCAST, dl, VT,
18856 DAG.getNode(Opcode, dl, MulVT, Odd0, Odd1));
18858 // Shuffle it back into the right order.
18859 SDValue Highs, Lows;
18860 if (VT == MVT::v8i32) {
18861 const int HighMask[] = {1, 9, 3, 11, 5, 13, 7, 15};
18862 Highs = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, HighMask);
18863 const int LowMask[] = {0, 8, 2, 10, 4, 12, 6, 14};
18864 Lows = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, LowMask);
18866 const int HighMask[] = {1, 5, 3, 7};
18867 Highs = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, HighMask);
18868 const int LowMask[] = {0, 4, 2, 6};
18869 Lows = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, LowMask);
18872 // If we have a signed multiply but no PMULDQ fix up the high parts of a
18873 // unsigned multiply.
18874 if (IsSigned && !Subtarget->hasSSE41()) {
18876 DAG.getConstant(31, DAG.getTargetLoweringInfo().getShiftAmountTy(VT));
18877 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
18878 DAG.getNode(ISD::SRA, dl, VT, Op0, ShAmt), Op1);
18879 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
18880 DAG.getNode(ISD::SRA, dl, VT, Op1, ShAmt), Op0);
18882 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
18883 Highs = DAG.getNode(ISD::SUB, dl, VT, Highs, Fixup);
18886 // The first result of MUL_LOHI is actually the low value, followed by the
18888 SDValue Ops[] = {Lows, Highs};
18889 return DAG.getMergeValues(Ops, dl);
18892 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
18893 const X86Subtarget *Subtarget) {
18894 MVT VT = Op.getSimpleValueType();
18896 SDValue R = Op.getOperand(0);
18897 SDValue Amt = Op.getOperand(1);
18899 // Optimize shl/srl/sra with constant shift amount.
18900 if (auto *BVAmt = dyn_cast<BuildVectorSDNode>(Amt)) {
18901 if (auto *ShiftConst = BVAmt->getConstantSplatNode()) {
18902 uint64_t ShiftAmt = ShiftConst->getZExtValue();
18904 if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
18905 (Subtarget->hasInt256() &&
18906 (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16)) ||
18907 (Subtarget->hasAVX512() &&
18908 (VT == MVT::v8i64 || VT == MVT::v16i32))) {
18909 if (Op.getOpcode() == ISD::SHL)
18910 return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
18912 if (Op.getOpcode() == ISD::SRL)
18913 return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
18915 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64)
18916 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
18920 if (VT == MVT::v16i8) {
18921 if (Op.getOpcode() == ISD::SHL) {
18922 // Make a large shift.
18923 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
18924 MVT::v8i16, R, ShiftAmt,
18926 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
18927 // Zero out the rightmost bits.
18928 SmallVector<SDValue, 16> V(16,
18929 DAG.getConstant(uint8_t(-1U << ShiftAmt),
18931 return DAG.getNode(ISD::AND, dl, VT, SHL,
18932 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
18934 if (Op.getOpcode() == ISD::SRL) {
18935 // Make a large shift.
18936 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
18937 MVT::v8i16, R, ShiftAmt,
18939 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
18940 // Zero out the leftmost bits.
18941 SmallVector<SDValue, 16> V(16,
18942 DAG.getConstant(uint8_t(-1U) >> ShiftAmt,
18944 return DAG.getNode(ISD::AND, dl, VT, SRL,
18945 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
18947 if (Op.getOpcode() == ISD::SRA) {
18948 if (ShiftAmt == 7) {
18949 // R s>> 7 === R s< 0
18950 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
18951 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
18954 // R s>> a === ((R u>> a) ^ m) - m
18955 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
18956 SmallVector<SDValue, 16> V(16, DAG.getConstant(128 >> ShiftAmt,
18958 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
18959 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
18960 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
18963 llvm_unreachable("Unknown shift opcode.");
18966 if (Subtarget->hasInt256() && VT == MVT::v32i8) {
18967 if (Op.getOpcode() == ISD::SHL) {
18968 // Make a large shift.
18969 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
18970 MVT::v16i16, R, ShiftAmt,
18972 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
18973 // Zero out the rightmost bits.
18974 SmallVector<SDValue, 32> V(32,
18975 DAG.getConstant(uint8_t(-1U << ShiftAmt),
18977 return DAG.getNode(ISD::AND, dl, VT, SHL,
18978 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
18980 if (Op.getOpcode() == ISD::SRL) {
18981 // Make a large shift.
18982 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
18983 MVT::v16i16, R, ShiftAmt,
18985 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
18986 // Zero out the leftmost bits.
18987 SmallVector<SDValue, 32> V(32,
18988 DAG.getConstant(uint8_t(-1U) >> ShiftAmt,
18990 return DAG.getNode(ISD::AND, dl, VT, SRL,
18991 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
18993 if (Op.getOpcode() == ISD::SRA) {
18994 if (ShiftAmt == 7) {
18995 // R s>> 7 === R s< 0
18996 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
18997 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
19000 // R s>> a === ((R u>> a) ^ m) - m
19001 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
19002 SmallVector<SDValue, 32> V(32, DAG.getConstant(128 >> ShiftAmt,
19004 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
19005 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
19006 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
19009 llvm_unreachable("Unknown shift opcode.");
19014 // Special case in 32-bit mode, where i64 is expanded into high and low parts.
19015 if (!Subtarget->is64Bit() &&
19016 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) &&
19017 Amt.getOpcode() == ISD::BITCAST &&
19018 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
19019 Amt = Amt.getOperand(0);
19020 unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
19021 VT.getVectorNumElements();
19022 unsigned RatioInLog2 = Log2_32_Ceil(Ratio);
19023 uint64_t ShiftAmt = 0;
19024 for (unsigned i = 0; i != Ratio; ++i) {
19025 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Amt.getOperand(i));
19029 ShiftAmt |= C->getZExtValue() << (i * (1 << (6 - RatioInLog2)));
19031 // Check remaining shift amounts.
19032 for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
19033 uint64_t ShAmt = 0;
19034 for (unsigned j = 0; j != Ratio; ++j) {
19035 ConstantSDNode *C =
19036 dyn_cast<ConstantSDNode>(Amt.getOperand(i + j));
19040 ShAmt |= C->getZExtValue() << (j * (1 << (6 - RatioInLog2)));
19042 if (ShAmt != ShiftAmt)
19045 switch (Op.getOpcode()) {
19047 llvm_unreachable("Unknown shift opcode!");
19049 return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
19052 return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
19055 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
19063 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
19064 const X86Subtarget* Subtarget) {
19065 MVT VT = Op.getSimpleValueType();
19067 SDValue R = Op.getOperand(0);
19068 SDValue Amt = Op.getOperand(1);
19070 if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) ||
19071 VT == MVT::v4i32 || VT == MVT::v8i16 ||
19072 (Subtarget->hasInt256() &&
19073 ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) ||
19074 VT == MVT::v8i32 || VT == MVT::v16i16)) ||
19075 (Subtarget->hasAVX512() && (VT == MVT::v8i64 || VT == MVT::v16i32))) {
19077 EVT EltVT = VT.getVectorElementType();
19079 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Amt)) {
19080 // Check if this build_vector node is doing a splat.
19081 // If so, then set BaseShAmt equal to the splat value.
19082 BaseShAmt = BV->getSplatValue();
19083 if (BaseShAmt && BaseShAmt.getOpcode() == ISD::UNDEF)
19084 BaseShAmt = SDValue();
19086 if (Amt.getOpcode() == ISD::EXTRACT_SUBVECTOR)
19087 Amt = Amt.getOperand(0);
19089 ShuffleVectorSDNode *SVN = dyn_cast<ShuffleVectorSDNode>(Amt);
19090 if (SVN && SVN->isSplat()) {
19091 unsigned SplatIdx = (unsigned)SVN->getSplatIndex();
19092 SDValue InVec = Amt.getOperand(0);
19093 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
19094 assert((SplatIdx < InVec.getValueType().getVectorNumElements()) &&
19095 "Unexpected shuffle index found!");
19096 BaseShAmt = InVec.getOperand(SplatIdx);
19097 } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) {
19098 if (ConstantSDNode *C =
19099 dyn_cast<ConstantSDNode>(InVec.getOperand(2))) {
19100 if (C->getZExtValue() == SplatIdx)
19101 BaseShAmt = InVec.getOperand(1);
19106 // Avoid introducing an extract element from a shuffle.
19107 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InVec,
19108 DAG.getIntPtrConstant(SplatIdx));
19112 if (BaseShAmt.getNode()) {
19113 assert(EltVT.bitsLE(MVT::i64) && "Unexpected element type!");
19114 if (EltVT != MVT::i64 && EltVT.bitsGT(MVT::i32))
19115 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt);
19116 else if (EltVT.bitsLT(MVT::i32))
19117 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt);
19119 switch (Op.getOpcode()) {
19121 llvm_unreachable("Unknown shift opcode!");
19123 switch (VT.SimpleTy) {
19124 default: return SDValue();
19133 return getTargetVShiftNode(X86ISD::VSHLI, dl, VT, R, BaseShAmt, DAG);
19136 switch (VT.SimpleTy) {
19137 default: return SDValue();
19144 return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, R, BaseShAmt, DAG);
19147 switch (VT.SimpleTy) {
19148 default: return SDValue();
19157 return getTargetVShiftNode(X86ISD::VSRLI, dl, VT, R, BaseShAmt, DAG);
19163 // Special case in 32-bit mode, where i64 is expanded into high and low parts.
19164 if (!Subtarget->is64Bit() &&
19165 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64) ||
19166 (Subtarget->hasAVX512() && VT == MVT::v8i64)) &&
19167 Amt.getOpcode() == ISD::BITCAST &&
19168 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
19169 Amt = Amt.getOperand(0);
19170 unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
19171 VT.getVectorNumElements();
19172 std::vector<SDValue> Vals(Ratio);
19173 for (unsigned i = 0; i != Ratio; ++i)
19174 Vals[i] = Amt.getOperand(i);
19175 for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
19176 for (unsigned j = 0; j != Ratio; ++j)
19177 if (Vals[j] != Amt.getOperand(i + j))
19180 switch (Op.getOpcode()) {
19182 llvm_unreachable("Unknown shift opcode!");
19184 return DAG.getNode(X86ISD::VSHL, dl, VT, R, Op.getOperand(1));
19186 return DAG.getNode(X86ISD::VSRL, dl, VT, R, Op.getOperand(1));
19188 return DAG.getNode(X86ISD::VSRA, dl, VT, R, Op.getOperand(1));
19195 static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
19196 SelectionDAG &DAG) {
19197 MVT VT = Op.getSimpleValueType();
19199 SDValue R = Op.getOperand(0);
19200 SDValue Amt = Op.getOperand(1);
19203 assert(VT.isVector() && "Custom lowering only for vector shifts!");
19204 assert(Subtarget->hasSSE2() && "Only custom lower when we have SSE2!");
19206 V = LowerScalarImmediateShift(Op, DAG, Subtarget);
19210 V = LowerScalarVariableShift(Op, DAG, Subtarget);
19214 if (Subtarget->hasAVX512() && (VT == MVT::v16i32 || VT == MVT::v8i64))
19216 // AVX2 has VPSLLV/VPSRAV/VPSRLV.
19217 if (Subtarget->hasInt256()) {
19218 if (Op.getOpcode() == ISD::SRL &&
19219 (VT == MVT::v2i64 || VT == MVT::v4i32 ||
19220 VT == MVT::v4i64 || VT == MVT::v8i32))
19222 if (Op.getOpcode() == ISD::SHL &&
19223 (VT == MVT::v2i64 || VT == MVT::v4i32 ||
19224 VT == MVT::v4i64 || VT == MVT::v8i32))
19226 if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32))
19230 // If possible, lower this packed shift into a vector multiply instead of
19231 // expanding it into a sequence of scalar shifts.
19232 // Do this only if the vector shift count is a constant build_vector.
19233 if (Op.getOpcode() == ISD::SHL &&
19234 (VT == MVT::v8i16 || VT == MVT::v4i32 ||
19235 (Subtarget->hasInt256() && VT == MVT::v16i16)) &&
19236 ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
19237 SmallVector<SDValue, 8> Elts;
19238 EVT SVT = VT.getScalarType();
19239 unsigned SVTBits = SVT.getSizeInBits();
19240 const APInt &One = APInt(SVTBits, 1);
19241 unsigned NumElems = VT.getVectorNumElements();
19243 for (unsigned i=0; i !=NumElems; ++i) {
19244 SDValue Op = Amt->getOperand(i);
19245 if (Op->getOpcode() == ISD::UNDEF) {
19246 Elts.push_back(Op);
19250 ConstantSDNode *ND = cast<ConstantSDNode>(Op);
19251 const APInt &C = APInt(SVTBits, ND->getAPIntValue().getZExtValue());
19252 uint64_t ShAmt = C.getZExtValue();
19253 if (ShAmt >= SVTBits) {
19254 Elts.push_back(DAG.getUNDEF(SVT));
19257 Elts.push_back(DAG.getConstant(One.shl(ShAmt), SVT));
19259 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
19260 return DAG.getNode(ISD::MUL, dl, VT, R, BV);
19263 // Lower SHL with variable shift amount.
19264 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
19265 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT));
19267 Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT));
19268 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op);
19269 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
19270 return DAG.getNode(ISD::MUL, dl, VT, Op, R);
19273 // If possible, lower this shift as a sequence of two shifts by
19274 // constant plus a MOVSS/MOVSD instead of scalarizing it.
19276 // (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
19278 // Could be rewritten as:
19279 // (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
19281 // The advantage is that the two shifts from the example would be
19282 // lowered as X86ISD::VSRLI nodes. This would be cheaper than scalarizing
19283 // the vector shift into four scalar shifts plus four pairs of vector
19285 if ((VT == MVT::v8i16 || VT == MVT::v4i32) &&
19286 ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
19287 unsigned TargetOpcode = X86ISD::MOVSS;
19288 bool CanBeSimplified;
19289 // The splat value for the first packed shift (the 'X' from the example).
19290 SDValue Amt1 = Amt->getOperand(0);
19291 // The splat value for the second packed shift (the 'Y' from the example).
19292 SDValue Amt2 = (VT == MVT::v4i32) ? Amt->getOperand(1) :
19293 Amt->getOperand(2);
19295 // See if it is possible to replace this node with a sequence of
19296 // two shifts followed by a MOVSS/MOVSD
19297 if (VT == MVT::v4i32) {
19298 // Check if it is legal to use a MOVSS.
19299 CanBeSimplified = Amt2 == Amt->getOperand(2) &&
19300 Amt2 == Amt->getOperand(3);
19301 if (!CanBeSimplified) {
19302 // Otherwise, check if we can still simplify this node using a MOVSD.
19303 CanBeSimplified = Amt1 == Amt->getOperand(1) &&
19304 Amt->getOperand(2) == Amt->getOperand(3);
19305 TargetOpcode = X86ISD::MOVSD;
19306 Amt2 = Amt->getOperand(2);
19309 // Do similar checks for the case where the machine value type
19311 CanBeSimplified = Amt1 == Amt->getOperand(1);
19312 for (unsigned i=3; i != 8 && CanBeSimplified; ++i)
19313 CanBeSimplified = Amt2 == Amt->getOperand(i);
19315 if (!CanBeSimplified) {
19316 TargetOpcode = X86ISD::MOVSD;
19317 CanBeSimplified = true;
19318 Amt2 = Amt->getOperand(4);
19319 for (unsigned i=0; i != 4 && CanBeSimplified; ++i)
19320 CanBeSimplified = Amt1 == Amt->getOperand(i);
19321 for (unsigned j=4; j != 8 && CanBeSimplified; ++j)
19322 CanBeSimplified = Amt2 == Amt->getOperand(j);
19326 if (CanBeSimplified && isa<ConstantSDNode>(Amt1) &&
19327 isa<ConstantSDNode>(Amt2)) {
19328 // Replace this node with two shifts followed by a MOVSS/MOVSD.
19329 EVT CastVT = MVT::v4i32;
19331 DAG.getConstant(cast<ConstantSDNode>(Amt1)->getAPIntValue(), VT);
19332 SDValue Shift1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat1);
19334 DAG.getConstant(cast<ConstantSDNode>(Amt2)->getAPIntValue(), VT);
19335 SDValue Shift2 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat2);
19336 if (TargetOpcode == X86ISD::MOVSD)
19337 CastVT = MVT::v2i64;
19338 SDValue BitCast1 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift1);
19339 SDValue BitCast2 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift2);
19340 SDValue Result = getTargetShuffleNode(TargetOpcode, dl, CastVT, BitCast2,
19342 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
19346 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
19347 assert(Subtarget->hasSSE2() && "Need SSE2 for pslli/pcmpeq.");
19350 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT));
19351 Op = DAG.getNode(ISD::BITCAST, dl, VT, Op);
19353 // Turn 'a' into a mask suitable for VSELECT
19354 SDValue VSelM = DAG.getConstant(0x80, VT);
19355 SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
19356 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
19358 SDValue CM1 = DAG.getConstant(0x0f, VT);
19359 SDValue CM2 = DAG.getConstant(0x3f, VT);
19361 // r = VSELECT(r, psllw(r & (char16)15, 4), a);
19362 SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1);
19363 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG);
19364 M = DAG.getNode(ISD::BITCAST, dl, VT, M);
19365 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
19368 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
19369 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
19370 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
19372 // r = VSELECT(r, psllw(r & (char16)63, 2), a);
19373 M = DAG.getNode(ISD::AND, dl, VT, R, CM2);
19374 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG);
19375 M = DAG.getNode(ISD::BITCAST, dl, VT, M);
19376 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
19379 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
19380 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
19381 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
19383 // return VSELECT(r, r+r, a);
19384 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel,
19385 DAG.getNode(ISD::ADD, dl, VT, R, R), R);
19389 // It's worth extending once and using the v8i32 shifts for 16-bit types, but
19390 // the extra overheads to get from v16i8 to v8i32 make the existing SSE
19391 // solution better.
19392 if (Subtarget->hasInt256() && VT == MVT::v8i16) {
19393 MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16;
19395 Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
19396 R = DAG.getNode(ExtOpc, dl, NewVT, R);
19397 Amt = DAG.getNode(ISD::ANY_EXTEND, dl, NewVT, Amt);
19398 return DAG.getNode(ISD::TRUNCATE, dl, VT,
19399 DAG.getNode(Op.getOpcode(), dl, NewVT, R, Amt));
19402 // Decompose 256-bit shifts into smaller 128-bit shifts.
19403 if (VT.is256BitVector()) {
19404 unsigned NumElems = VT.getVectorNumElements();
19405 MVT EltVT = VT.getVectorElementType();
19406 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
19408 // Extract the two vectors
19409 SDValue V1 = Extract128BitVector(R, 0, DAG, dl);
19410 SDValue V2 = Extract128BitVector(R, NumElems/2, DAG, dl);
19412 // Recreate the shift amount vectors
19413 SDValue Amt1, Amt2;
19414 if (Amt.getOpcode() == ISD::BUILD_VECTOR) {
19415 // Constant shift amount
19416 SmallVector<SDValue, 4> Amt1Csts;
19417 SmallVector<SDValue, 4> Amt2Csts;
19418 for (unsigned i = 0; i != NumElems/2; ++i)
19419 Amt1Csts.push_back(Amt->getOperand(i));
19420 for (unsigned i = NumElems/2; i != NumElems; ++i)
19421 Amt2Csts.push_back(Amt->getOperand(i));
19423 Amt1 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt1Csts);
19424 Amt2 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt2Csts);
19426 // Variable shift amount
19427 Amt1 = Extract128BitVector(Amt, 0, DAG, dl);
19428 Amt2 = Extract128BitVector(Amt, NumElems/2, DAG, dl);
19431 // Issue new vector shifts for the smaller types
19432 V1 = DAG.getNode(Op.getOpcode(), dl, NewVT, V1, Amt1);
19433 V2 = DAG.getNode(Op.getOpcode(), dl, NewVT, V2, Amt2);
19435 // Concatenate the result back
19436 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2);
19442 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
19443 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
19444 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
19445 // looks for this combo and may remove the "setcc" instruction if the "setcc"
19446 // has only one use.
19447 SDNode *N = Op.getNode();
19448 SDValue LHS = N->getOperand(0);
19449 SDValue RHS = N->getOperand(1);
19450 unsigned BaseOp = 0;
19453 switch (Op.getOpcode()) {
19454 default: llvm_unreachable("Unknown ovf instruction!");
19456 // A subtract of one will be selected as a INC. Note that INC doesn't
19457 // set CF, so we can't do this for UADDO.
19458 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
19460 BaseOp = X86ISD::INC;
19461 Cond = X86::COND_O;
19464 BaseOp = X86ISD::ADD;
19465 Cond = X86::COND_O;
19468 BaseOp = X86ISD::ADD;
19469 Cond = X86::COND_B;
19472 // A subtract of one will be selected as a DEC. Note that DEC doesn't
19473 // set CF, so we can't do this for USUBO.
19474 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
19476 BaseOp = X86ISD::DEC;
19477 Cond = X86::COND_O;
19480 BaseOp = X86ISD::SUB;
19481 Cond = X86::COND_O;
19484 BaseOp = X86ISD::SUB;
19485 Cond = X86::COND_B;
19488 BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL;
19489 Cond = X86::COND_O;
19491 case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs
19492 if (N->getValueType(0) == MVT::i8) {
19493 BaseOp = X86ISD::UMUL8;
19494 Cond = X86::COND_O;
19497 SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0),
19499 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
19502 DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
19503 DAG.getConstant(X86::COND_O, MVT::i32),
19504 SDValue(Sum.getNode(), 2));
19506 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
19510 // Also sets EFLAGS.
19511 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
19512 SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
19515 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
19516 DAG.getConstant(Cond, MVT::i32),
19517 SDValue(Sum.getNode(), 1));
19519 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
19522 // Sign extension of the low part of vector elements. This may be used either
19523 // when sign extend instructions are not available or if the vector element
19524 // sizes already match the sign-extended size. If the vector elements are in
19525 // their pre-extended size and sign extend instructions are available, that will
19526 // be handled by LowerSIGN_EXTEND.
19527 SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
19528 SelectionDAG &DAG) const {
19530 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
19531 MVT VT = Op.getSimpleValueType();
19533 if (!Subtarget->hasSSE2() || !VT.isVector())
19536 unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
19537 ExtraVT.getScalarType().getSizeInBits();
19539 switch (VT.SimpleTy) {
19540 default: return SDValue();
19543 if (!Subtarget->hasFp256())
19545 if (!Subtarget->hasInt256()) {
19546 // needs to be split
19547 unsigned NumElems = VT.getVectorNumElements();
19549 // Extract the LHS vectors
19550 SDValue LHS = Op.getOperand(0);
19551 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
19552 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
19554 MVT EltVT = VT.getVectorElementType();
19555 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
19557 EVT ExtraEltVT = ExtraVT.getVectorElementType();
19558 unsigned ExtraNumElems = ExtraVT.getVectorNumElements();
19559 ExtraVT = EVT::getVectorVT(*DAG.getContext(), ExtraEltVT,
19561 SDValue Extra = DAG.getValueType(ExtraVT);
19563 LHS1 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, Extra);
19564 LHS2 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, Extra);
19566 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, LHS1, LHS2);
19571 SDValue Op0 = Op.getOperand(0);
19573 // This is a sign extension of some low part of vector elements without
19574 // changing the size of the vector elements themselves:
19575 // Shift-Left + Shift-Right-Algebraic.
19576 SDValue Shl = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Op0,
19578 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Shl, BitsDiff,
19584 /// Returns true if the operand type is exactly twice the native width, and
19585 /// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
19586 /// Used to know whether to use cmpxchg8/16b when expanding atomic operations
19587 /// (otherwise we leave them alone to become __sync_fetch_and_... calls).
19588 bool X86TargetLowering::needsCmpXchgNb(const Type *MemType) const {
19589 unsigned OpWidth = MemType->getPrimitiveSizeInBits();
19592 return !Subtarget->is64Bit(); // FIXME this should be Subtarget.hasCmpxchg8b
19593 else if (OpWidth == 128)
19594 return Subtarget->hasCmpxchg16b();
19599 bool X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
19600 return needsCmpXchgNb(SI->getValueOperand()->getType());
19603 // Note: this turns large loads into lock cmpxchg8b/16b.
19604 // FIXME: On 32 bits x86, fild/movq might be faster than lock cmpxchg8b.
19605 bool X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
19606 auto PTy = cast<PointerType>(LI->getPointerOperand()->getType());
19607 return needsCmpXchgNb(PTy->getElementType());
19610 bool X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
19611 unsigned NativeWidth = Subtarget->is64Bit() ? 64 : 32;
19612 const Type *MemType = AI->getType();
19614 // If the operand is too big, we must see if cmpxchg8/16b is available
19615 // and default to library calls otherwise.
19616 if (MemType->getPrimitiveSizeInBits() > NativeWidth)
19617 return needsCmpXchgNb(MemType);
19619 AtomicRMWInst::BinOp Op = AI->getOperation();
19622 llvm_unreachable("Unknown atomic operation");
19623 case AtomicRMWInst::Xchg:
19624 case AtomicRMWInst::Add:
19625 case AtomicRMWInst::Sub:
19626 // It's better to use xadd, xsub or xchg for these in all cases.
19628 case AtomicRMWInst::Or:
19629 case AtomicRMWInst::And:
19630 case AtomicRMWInst::Xor:
19631 // If the atomicrmw's result isn't actually used, we can just add a "lock"
19632 // prefix to a normal instruction for these operations.
19633 return !AI->use_empty();
19634 case AtomicRMWInst::Nand:
19635 case AtomicRMWInst::Max:
19636 case AtomicRMWInst::Min:
19637 case AtomicRMWInst::UMax:
19638 case AtomicRMWInst::UMin:
19639 // These always require a non-trivial set of data operations on x86. We must
19640 // use a cmpxchg loop.
19645 static bool hasMFENCE(const X86Subtarget& Subtarget) {
19646 // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
19647 // no-sse2). There isn't any reason to disable it if the target processor
19649 return Subtarget.hasSSE2() || Subtarget.is64Bit();
19653 X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
19654 unsigned NativeWidth = Subtarget->is64Bit() ? 64 : 32;
19655 const Type *MemType = AI->getType();
19656 // Accesses larger than the native width are turned into cmpxchg/libcalls, so
19657 // there is no benefit in turning such RMWs into loads, and it is actually
19658 // harmful as it introduces a mfence.
19659 if (MemType->getPrimitiveSizeInBits() > NativeWidth)
19662 auto Builder = IRBuilder<>(AI);
19663 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
19664 auto SynchScope = AI->getSynchScope();
19665 // We must restrict the ordering to avoid generating loads with Release or
19666 // ReleaseAcquire orderings.
19667 auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
19668 auto Ptr = AI->getPointerOperand();
19670 // Before the load we need a fence. Here is an example lifted from
19671 // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
19674 // x.store(1, relaxed);
19675 // r1 = y.fetch_add(0, release);
19677 // y.fetch_add(42, acquire);
19678 // r2 = x.load(relaxed);
19679 // r1 = r2 = 0 is impossible, but becomes possible if the idempotent rmw is
19680 // lowered to just a load without a fence. A mfence flushes the store buffer,
19681 // making the optimization clearly correct.
19682 // FIXME: it is required if isAtLeastRelease(Order) but it is not clear
19683 // otherwise, we might be able to be more agressive on relaxed idempotent
19684 // rmw. In practice, they do not look useful, so we don't try to be
19685 // especially clever.
19686 if (SynchScope == SingleThread) {
19687 // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at
19688 // the IR level, so we must wrap it in an intrinsic.
19690 } else if (hasMFENCE(*Subtarget)) {
19691 Function *MFence = llvm::Intrinsic::getDeclaration(M,
19692 Intrinsic::x86_sse2_mfence);
19693 Builder.CreateCall(MFence);
19695 // FIXME: it might make sense to use a locked operation here but on a
19696 // different cache-line to prevent cache-line bouncing. In practice it
19697 // is probably a small win, and x86 processors without mfence are rare
19698 // enough that we do not bother.
19702 // Finally we can emit the atomic load.
19703 LoadInst *Loaded = Builder.CreateAlignedLoad(Ptr,
19704 AI->getType()->getPrimitiveSizeInBits());
19705 Loaded->setAtomic(Order, SynchScope);
19706 AI->replaceAllUsesWith(Loaded);
19707 AI->eraseFromParent();
19711 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget *Subtarget,
19712 SelectionDAG &DAG) {
19714 AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
19715 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
19716 SynchronizationScope FenceScope = static_cast<SynchronizationScope>(
19717 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
19719 // The only fence that needs an instruction is a sequentially-consistent
19720 // cross-thread fence.
19721 if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
19722 if (hasMFENCE(*Subtarget))
19723 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
19725 SDValue Chain = Op.getOperand(0);
19726 SDValue Zero = DAG.getConstant(0, MVT::i32);
19728 DAG.getRegister(X86::ESP, MVT::i32), // Base
19729 DAG.getTargetConstant(1, MVT::i8), // Scale
19730 DAG.getRegister(0, MVT::i32), // Index
19731 DAG.getTargetConstant(0, MVT::i32), // Disp
19732 DAG.getRegister(0, MVT::i32), // Segment.
19736 SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops);
19737 return SDValue(Res, 0);
19740 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
19741 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
19744 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget,
19745 SelectionDAG &DAG) {
19746 MVT T = Op.getSimpleValueType();
19750 switch(T.SimpleTy) {
19751 default: llvm_unreachable("Invalid value type!");
19752 case MVT::i8: Reg = X86::AL; size = 1; break;
19753 case MVT::i16: Reg = X86::AX; size = 2; break;
19754 case MVT::i32: Reg = X86::EAX; size = 4; break;
19756 assert(Subtarget->is64Bit() && "Node not type legal!");
19757 Reg = X86::RAX; size = 8;
19760 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
19761 Op.getOperand(2), SDValue());
19762 SDValue Ops[] = { cpIn.getValue(0),
19765 DAG.getTargetConstant(size, MVT::i8),
19766 cpIn.getValue(1) };
19767 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
19768 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
19769 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
19773 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
19774 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
19775 MVT::i32, cpOut.getValue(2));
19776 SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
19777 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
19779 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), cpOut);
19780 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
19781 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), EFLAGS.getValue(1));
19785 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget *Subtarget,
19786 SelectionDAG &DAG) {
19787 MVT SrcVT = Op.getOperand(0).getSimpleValueType();
19788 MVT DstVT = Op.getSimpleValueType();
19790 if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) {
19791 assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
19792 if (DstVT != MVT::f64)
19793 // This conversion needs to be expanded.
19796 SDValue InVec = Op->getOperand(0);
19798 unsigned NumElts = SrcVT.getVectorNumElements();
19799 EVT SVT = SrcVT.getVectorElementType();
19801 // Widen the vector in input in the case of MVT::v2i32.
19802 // Example: from MVT::v2i32 to MVT::v4i32.
19803 SmallVector<SDValue, 16> Elts;
19804 for (unsigned i = 0, e = NumElts; i != e; ++i)
19805 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, InVec,
19806 DAG.getIntPtrConstant(i)));
19808 // Explicitly mark the extra elements as Undef.
19809 Elts.append(NumElts, DAG.getUNDEF(SVT));
19811 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
19812 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Elts);
19813 SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV);
19814 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64,
19815 DAG.getIntPtrConstant(0));
19818 assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
19819 Subtarget->hasMMX() && "Unexpected custom BITCAST");
19820 assert((DstVT == MVT::i64 ||
19821 (DstVT.isVector() && DstVT.getSizeInBits()==64)) &&
19822 "Unexpected custom BITCAST");
19823 // i64 <=> MMX conversions are Legal.
19824 if (SrcVT==MVT::i64 && DstVT.isVector())
19826 if (DstVT==MVT::i64 && SrcVT.isVector())
19828 // MMX <=> MMX conversions are Legal.
19829 if (SrcVT.isVector() && DstVT.isVector())
19831 // All other conversions need to be expanded.
19835 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget *Subtarget,
19836 SelectionDAG &DAG) {
19837 SDNode *Node = Op.getNode();
19840 Op = Op.getOperand(0);
19841 EVT VT = Op.getValueType();
19842 assert((VT.is128BitVector() || VT.is256BitVector()) &&
19843 "CTPOP lowering only implemented for 128/256-bit wide vector types");
19845 unsigned NumElts = VT.getVectorNumElements();
19846 EVT EltVT = VT.getVectorElementType();
19847 unsigned Len = EltVT.getSizeInBits();
19849 // This is the vectorized version of the "best" algorithm from
19850 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
19851 // with a minor tweak to use a series of adds + shifts instead of vector
19852 // multiplications. Implemented for the v2i64, v4i64, v4i32, v8i32 types:
19854 // v2i64, v4i64, v4i32 => Only profitable w/ popcnt disabled
19855 // v8i32 => Always profitable
19857 // FIXME: There a couple of possible improvements:
19859 // 1) Support for i8 and i16 vectors (needs measurements if popcnt enabled).
19860 // 2) Use strategies from http://wm.ite.pl/articles/sse-popcount.html
19862 assert(EltVT.isInteger() && (Len == 32 || Len == 64) && Len % 8 == 0 &&
19863 "CTPOP not implemented for this vector element type.");
19865 // X86 canonicalize ANDs to vXi64, generate the appropriate bitcasts to avoid
19866 // extra legalization.
19867 bool NeedsBitcast = EltVT == MVT::i32;
19868 MVT BitcastVT = VT.is256BitVector() ? MVT::v4i64 : MVT::v2i64;
19870 SDValue Cst55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), EltVT);
19871 SDValue Cst33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), EltVT);
19872 SDValue Cst0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), EltVT);
19874 // v = v - ((v >> 1) & 0x55555555...)
19875 SmallVector<SDValue, 8> Ones(NumElts, DAG.getConstant(1, EltVT));
19876 SDValue OnesV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ones);
19877 SDValue Srl = DAG.getNode(ISD::SRL, dl, VT, Op, OnesV);
19879 Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl);
19881 SmallVector<SDValue, 8> Mask55(NumElts, Cst55);
19882 SDValue M55 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask55);
19884 M55 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M55);
19886 SDValue And = DAG.getNode(ISD::AND, dl, Srl.getValueType(), Srl, M55);
19887 if (VT != And.getValueType())
19888 And = DAG.getNode(ISD::BITCAST, dl, VT, And);
19889 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Op, And);
19891 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
19892 SmallVector<SDValue, 8> Mask33(NumElts, Cst33);
19893 SDValue M33 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask33);
19894 SmallVector<SDValue, 8> Twos(NumElts, DAG.getConstant(2, EltVT));
19895 SDValue TwosV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Twos);
19897 Srl = DAG.getNode(ISD::SRL, dl, VT, Sub, TwosV);
19898 if (NeedsBitcast) {
19899 Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl);
19900 M33 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M33);
19901 Sub = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Sub);
19904 SDValue AndRHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Srl, M33);
19905 SDValue AndLHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Sub, M33);
19906 if (VT != AndRHS.getValueType()) {
19907 AndRHS = DAG.getNode(ISD::BITCAST, dl, VT, AndRHS);
19908 AndLHS = DAG.getNode(ISD::BITCAST, dl, VT, AndLHS);
19910 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, AndLHS, AndRHS);
19912 // v = (v + (v >> 4)) & 0x0F0F0F0F...
19913 SmallVector<SDValue, 8> Fours(NumElts, DAG.getConstant(4, EltVT));
19914 SDValue FoursV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Fours);
19915 Srl = DAG.getNode(ISD::SRL, dl, VT, Add, FoursV);
19916 Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl);
19918 SmallVector<SDValue, 8> Mask0F(NumElts, Cst0F);
19919 SDValue M0F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask0F);
19920 if (NeedsBitcast) {
19921 Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add);
19922 M0F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M0F);
19924 And = DAG.getNode(ISD::AND, dl, M0F.getValueType(), Add, M0F);
19925 if (VT != And.getValueType())
19926 And = DAG.getNode(ISD::BITCAST, dl, VT, And);
19928 // The algorithm mentioned above uses:
19929 // v = (v * 0x01010101...) >> (Len - 8)
19931 // Change it to use vector adds + vector shifts which yield faster results on
19932 // Haswell than using vector integer multiplication.
19934 // For i32 elements:
19935 // v = v + (v >> 8)
19936 // v = v + (v >> 16)
19938 // For i64 elements:
19939 // v = v + (v >> 8)
19940 // v = v + (v >> 16)
19941 // v = v + (v >> 32)
19944 SmallVector<SDValue, 8> Csts;
19945 for (unsigned i = 8; i <= Len/2; i *= 2) {
19946 Csts.assign(NumElts, DAG.getConstant(i, EltVT));
19947 SDValue CstsV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Csts);
19948 Srl = DAG.getNode(ISD::SRL, dl, VT, Add, CstsV);
19949 Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl);
19953 // The result is on the least significant 6-bits on i32 and 7-bits on i64.
19954 SDValue Cst3F = DAG.getConstant(APInt(Len, Len == 32 ? 0x3F : 0x7F), EltVT);
19955 SmallVector<SDValue, 8> Cst3FV(NumElts, Cst3F);
19956 SDValue M3F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Cst3FV);
19957 if (NeedsBitcast) {
19958 Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add);
19959 M3F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M3F);
19961 And = DAG.getNode(ISD::AND, dl, M3F.getValueType(), Add, M3F);
19962 if (VT != And.getValueType())
19963 And = DAG.getNode(ISD::BITCAST, dl, VT, And);
19968 static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
19969 SDNode *Node = Op.getNode();
19971 EVT T = Node->getValueType(0);
19972 SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
19973 DAG.getConstant(0, T), Node->getOperand(2));
19974 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
19975 cast<AtomicSDNode>(Node)->getMemoryVT(),
19976 Node->getOperand(0),
19977 Node->getOperand(1), negOp,
19978 cast<AtomicSDNode>(Node)->getMemOperand(),
19979 cast<AtomicSDNode>(Node)->getOrdering(),
19980 cast<AtomicSDNode>(Node)->getSynchScope());
19983 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
19984 SDNode *Node = Op.getNode();
19986 EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
19988 // Convert seq_cst store -> xchg
19989 // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
19990 // FIXME: On 32-bit, store -> fist or movq would be more efficient
19991 // (The only way to get a 16-byte store is cmpxchg16b)
19992 // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
19993 if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent ||
19994 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
19995 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
19996 cast<AtomicSDNode>(Node)->getMemoryVT(),
19997 Node->getOperand(0),
19998 Node->getOperand(1), Node->getOperand(2),
19999 cast<AtomicSDNode>(Node)->getMemOperand(),
20000 cast<AtomicSDNode>(Node)->getOrdering(),
20001 cast<AtomicSDNode>(Node)->getSynchScope());
20002 return Swap.getValue(1);
20004 // Other atomic stores have a simple pattern.
20008 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
20009 EVT VT = Op.getNode()->getSimpleValueType(0);
20011 // Let legalize expand this if it isn't a legal type yet.
20012 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
20015 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
20018 bool ExtraOp = false;
20019 switch (Op.getOpcode()) {
20020 default: llvm_unreachable("Invalid code");
20021 case ISD::ADDC: Opc = X86ISD::ADD; break;
20022 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
20023 case ISD::SUBC: Opc = X86ISD::SUB; break;
20024 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
20028 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
20030 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
20031 Op.getOperand(1), Op.getOperand(2));
20034 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget *Subtarget,
20035 SelectionDAG &DAG) {
20036 assert(Subtarget->isTargetDarwin() && Subtarget->is64Bit());
20038 // For MacOSX, we want to call an alternative entry point: __sincos_stret,
20039 // which returns the values as { float, float } (in XMM0) or
20040 // { double, double } (which is returned in XMM0, XMM1).
20042 SDValue Arg = Op.getOperand(0);
20043 EVT ArgVT = Arg.getValueType();
20044 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
20046 TargetLowering::ArgListTy Args;
20047 TargetLowering::ArgListEntry Entry;
20051 Entry.isSExt = false;
20052 Entry.isZExt = false;
20053 Args.push_back(Entry);
20055 bool isF64 = ArgVT == MVT::f64;
20056 // Only optimize x86_64 for now. i386 is a bit messy. For f32,
20057 // the small struct {f32, f32} is returned in (eax, edx). For f64,
20058 // the results are returned via SRet in memory.
20059 const char *LibcallName = isF64 ? "__sincos_stret" : "__sincosf_stret";
20060 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20061 SDValue Callee = DAG.getExternalSymbol(LibcallName, TLI.getPointerTy());
20063 Type *RetTy = isF64
20064 ? (Type*)StructType::get(ArgTy, ArgTy, nullptr)
20065 : (Type*)VectorType::get(ArgTy, 4);
20067 TargetLowering::CallLoweringInfo CLI(DAG);
20068 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
20069 .setCallee(CallingConv::C, RetTy, Callee, std::move(Args), 0);
20071 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
20074 // Returned in xmm0 and xmm1.
20075 return CallResult.first;
20077 // Returned in bits 0:31 and 32:64 xmm0.
20078 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
20079 CallResult.first, DAG.getIntPtrConstant(0));
20080 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
20081 CallResult.first, DAG.getIntPtrConstant(1));
20082 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
20083 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
20086 /// LowerOperation - Provide custom lowering hooks for some operations.
20088 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
20089 switch (Op.getOpcode()) {
20090 default: llvm_unreachable("Should not custom lower this!");
20091 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG);
20092 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
20093 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
20094 return LowerCMP_SWAP(Op, Subtarget, DAG);
20095 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG);
20096 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);
20097 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG);
20098 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
20099 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
20100 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
20101 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
20102 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
20103 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
20104 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
20105 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
20106 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
20107 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
20108 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
20109 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
20110 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
20111 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
20112 case ISD::SHL_PARTS:
20113 case ISD::SRA_PARTS:
20114 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
20115 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
20116 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
20117 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
20118 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG);
20119 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG);
20120 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG);
20121 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
20122 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
20123 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
20124 case ISD::LOAD: return LowerExtendedLoad(Op, Subtarget, DAG);
20126 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG);
20127 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
20128 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
20129 case ISD::SETCC: return LowerSETCC(Op, DAG);
20130 case ISD::SELECT: return LowerSELECT(Op, DAG);
20131 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
20132 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
20133 case ISD::VASTART: return LowerVASTART(Op, DAG);
20134 case ISD::VAARG: return LowerVAARG(Op, DAG);
20135 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
20136 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, Subtarget, DAG);
20137 case ISD::INTRINSIC_VOID:
20138 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
20139 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
20140 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
20141 case ISD::FRAME_TO_ARGS_OFFSET:
20142 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
20143 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
20144 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
20145 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
20146 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
20147 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
20148 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
20149 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
20150 case ISD::CTLZ: return LowerCTLZ(Op, DAG);
20151 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ_ZERO_UNDEF(Op, DAG);
20152 case ISD::CTTZ: return LowerCTTZ(Op, DAG);
20153 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
20154 case ISD::UMUL_LOHI:
20155 case ISD::SMUL_LOHI: return LowerMUL_LOHI(Op, Subtarget, DAG);
20158 case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
20164 case ISD::UMULO: return LowerXALUO(Op, DAG);
20165 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
20166 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG);
20170 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
20171 case ISD::ADD: return LowerADD(Op, DAG);
20172 case ISD::SUB: return LowerSUB(Op, DAG);
20173 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG);
20177 /// ReplaceNodeResults - Replace a node with an illegal result type
20178 /// with a new node built out of custom code.
20179 void X86TargetLowering::ReplaceNodeResults(SDNode *N,
20180 SmallVectorImpl<SDValue>&Results,
20181 SelectionDAG &DAG) const {
20183 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20184 switch (N->getOpcode()) {
20186 llvm_unreachable("Do not know how to custom type legalize this operation!");
20187 // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
20188 case X86ISD::FMINC:
20190 case X86ISD::FMAXC:
20191 case X86ISD::FMAX: {
20192 EVT VT = N->getValueType(0);
20193 if (VT != MVT::v2f32)
20194 llvm_unreachable("Unexpected type (!= v2f32) on FMIN/FMAX.");
20195 SDValue UNDEF = DAG.getUNDEF(VT);
20196 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
20197 N->getOperand(0), UNDEF);
20198 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
20199 N->getOperand(1), UNDEF);
20200 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));
20203 case ISD::SIGN_EXTEND_INREG:
20208 // We don't want to expand or promote these.
20215 case ISD::UDIVREM: {
20216 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
20217 Results.push_back(V);
20220 case ISD::FP_TO_SINT:
20221 case ISD::FP_TO_UINT: {
20222 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
20224 if (!IsSigned && !isIntegerTypeFTOL(SDValue(N, 0).getValueType()))
20227 std::pair<SDValue,SDValue> Vals =
20228 FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, /*IsReplace=*/ true);
20229 SDValue FIST = Vals.first, StackSlot = Vals.second;
20230 if (FIST.getNode()) {
20231 EVT VT = N->getValueType(0);
20232 // Return a load from the stack slot.
20233 if (StackSlot.getNode())
20234 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot,
20235 MachinePointerInfo(),
20236 false, false, false, 0));
20238 Results.push_back(FIST);
20242 case ISD::UINT_TO_FP: {
20243 assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
20244 if (N->getOperand(0).getValueType() != MVT::v2i32 ||
20245 N->getValueType(0) != MVT::v2f32)
20247 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64,
20249 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
20251 SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias);
20252 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
20253 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias));
20254 Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or);
20255 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
20256 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
20259 case ISD::FP_ROUND: {
20260 if (!TLI.isTypeLegal(N->getOperand(0).getValueType()))
20262 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
20263 Results.push_back(V);
20266 case ISD::INTRINSIC_W_CHAIN: {
20267 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
20269 default : llvm_unreachable("Do not know how to custom type "
20270 "legalize this intrinsic operation!");
20271 case Intrinsic::x86_rdtsc:
20272 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
20274 case Intrinsic::x86_rdtscp:
20275 return getReadTimeStampCounter(N, dl, X86ISD::RDTSCP_DAG, DAG, Subtarget,
20277 case Intrinsic::x86_rdpmc:
20278 return getReadPerformanceCounter(N, dl, DAG, Subtarget, Results);
20281 case ISD::READCYCLECOUNTER: {
20282 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
20285 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
20286 EVT T = N->getValueType(0);
20287 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");
20288 bool Regs64bit = T == MVT::i128;
20289 EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
20290 SDValue cpInL, cpInH;
20291 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
20292 DAG.getConstant(0, HalfT));
20293 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
20294 DAG.getConstant(1, HalfT));
20295 cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
20296 Regs64bit ? X86::RAX : X86::EAX,
20298 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl,
20299 Regs64bit ? X86::RDX : X86::EDX,
20300 cpInH, cpInL.getValue(1));
20301 SDValue swapInL, swapInH;
20302 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
20303 DAG.getConstant(0, HalfT));
20304 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
20305 DAG.getConstant(1, HalfT));
20306 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl,
20307 Regs64bit ? X86::RBX : X86::EBX,
20308 swapInL, cpInH.getValue(1));
20309 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl,
20310 Regs64bit ? X86::RCX : X86::ECX,
20311 swapInH, swapInL.getValue(1));
20312 SDValue Ops[] = { swapInH.getValue(0),
20314 swapInH.getValue(1) };
20315 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
20316 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
20317 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG :
20318 X86ISD::LCMPXCHG8_DAG;
20319 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, T, MMO);
20320 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
20321 Regs64bit ? X86::RAX : X86::EAX,
20322 HalfT, Result.getValue(1));
20323 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
20324 Regs64bit ? X86::RDX : X86::EDX,
20325 HalfT, cpOutL.getValue(2));
20326 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
20328 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
20329 MVT::i32, cpOutH.getValue(2));
20331 DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
20332 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
20333 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
20335 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
20336 Results.push_back(Success);
20337 Results.push_back(EFLAGS.getValue(1));
20340 case ISD::ATOMIC_SWAP:
20341 case ISD::ATOMIC_LOAD_ADD:
20342 case ISD::ATOMIC_LOAD_SUB:
20343 case ISD::ATOMIC_LOAD_AND:
20344 case ISD::ATOMIC_LOAD_OR:
20345 case ISD::ATOMIC_LOAD_XOR:
20346 case ISD::ATOMIC_LOAD_NAND:
20347 case ISD::ATOMIC_LOAD_MIN:
20348 case ISD::ATOMIC_LOAD_MAX:
20349 case ISD::ATOMIC_LOAD_UMIN:
20350 case ISD::ATOMIC_LOAD_UMAX:
20351 case ISD::ATOMIC_LOAD: {
20352 // Delegate to generic TypeLegalization. Situations we can really handle
20353 // should have already been dealt with by AtomicExpandPass.cpp.
20356 case ISD::BITCAST: {
20357 assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
20358 EVT DstVT = N->getValueType(0);
20359 EVT SrcVT = N->getOperand(0)->getValueType(0);
20361 if (SrcVT != MVT::f64 ||
20362 (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8))
20365 unsigned NumElts = DstVT.getVectorNumElements();
20366 EVT SVT = DstVT.getVectorElementType();
20367 EVT WiderVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
20368 SDValue Expanded = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
20369 MVT::v2f64, N->getOperand(0));
20370 SDValue ToVecInt = DAG.getNode(ISD::BITCAST, dl, WiderVT, Expanded);
20372 if (ExperimentalVectorWideningLegalization) {
20373 // If we are legalizing vectors by widening, we already have the desired
20374 // legal vector type, just return it.
20375 Results.push_back(ToVecInt);
20379 SmallVector<SDValue, 8> Elts;
20380 for (unsigned i = 0, e = NumElts; i != e; ++i)
20381 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT,
20382 ToVecInt, DAG.getIntPtrConstant(i)));
20384 Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, dl, DstVT, Elts));
20389 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
20391 default: return nullptr;
20392 case X86ISD::BSF: return "X86ISD::BSF";
20393 case X86ISD::BSR: return "X86ISD::BSR";
20394 case X86ISD::SHLD: return "X86ISD::SHLD";
20395 case X86ISD::SHRD: return "X86ISD::SHRD";
20396 case X86ISD::FAND: return "X86ISD::FAND";
20397 case X86ISD::FANDN: return "X86ISD::FANDN";
20398 case X86ISD::FOR: return "X86ISD::FOR";
20399 case X86ISD::FXOR: return "X86ISD::FXOR";
20400 case X86ISD::FSRL: return "X86ISD::FSRL";
20401 case X86ISD::FILD: return "X86ISD::FILD";
20402 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
20403 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
20404 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
20405 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
20406 case X86ISD::FLD: return "X86ISD::FLD";
20407 case X86ISD::FST: return "X86ISD::FST";
20408 case X86ISD::CALL: return "X86ISD::CALL";
20409 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
20410 case X86ISD::RDTSCP_DAG: return "X86ISD::RDTSCP_DAG";
20411 case X86ISD::RDPMC_DAG: return "X86ISD::RDPMC_DAG";
20412 case X86ISD::BT: return "X86ISD::BT";
20413 case X86ISD::CMP: return "X86ISD::CMP";
20414 case X86ISD::COMI: return "X86ISD::COMI";
20415 case X86ISD::UCOMI: return "X86ISD::UCOMI";
20416 case X86ISD::CMPM: return "X86ISD::CMPM";
20417 case X86ISD::CMPMU: return "X86ISD::CMPMU";
20418 case X86ISD::SETCC: return "X86ISD::SETCC";
20419 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
20420 case X86ISD::FSETCC: return "X86ISD::FSETCC";
20421 case X86ISD::CMOV: return "X86ISD::CMOV";
20422 case X86ISD::BRCOND: return "X86ISD::BRCOND";
20423 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
20424 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
20425 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
20426 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
20427 case X86ISD::Wrapper: return "X86ISD::Wrapper";
20428 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
20429 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
20430 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
20431 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
20432 case X86ISD::PINSRB: return "X86ISD::PINSRB";
20433 case X86ISD::PINSRW: return "X86ISD::PINSRW";
20434 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
20435 case X86ISD::ANDNP: return "X86ISD::ANDNP";
20436 case X86ISD::PSIGN: return "X86ISD::PSIGN";
20437 case X86ISD::BLENDI: return "X86ISD::BLENDI";
20438 case X86ISD::SHRUNKBLEND: return "X86ISD::SHRUNKBLEND";
20439 case X86ISD::SUBUS: return "X86ISD::SUBUS";
20440 case X86ISD::HADD: return "X86ISD::HADD";
20441 case X86ISD::HSUB: return "X86ISD::HSUB";
20442 case X86ISD::FHADD: return "X86ISD::FHADD";
20443 case X86ISD::FHSUB: return "X86ISD::FHSUB";
20444 case X86ISD::UMAX: return "X86ISD::UMAX";
20445 case X86ISD::UMIN: return "X86ISD::UMIN";
20446 case X86ISD::SMAX: return "X86ISD::SMAX";
20447 case X86ISD::SMIN: return "X86ISD::SMIN";
20448 case X86ISD::FMAX: return "X86ISD::FMAX";
20449 case X86ISD::FMIN: return "X86ISD::FMIN";
20450 case X86ISD::FMAXC: return "X86ISD::FMAXC";
20451 case X86ISD::FMINC: return "X86ISD::FMINC";
20452 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
20453 case X86ISD::FRCP: return "X86ISD::FRCP";
20454 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
20455 case X86ISD::TLSBASEADDR: return "X86ISD::TLSBASEADDR";
20456 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
20457 case X86ISD::EH_SJLJ_SETJMP: return "X86ISD::EH_SJLJ_SETJMP";
20458 case X86ISD::EH_SJLJ_LONGJMP: return "X86ISD::EH_SJLJ_LONGJMP";
20459 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
20460 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
20461 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
20462 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r";
20463 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
20464 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
20465 case X86ISD::LCMPXCHG16_DAG: return "X86ISD::LCMPXCHG16_DAG";
20466 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
20467 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
20468 case X86ISD::VZEXT: return "X86ISD::VZEXT";
20469 case X86ISD::VSEXT: return "X86ISD::VSEXT";
20470 case X86ISD::VTRUNC: return "X86ISD::VTRUNC";
20471 case X86ISD::VTRUNCM: return "X86ISD::VTRUNCM";
20472 case X86ISD::VINSERT: return "X86ISD::VINSERT";
20473 case X86ISD::VFPEXT: return "X86ISD::VFPEXT";
20474 case X86ISD::VFPROUND: return "X86ISD::VFPROUND";
20475 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ";
20476 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ";
20477 case X86ISD::VSHL: return "X86ISD::VSHL";
20478 case X86ISD::VSRL: return "X86ISD::VSRL";
20479 case X86ISD::VSRA: return "X86ISD::VSRA";
20480 case X86ISD::VSHLI: return "X86ISD::VSHLI";
20481 case X86ISD::VSRLI: return "X86ISD::VSRLI";
20482 case X86ISD::VSRAI: return "X86ISD::VSRAI";
20483 case X86ISD::CMPP: return "X86ISD::CMPP";
20484 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ";
20485 case X86ISD::PCMPGT: return "X86ISD::PCMPGT";
20486 case X86ISD::PCMPEQM: return "X86ISD::PCMPEQM";
20487 case X86ISD::PCMPGTM: return "X86ISD::PCMPGTM";
20488 case X86ISD::ADD: return "X86ISD::ADD";
20489 case X86ISD::SUB: return "X86ISD::SUB";
20490 case X86ISD::ADC: return "X86ISD::ADC";
20491 case X86ISD::SBB: return "X86ISD::SBB";
20492 case X86ISD::SMUL: return "X86ISD::SMUL";
20493 case X86ISD::UMUL: return "X86ISD::UMUL";
20494 case X86ISD::SMUL8: return "X86ISD::SMUL8";
20495 case X86ISD::UMUL8: return "X86ISD::UMUL8";
20496 case X86ISD::SDIVREM8_SEXT_HREG: return "X86ISD::SDIVREM8_SEXT_HREG";
20497 case X86ISD::UDIVREM8_ZEXT_HREG: return "X86ISD::UDIVREM8_ZEXT_HREG";
20498 case X86ISD::INC: return "X86ISD::INC";
20499 case X86ISD::DEC: return "X86ISD::DEC";
20500 case X86ISD::OR: return "X86ISD::OR";
20501 case X86ISD::XOR: return "X86ISD::XOR";
20502 case X86ISD::AND: return "X86ISD::AND";
20503 case X86ISD::BEXTR: return "X86ISD::BEXTR";
20504 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
20505 case X86ISD::PTEST: return "X86ISD::PTEST";
20506 case X86ISD::TESTP: return "X86ISD::TESTP";
20507 case X86ISD::TESTM: return "X86ISD::TESTM";
20508 case X86ISD::TESTNM: return "X86ISD::TESTNM";
20509 case X86ISD::KORTEST: return "X86ISD::KORTEST";
20510 case X86ISD::PACKSS: return "X86ISD::PACKSS";
20511 case X86ISD::PACKUS: return "X86ISD::PACKUS";
20512 case X86ISD::PALIGNR: return "X86ISD::PALIGNR";
20513 case X86ISD::VALIGN: return "X86ISD::VALIGN";
20514 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
20515 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
20516 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
20517 case X86ISD::SHUFP: return "X86ISD::SHUFP";
20518 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
20519 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD";
20520 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
20521 case X86ISD::MOVLPS: return "X86ISD::MOVLPS";
20522 case X86ISD::MOVLPD: return "X86ISD::MOVLPD";
20523 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
20524 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
20525 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
20526 case X86ISD::MOVSD: return "X86ISD::MOVSD";
20527 case X86ISD::MOVSS: return "X86ISD::MOVSS";
20528 case X86ISD::UNPCKL: return "X86ISD::UNPCKL";
20529 case X86ISD::UNPCKH: return "X86ISD::UNPCKH";
20530 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
20531 case X86ISD::VBROADCASTM: return "X86ISD::VBROADCASTM";
20532 case X86ISD::VEXTRACT: return "X86ISD::VEXTRACT";
20533 case X86ISD::VPERMILPI: return "X86ISD::VPERMILPI";
20534 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128";
20535 case X86ISD::VPERMV: return "X86ISD::VPERMV";
20536 case X86ISD::VPERMV3: return "X86ISD::VPERMV3";
20537 case X86ISD::VPERMIV3: return "X86ISD::VPERMIV3";
20538 case X86ISD::VPERMI: return "X86ISD::VPERMI";
20539 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ";
20540 case X86ISD::PMULDQ: return "X86ISD::PMULDQ";
20541 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
20542 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
20543 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
20544 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
20545 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
20546 case X86ISD::WIN_FTOL: return "X86ISD::WIN_FTOL";
20547 case X86ISD::SAHF: return "X86ISD::SAHF";
20548 case X86ISD::RDRAND: return "X86ISD::RDRAND";
20549 case X86ISD::RDSEED: return "X86ISD::RDSEED";
20550 case X86ISD::FMADD: return "X86ISD::FMADD";
20551 case X86ISD::FMSUB: return "X86ISD::FMSUB";
20552 case X86ISD::FNMADD: return "X86ISD::FNMADD";
20553 case X86ISD::FNMSUB: return "X86ISD::FNMSUB";
20554 case X86ISD::FMADDSUB: return "X86ISD::FMADDSUB";
20555 case X86ISD::FMSUBADD: return "X86ISD::FMSUBADD";
20556 case X86ISD::PCMPESTRI: return "X86ISD::PCMPESTRI";
20557 case X86ISD::PCMPISTRI: return "X86ISD::PCMPISTRI";
20558 case X86ISD::XTEST: return "X86ISD::XTEST";
20559 case X86ISD::COMPRESS: return "X86ISD::COMPRESS";
20560 case X86ISD::EXPAND: return "X86ISD::EXPAND";
20561 case X86ISD::SELECT: return "X86ISD::SELECT";
20562 case X86ISD::ADDSUB: return "X86ISD::ADDSUB";
20563 case X86ISD::RCP28: return "X86ISD::RCP28";
20564 case X86ISD::RSQRT28: return "X86ISD::RSQRT28";
20568 // isLegalAddressingMode - Return true if the addressing mode represented
20569 // by AM is legal for this target, for a load/store of the specified type.
20570 bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
20572 // X86 supports extremely general addressing modes.
20573 CodeModel::Model M = getTargetMachine().getCodeModel();
20574 Reloc::Model R = getTargetMachine().getRelocationModel();
20576 // X86 allows a sign-extended 32-bit immediate field as a displacement.
20577 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
20582 Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine());
20584 // If a reference to this global requires an extra load, we can't fold it.
20585 if (isGlobalStubReference(GVFlags))
20588 // If BaseGV requires a register for the PIC base, we cannot also have a
20589 // BaseReg specified.
20590 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
20593 // If lower 4G is not available, then we must use rip-relative addressing.
20594 if ((M != CodeModel::Small || R != Reloc::Static) &&
20595 Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1))
20599 switch (AM.Scale) {
20605 // These scales always work.
20610 // These scales are formed with basereg+scalereg. Only accept if there is
20615 default: // Other stuff never works.
20622 bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
20623 unsigned Bits = Ty->getScalarSizeInBits();
20625 // 8-bit shifts are always expensive, but versions with a scalar amount aren't
20626 // particularly cheaper than those without.
20630 // On AVX2 there are new vpsllv[dq] instructions (and other shifts), that make
20631 // variable shifts just as cheap as scalar ones.
20632 if (Subtarget->hasInt256() && (Bits == 32 || Bits == 64))
20635 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
20636 // fully general vector.
20640 bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
20641 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
20643 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
20644 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
20645 return NumBits1 > NumBits2;
20648 bool X86TargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
20649 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
20652 if (!isTypeLegal(EVT::getEVT(Ty1)))
20655 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
20657 // Assuming the caller doesn't have a zeroext or signext return parameter,
20658 // truncation all the way down to i1 is valid.
20662 bool X86TargetLowering::isLegalICmpImmediate(int64_t Imm) const {
20663 return isInt<32>(Imm);
20666 bool X86TargetLowering::isLegalAddImmediate(int64_t Imm) const {
20667 // Can also use sub to handle negated immediates.
20668 return isInt<32>(Imm);
20671 bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
20672 if (!VT1.isInteger() || !VT2.isInteger())
20674 unsigned NumBits1 = VT1.getSizeInBits();
20675 unsigned NumBits2 = VT2.getSizeInBits();
20676 return NumBits1 > NumBits2;
20679 bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
20680 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
20681 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit();
20684 bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
20685 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
20686 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit();
20689 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
20690 EVT VT1 = Val.getValueType();
20691 if (isZExtFree(VT1, VT2))
20694 if (Val.getOpcode() != ISD::LOAD)
20697 if (!VT1.isSimple() || !VT1.isInteger() ||
20698 !VT2.isSimple() || !VT2.isInteger())
20701 switch (VT1.getSimpleVT().SimpleTy) {
20706 // X86 has 8, 16, and 32-bit zero-extending loads.
20713 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue) const { return true; }
20716 X86TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
20717 if (!(Subtarget->hasFMA() || Subtarget->hasFMA4()))
20720 VT = VT.getScalarType();
20722 if (!VT.isSimple())
20725 switch (VT.getSimpleVT().SimpleTy) {
20736 bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const {
20737 // i16 instructions are longer (0x66 prefix) and potentially slower.
20738 return !(VT1 == MVT::i32 && VT2 == MVT::i16);
20741 /// isShuffleMaskLegal - Targets can use this to indicate that they only
20742 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
20743 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
20744 /// are assumed to be legal.
20746 X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
20748 if (!VT.isSimple())
20751 MVT SVT = VT.getSimpleVT();
20753 // Very little shuffling can be done for 64-bit vectors right now.
20754 if (VT.getSizeInBits() == 64)
20757 // This is an experimental legality test that is tailored to match the
20758 // legality test of the experimental lowering more closely. They are gated
20759 // separately to ease testing of performance differences.
20760 if (ExperimentalVectorShuffleLegality)
20761 // We only care that the types being shuffled are legal. The lowering can
20762 // handle any possible shuffle mask that results.
20763 return isTypeLegal(SVT);
20765 // If this is a single-input shuffle with no 128 bit lane crossings we can
20766 // lower it into pshufb.
20767 if ((SVT.is128BitVector() && Subtarget->hasSSSE3()) ||
20768 (SVT.is256BitVector() && Subtarget->hasInt256())) {
20769 bool isLegal = true;
20770 for (unsigned I = 0, E = M.size(); I != E; ++I) {
20771 if (M[I] >= (int)SVT.getVectorNumElements() ||
20772 ShuffleCrosses128bitLane(SVT, I, M[I])) {
20781 // FIXME: blends, shifts.
20782 return (SVT.getVectorNumElements() == 2 ||
20783 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
20784 isMOVLMask(M, SVT) ||
20785 isCommutedMOVLMask(M, SVT) ||
20786 isMOVHLPSMask(M, SVT) ||
20787 isSHUFPMask(M, SVT) ||
20788 isSHUFPMask(M, SVT, /* Commuted */ true) ||
20789 isPSHUFDMask(M, SVT) ||
20790 isPSHUFDMask(M, SVT, /* SecondOperand */ true) ||
20791 isPSHUFHWMask(M, SVT, Subtarget->hasInt256()) ||
20792 isPSHUFLWMask(M, SVT, Subtarget->hasInt256()) ||
20793 isPALIGNRMask(M, SVT, Subtarget) ||
20794 isUNPCKLMask(M, SVT, Subtarget->hasInt256()) ||
20795 isUNPCKHMask(M, SVT, Subtarget->hasInt256()) ||
20796 isUNPCKL_v_undef_Mask(M, SVT, Subtarget->hasInt256()) ||
20797 isUNPCKH_v_undef_Mask(M, SVT, Subtarget->hasInt256()) ||
20798 isBlendMask(M, SVT, Subtarget->hasSSE41(), Subtarget->hasInt256()) ||
20799 (Subtarget->hasSSE41() && isINSERTPSMask(M, SVT)));
20803 X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
20805 if (!VT.isSimple())
20808 MVT SVT = VT.getSimpleVT();
20810 // This is an experimental legality test that is tailored to match the
20811 // legality test of the experimental lowering more closely. They are gated
20812 // separately to ease testing of performance differences.
20813 if (ExperimentalVectorShuffleLegality)
20814 // The new vector shuffle lowering is very good at managing zero-inputs.
20815 return isShuffleMaskLegal(Mask, VT);
20817 unsigned NumElts = SVT.getVectorNumElements();
20818 // FIXME: This collection of masks seems suspect.
20821 if (NumElts == 4 && SVT.is128BitVector()) {
20822 return (isMOVLMask(Mask, SVT) ||
20823 isCommutedMOVLMask(Mask, SVT, true) ||
20824 isSHUFPMask(Mask, SVT) ||
20825 isSHUFPMask(Mask, SVT, /* Commuted */ true) ||
20826 isBlendMask(Mask, SVT, Subtarget->hasSSE41(),
20827 Subtarget->hasInt256()));
20832 //===----------------------------------------------------------------------===//
20833 // X86 Scheduler Hooks
20834 //===----------------------------------------------------------------------===//
20836 /// Utility function to emit xbegin specifying the start of an RTM region.
20837 static MachineBasicBlock *EmitXBegin(MachineInstr *MI, MachineBasicBlock *MBB,
20838 const TargetInstrInfo *TII) {
20839 DebugLoc DL = MI->getDebugLoc();
20841 const BasicBlock *BB = MBB->getBasicBlock();
20842 MachineFunction::iterator I = MBB;
20845 // For the v = xbegin(), we generate
20856 MachineBasicBlock *thisMBB = MBB;
20857 MachineFunction *MF = MBB->getParent();
20858 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
20859 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
20860 MF->insert(I, mainMBB);
20861 MF->insert(I, sinkMBB);
20863 // Transfer the remainder of BB and its successor edges to sinkMBB.
20864 sinkMBB->splice(sinkMBB->begin(), MBB,
20865 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
20866 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
20870 // # fallthrough to mainMBB
20871 // # abortion to sinkMBB
20872 BuildMI(thisMBB, DL, TII->get(X86::XBEGIN_4)).addMBB(sinkMBB);
20873 thisMBB->addSuccessor(mainMBB);
20874 thisMBB->addSuccessor(sinkMBB);
20878 BuildMI(mainMBB, DL, TII->get(X86::MOV32ri), X86::EAX).addImm(-1);
20879 mainMBB->addSuccessor(sinkMBB);
20882 // EAX is live into the sinkMBB
20883 sinkMBB->addLiveIn(X86::EAX);
20884 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
20885 TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
20888 MI->eraseFromParent();
20892 // FIXME: When we get size specific XMM0 registers, i.e. XMM0_V16I8
20893 // or XMM0_V32I8 in AVX all of this code can be replaced with that
20894 // in the .td file.
20895 static MachineBasicBlock *EmitPCMPSTRM(MachineInstr *MI, MachineBasicBlock *BB,
20896 const TargetInstrInfo *TII) {
20898 switch (MI->getOpcode()) {
20899 default: llvm_unreachable("illegal opcode!");
20900 case X86::PCMPISTRM128REG: Opc = X86::PCMPISTRM128rr; break;
20901 case X86::VPCMPISTRM128REG: Opc = X86::VPCMPISTRM128rr; break;
20902 case X86::PCMPISTRM128MEM: Opc = X86::PCMPISTRM128rm; break;
20903 case X86::VPCMPISTRM128MEM: Opc = X86::VPCMPISTRM128rm; break;
20904 case X86::PCMPESTRM128REG: Opc = X86::PCMPESTRM128rr; break;
20905 case X86::VPCMPESTRM128REG: Opc = X86::VPCMPESTRM128rr; break;
20906 case X86::PCMPESTRM128MEM: Opc = X86::PCMPESTRM128rm; break;
20907 case X86::VPCMPESTRM128MEM: Opc = X86::VPCMPESTRM128rm; break;
20910 DebugLoc dl = MI->getDebugLoc();
20911 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
20913 unsigned NumArgs = MI->getNumOperands();
20914 for (unsigned i = 1; i < NumArgs; ++i) {
20915 MachineOperand &Op = MI->getOperand(i);
20916 if (!(Op.isReg() && Op.isImplicit()))
20917 MIB.addOperand(Op);
20919 if (MI->hasOneMemOperand())
20920 MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
20922 BuildMI(*BB, MI, dl,
20923 TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
20924 .addReg(X86::XMM0);
20926 MI->eraseFromParent();
20930 // FIXME: Custom handling because TableGen doesn't support multiple implicit
20931 // defs in an instruction pattern
20932 static MachineBasicBlock *EmitPCMPSTRI(MachineInstr *MI, MachineBasicBlock *BB,
20933 const TargetInstrInfo *TII) {
20935 switch (MI->getOpcode()) {
20936 default: llvm_unreachable("illegal opcode!");
20937 case X86::PCMPISTRIREG: Opc = X86::PCMPISTRIrr; break;
20938 case X86::VPCMPISTRIREG: Opc = X86::VPCMPISTRIrr; break;
20939 case X86::PCMPISTRIMEM: Opc = X86::PCMPISTRIrm; break;
20940 case X86::VPCMPISTRIMEM: Opc = X86::VPCMPISTRIrm; break;
20941 case X86::PCMPESTRIREG: Opc = X86::PCMPESTRIrr; break;
20942 case X86::VPCMPESTRIREG: Opc = X86::VPCMPESTRIrr; break;
20943 case X86::PCMPESTRIMEM: Opc = X86::PCMPESTRIrm; break;
20944 case X86::VPCMPESTRIMEM: Opc = X86::VPCMPESTRIrm; break;
20947 DebugLoc dl = MI->getDebugLoc();
20948 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
20950 unsigned NumArgs = MI->getNumOperands(); // remove the results
20951 for (unsigned i = 1; i < NumArgs; ++i) {
20952 MachineOperand &Op = MI->getOperand(i);
20953 if (!(Op.isReg() && Op.isImplicit()))
20954 MIB.addOperand(Op);
20956 if (MI->hasOneMemOperand())
20957 MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
20959 BuildMI(*BB, MI, dl,
20960 TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
20963 MI->eraseFromParent();
20967 static MachineBasicBlock *EmitMonitor(MachineInstr *MI, MachineBasicBlock *BB,
20968 const X86Subtarget *Subtarget) {
20969 DebugLoc dl = MI->getDebugLoc();
20970 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
20971 // Address into RAX/EAX, other two args into ECX, EDX.
20972 unsigned MemOpc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r;
20973 unsigned MemReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
20974 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(MemOpc), MemReg);
20975 for (int i = 0; i < X86::AddrNumOperands; ++i)
20976 MIB.addOperand(MI->getOperand(i));
20978 unsigned ValOps = X86::AddrNumOperands;
20979 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX)
20980 .addReg(MI->getOperand(ValOps).getReg());
20981 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EDX)
20982 .addReg(MI->getOperand(ValOps+1).getReg());
20984 // The instruction doesn't actually take any operands though.
20985 BuildMI(*BB, MI, dl, TII->get(X86::MONITORrrr));
20987 MI->eraseFromParent(); // The pseudo is gone now.
20991 MachineBasicBlock *
20992 X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
20993 MachineBasicBlock *MBB) const {
20994 // Emit va_arg instruction on X86-64.
20996 // Operands to this pseudo-instruction:
20997 // 0 ) Output : destination address (reg)
20998 // 1-5) Input : va_list address (addr, i64mem)
20999 // 6 ) ArgSize : Size (in bytes) of vararg type
21000 // 7 ) ArgMode : 0=overflow only, 1=use gp_offset, 2=use fp_offset
21001 // 8 ) Align : Alignment of type
21002 // 9 ) EFLAGS (implicit-def)
21004 assert(MI->getNumOperands() == 10 && "VAARG_64 should have 10 operands!");
21005 assert(X86::AddrNumOperands == 5 && "VAARG_64 assumes 5 address operands");
21007 unsigned DestReg = MI->getOperand(0).getReg();
21008 MachineOperand &Base = MI->getOperand(1);
21009 MachineOperand &Scale = MI->getOperand(2);
21010 MachineOperand &Index = MI->getOperand(3);
21011 MachineOperand &Disp = MI->getOperand(4);
21012 MachineOperand &Segment = MI->getOperand(5);
21013 unsigned ArgSize = MI->getOperand(6).getImm();
21014 unsigned ArgMode = MI->getOperand(7).getImm();
21015 unsigned Align = MI->getOperand(8).getImm();
21017 // Memory Reference
21018 assert(MI->hasOneMemOperand() && "Expected VAARG_64 to have one memoperand");
21019 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
21020 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
21022 // Machine Information
21023 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21024 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
21025 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64);
21026 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
21027 DebugLoc DL = MI->getDebugLoc();
21029 // struct va_list {
21032 // i64 overflow_area (address)
21033 // i64 reg_save_area (address)
21035 // sizeof(va_list) = 24
21036 // alignment(va_list) = 8
21038 unsigned TotalNumIntRegs = 6;
21039 unsigned TotalNumXMMRegs = 8;
21040 bool UseGPOffset = (ArgMode == 1);
21041 bool UseFPOffset = (ArgMode == 2);
21042 unsigned MaxOffset = TotalNumIntRegs * 8 +
21043 (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
21045 /* Align ArgSize to a multiple of 8 */
21046 unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
21047 bool NeedsAlign = (Align > 8);
21049 MachineBasicBlock *thisMBB = MBB;
21050 MachineBasicBlock *overflowMBB;
21051 MachineBasicBlock *offsetMBB;
21052 MachineBasicBlock *endMBB;
21054 unsigned OffsetDestReg = 0; // Argument address computed by offsetMBB
21055 unsigned OverflowDestReg = 0; // Argument address computed by overflowMBB
21056 unsigned OffsetReg = 0;
21058 if (!UseGPOffset && !UseFPOffset) {
21059 // If we only pull from the overflow region, we don't create a branch.
21060 // We don't need to alter control flow.
21061 OffsetDestReg = 0; // unused
21062 OverflowDestReg = DestReg;
21064 offsetMBB = nullptr;
21065 overflowMBB = thisMBB;
21068 // First emit code to check if gp_offset (or fp_offset) is below the bound.
21069 // If so, pull the argument from reg_save_area. (branch to offsetMBB)
21070 // If not, pull from overflow_area. (branch to overflowMBB)
21075 // offsetMBB overflowMBB
21080 // Registers for the PHI in endMBB
21081 OffsetDestReg = MRI.createVirtualRegister(AddrRegClass);
21082 OverflowDestReg = MRI.createVirtualRegister(AddrRegClass);
21084 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
21085 MachineFunction *MF = MBB->getParent();
21086 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21087 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21088 endMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21090 MachineFunction::iterator MBBIter = MBB;
21093 // Insert the new basic blocks
21094 MF->insert(MBBIter, offsetMBB);
21095 MF->insert(MBBIter, overflowMBB);
21096 MF->insert(MBBIter, endMBB);
21098 // Transfer the remainder of MBB and its successor edges to endMBB.
21099 endMBB->splice(endMBB->begin(), thisMBB,
21100 std::next(MachineBasicBlock::iterator(MI)), thisMBB->end());
21101 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
21103 // Make offsetMBB and overflowMBB successors of thisMBB
21104 thisMBB->addSuccessor(offsetMBB);
21105 thisMBB->addSuccessor(overflowMBB);
21107 // endMBB is a successor of both offsetMBB and overflowMBB
21108 offsetMBB->addSuccessor(endMBB);
21109 overflowMBB->addSuccessor(endMBB);
21111 // Load the offset value into a register
21112 OffsetReg = MRI.createVirtualRegister(OffsetRegClass);
21113 BuildMI(thisMBB, DL, TII->get(X86::MOV32rm), OffsetReg)
21117 .addDisp(Disp, UseFPOffset ? 4 : 0)
21118 .addOperand(Segment)
21119 .setMemRefs(MMOBegin, MMOEnd);
21121 // Check if there is enough room left to pull this argument.
21122 BuildMI(thisMBB, DL, TII->get(X86::CMP32ri))
21124 .addImm(MaxOffset + 8 - ArgSizeA8);
21126 // Branch to "overflowMBB" if offset >= max
21127 // Fall through to "offsetMBB" otherwise
21128 BuildMI(thisMBB, DL, TII->get(X86::GetCondBranchFromCond(X86::COND_AE)))
21129 .addMBB(overflowMBB);
21132 // In offsetMBB, emit code to use the reg_save_area.
21134 assert(OffsetReg != 0);
21136 // Read the reg_save_area address.
21137 unsigned RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
21138 BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg)
21143 .addOperand(Segment)
21144 .setMemRefs(MMOBegin, MMOEnd);
21146 // Zero-extend the offset
21147 unsigned OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
21148 BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
21151 .addImm(X86::sub_32bit);
21153 // Add the offset to the reg_save_area to get the final address.
21154 BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg)
21155 .addReg(OffsetReg64)
21156 .addReg(RegSaveReg);
21158 // Compute the offset for the next argument
21159 unsigned NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
21160 BuildMI(offsetMBB, DL, TII->get(X86::ADD32ri), NextOffsetReg)
21162 .addImm(UseFPOffset ? 16 : 8);
21164 // Store it back into the va_list.
21165 BuildMI(offsetMBB, DL, TII->get(X86::MOV32mr))
21169 .addDisp(Disp, UseFPOffset ? 4 : 0)
21170 .addOperand(Segment)
21171 .addReg(NextOffsetReg)
21172 .setMemRefs(MMOBegin, MMOEnd);
21175 BuildMI(offsetMBB, DL, TII->get(X86::JMP_1))
21180 // Emit code to use overflow area
21183 // Load the overflow_area address into a register.
21184 unsigned OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
21185 BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg)
21190 .addOperand(Segment)
21191 .setMemRefs(MMOBegin, MMOEnd);
21193 // If we need to align it, do so. Otherwise, just copy the address
21194 // to OverflowDestReg.
21196 // Align the overflow address
21197 assert((Align & (Align-1)) == 0 && "Alignment must be a power of 2");
21198 unsigned TmpReg = MRI.createVirtualRegister(AddrRegClass);
21200 // aligned_addr = (addr + (align-1)) & ~(align-1)
21201 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg)
21202 .addReg(OverflowAddrReg)
21205 BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg)
21207 .addImm(~(uint64_t)(Align-1));
21209 BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg)
21210 .addReg(OverflowAddrReg);
21213 // Compute the next overflow address after this argument.
21214 // (the overflow address should be kept 8-byte aligned)
21215 unsigned NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
21216 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg)
21217 .addReg(OverflowDestReg)
21218 .addImm(ArgSizeA8);
21220 // Store the new overflow address.
21221 BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr))
21226 .addOperand(Segment)
21227 .addReg(NextAddrReg)
21228 .setMemRefs(MMOBegin, MMOEnd);
21230 // If we branched, emit the PHI to the front of endMBB.
21232 BuildMI(*endMBB, endMBB->begin(), DL,
21233 TII->get(X86::PHI), DestReg)
21234 .addReg(OffsetDestReg).addMBB(offsetMBB)
21235 .addReg(OverflowDestReg).addMBB(overflowMBB);
21238 // Erase the pseudo instruction
21239 MI->eraseFromParent();
21244 MachineBasicBlock *
21245 X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
21247 MachineBasicBlock *MBB) const {
21248 // Emit code to save XMM registers to the stack. The ABI says that the
21249 // number of registers to save is given in %al, so it's theoretically
21250 // possible to do an indirect jump trick to avoid saving all of them,
21251 // however this code takes a simpler approach and just executes all
21252 // of the stores if %al is non-zero. It's less code, and it's probably
21253 // easier on the hardware branch predictor, and stores aren't all that
21254 // expensive anyway.
21256 // Create the new basic blocks. One block contains all the XMM stores,
21257 // and one block is the final destination regardless of whether any
21258 // stores were performed.
21259 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
21260 MachineFunction *F = MBB->getParent();
21261 MachineFunction::iterator MBBIter = MBB;
21263 MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB);
21264 MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB);
21265 F->insert(MBBIter, XMMSaveMBB);
21266 F->insert(MBBIter, EndMBB);
21268 // Transfer the remainder of MBB and its successor edges to EndMBB.
21269 EndMBB->splice(EndMBB->begin(), MBB,
21270 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
21271 EndMBB->transferSuccessorsAndUpdatePHIs(MBB);
21273 // The original block will now fall through to the XMM save block.
21274 MBB->addSuccessor(XMMSaveMBB);
21275 // The XMMSaveMBB will fall through to the end block.
21276 XMMSaveMBB->addSuccessor(EndMBB);
21278 // Now add the instructions.
21279 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21280 DebugLoc DL = MI->getDebugLoc();
21282 unsigned CountReg = MI->getOperand(0).getReg();
21283 int64_t RegSaveFrameIndex = MI->getOperand(1).getImm();
21284 int64_t VarArgsFPOffset = MI->getOperand(2).getImm();
21286 if (!Subtarget->isTargetWin64()) {
21287 // If %al is 0, branch around the XMM save block.
21288 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg);
21289 BuildMI(MBB, DL, TII->get(X86::JE_1)).addMBB(EndMBB);
21290 MBB->addSuccessor(EndMBB);
21293 // Make sure the last operand is EFLAGS, which gets clobbered by the branch
21294 // that was just emitted, but clearly shouldn't be "saved".
21295 assert((MI->getNumOperands() <= 3 ||
21296 !MI->getOperand(MI->getNumOperands() - 1).isReg() ||
21297 MI->getOperand(MI->getNumOperands() - 1).getReg() == X86::EFLAGS)
21298 && "Expected last argument to be EFLAGS");
21299 unsigned MOVOpc = Subtarget->hasFp256() ? X86::VMOVAPSmr : X86::MOVAPSmr;
21300 // In the XMM save block, save all the XMM argument registers.
21301 for (int i = 3, e = MI->getNumOperands() - 1; i != e; ++i) {
21302 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset;
21303 MachineMemOperand *MMO =
21304 F->getMachineMemOperand(
21305 MachinePointerInfo::getFixedStack(RegSaveFrameIndex, Offset),
21306 MachineMemOperand::MOStore,
21307 /*Size=*/16, /*Align=*/16);
21308 BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc))
21309 .addFrameIndex(RegSaveFrameIndex)
21310 .addImm(/*Scale=*/1)
21311 .addReg(/*IndexReg=*/0)
21312 .addImm(/*Disp=*/Offset)
21313 .addReg(/*Segment=*/0)
21314 .addReg(MI->getOperand(i).getReg())
21315 .addMemOperand(MMO);
21318 MI->eraseFromParent(); // The pseudo instruction is gone now.
21323 // The EFLAGS operand of SelectItr might be missing a kill marker
21324 // because there were multiple uses of EFLAGS, and ISel didn't know
21325 // which to mark. Figure out whether SelectItr should have had a
21326 // kill marker, and set it if it should. Returns the correct kill
21328 static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
21329 MachineBasicBlock* BB,
21330 const TargetRegisterInfo* TRI) {
21331 // Scan forward through BB for a use/def of EFLAGS.
21332 MachineBasicBlock::iterator miI(std::next(SelectItr));
21333 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
21334 const MachineInstr& mi = *miI;
21335 if (mi.readsRegister(X86::EFLAGS))
21337 if (mi.definesRegister(X86::EFLAGS))
21338 break; // Should have kill-flag - update below.
21341 // If we hit the end of the block, check whether EFLAGS is live into a
21343 if (miI == BB->end()) {
21344 for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
21345 sEnd = BB->succ_end();
21346 sItr != sEnd; ++sItr) {
21347 MachineBasicBlock* succ = *sItr;
21348 if (succ->isLiveIn(X86::EFLAGS))
21353 // We found a def, or hit the end of the basic block and EFLAGS wasn't live
21354 // out. SelectMI should have a kill flag on EFLAGS.
21355 SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
21359 MachineBasicBlock *
21360 X86TargetLowering::EmitLoweredSelect(MachineInstr *MI,
21361 MachineBasicBlock *BB) const {
21362 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21363 DebugLoc DL = MI->getDebugLoc();
21365 // To "insert" a SELECT_CC instruction, we actually have to insert the
21366 // diamond control-flow pattern. The incoming instruction knows the
21367 // destination vreg to set, the condition code register to branch on, the
21368 // true/false values to select between, and a branch opcode to use.
21369 const BasicBlock *LLVM_BB = BB->getBasicBlock();
21370 MachineFunction::iterator It = BB;
21376 // cmpTY ccX, r1, r2
21378 // fallthrough --> copy0MBB
21379 MachineBasicBlock *thisMBB = BB;
21380 MachineFunction *F = BB->getParent();
21381 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
21382 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
21383 F->insert(It, copy0MBB);
21384 F->insert(It, sinkMBB);
21386 // If the EFLAGS register isn't dead in the terminator, then claim that it's
21387 // live into the sink and copy blocks.
21388 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
21389 if (!MI->killsRegister(X86::EFLAGS) &&
21390 !checkAndUpdateEFLAGSKill(MI, BB, TRI)) {
21391 copy0MBB->addLiveIn(X86::EFLAGS);
21392 sinkMBB->addLiveIn(X86::EFLAGS);
21395 // Transfer the remainder of BB and its successor edges to sinkMBB.
21396 sinkMBB->splice(sinkMBB->begin(), BB,
21397 std::next(MachineBasicBlock::iterator(MI)), BB->end());
21398 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
21400 // Add the true and fallthrough blocks as its successors.
21401 BB->addSuccessor(copy0MBB);
21402 BB->addSuccessor(sinkMBB);
21404 // Create the conditional branch instruction.
21406 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
21407 BuildMI(BB, DL, TII->get(Opc)).addMBB(sinkMBB);
21410 // %FalseValue = ...
21411 // # fallthrough to sinkMBB
21412 copy0MBB->addSuccessor(sinkMBB);
21415 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
21417 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
21418 TII->get(X86::PHI), MI->getOperand(0).getReg())
21419 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
21420 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
21422 MI->eraseFromParent(); // The pseudo instruction is gone now.
21426 MachineBasicBlock *
21427 X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI,
21428 MachineBasicBlock *BB) const {
21429 MachineFunction *MF = BB->getParent();
21430 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21431 DebugLoc DL = MI->getDebugLoc();
21432 const BasicBlock *LLVM_BB = BB->getBasicBlock();
21434 assert(MF->shouldSplitStack());
21436 const bool Is64Bit = Subtarget->is64Bit();
21437 const bool IsLP64 = Subtarget->isTarget64BitLP64();
21439 const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
21440 const unsigned TlsOffset = IsLP64 ? 0x70 : Is64Bit ? 0x40 : 0x30;
21443 // ... [Till the alloca]
21444 // If stacklet is not large enough, jump to mallocMBB
21447 // Allocate by subtracting from RSP
21448 // Jump to continueMBB
21451 // Allocate by call to runtime
21455 // [rest of original BB]
21458 MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21459 MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21460 MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21462 MachineRegisterInfo &MRI = MF->getRegInfo();
21463 const TargetRegisterClass *AddrRegClass =
21464 getRegClassFor(getPointerTy());
21466 unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
21467 bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
21468 tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
21469 SPLimitVReg = MRI.createVirtualRegister(AddrRegClass),
21470 sizeVReg = MI->getOperand(1).getReg(),
21471 physSPReg = IsLP64 || Subtarget->isTargetNaCl64() ? X86::RSP : X86::ESP;
21473 MachineFunction::iterator MBBIter = BB;
21476 MF->insert(MBBIter, bumpMBB);
21477 MF->insert(MBBIter, mallocMBB);
21478 MF->insert(MBBIter, continueMBB);
21480 continueMBB->splice(continueMBB->begin(), BB,
21481 std::next(MachineBasicBlock::iterator(MI)), BB->end());
21482 continueMBB->transferSuccessorsAndUpdatePHIs(BB);
21484 // Add code to the main basic block to check if the stack limit has been hit,
21485 // and if so, jump to mallocMBB otherwise to bumpMBB.
21486 BuildMI(BB, DL, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
21487 BuildMI(BB, DL, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
21488 .addReg(tmpSPVReg).addReg(sizeVReg);
21489 BuildMI(BB, DL, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
21490 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
21491 .addReg(SPLimitVReg);
21492 BuildMI(BB, DL, TII->get(X86::JG_1)).addMBB(mallocMBB);
21494 // bumpMBB simply decreases the stack pointer, since we know the current
21495 // stacklet has enough space.
21496 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), physSPReg)
21497 .addReg(SPLimitVReg);
21498 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
21499 .addReg(SPLimitVReg);
21500 BuildMI(bumpMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
21502 // Calls into a routine in libgcc to allocate more space from the heap.
21503 const uint32_t *RegMask =
21504 Subtarget->getRegisterInfo()->getCallPreservedMask(CallingConv::C);
21506 BuildMI(mallocMBB, DL, TII->get(X86::MOV64rr), X86::RDI)
21508 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
21509 .addExternalSymbol("__morestack_allocate_stack_space")
21510 .addRegMask(RegMask)
21511 .addReg(X86::RDI, RegState::Implicit)
21512 .addReg(X86::RAX, RegState::ImplicitDefine);
21513 } else if (Is64Bit) {
21514 BuildMI(mallocMBB, DL, TII->get(X86::MOV32rr), X86::EDI)
21516 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
21517 .addExternalSymbol("__morestack_allocate_stack_space")
21518 .addRegMask(RegMask)
21519 .addReg(X86::EDI, RegState::Implicit)
21520 .addReg(X86::EAX, RegState::ImplicitDefine);
21522 BuildMI(mallocMBB, DL, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
21524 BuildMI(mallocMBB, DL, TII->get(X86::PUSH32r)).addReg(sizeVReg);
21525 BuildMI(mallocMBB, DL, TII->get(X86::CALLpcrel32))
21526 .addExternalSymbol("__morestack_allocate_stack_space")
21527 .addRegMask(RegMask)
21528 .addReg(X86::EAX, RegState::ImplicitDefine);
21532 BuildMI(mallocMBB, DL, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
21535 BuildMI(mallocMBB, DL, TII->get(TargetOpcode::COPY), mallocPtrVReg)
21536 .addReg(IsLP64 ? X86::RAX : X86::EAX);
21537 BuildMI(mallocMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
21539 // Set up the CFG correctly.
21540 BB->addSuccessor(bumpMBB);
21541 BB->addSuccessor(mallocMBB);
21542 mallocMBB->addSuccessor(continueMBB);
21543 bumpMBB->addSuccessor(continueMBB);
21545 // Take care of the PHI nodes.
21546 BuildMI(*continueMBB, continueMBB->begin(), DL, TII->get(X86::PHI),
21547 MI->getOperand(0).getReg())
21548 .addReg(mallocPtrVReg).addMBB(mallocMBB)
21549 .addReg(bumpSPPtrVReg).addMBB(bumpMBB);
21551 // Delete the original pseudo instruction.
21552 MI->eraseFromParent();
21555 return continueMBB;
21558 MachineBasicBlock *
21559 X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI,
21560 MachineBasicBlock *BB) const {
21561 DebugLoc DL = MI->getDebugLoc();
21563 assert(!Subtarget->isTargetMachO());
21565 X86FrameLowering::emitStackProbeCall(*BB->getParent(), *BB, MI, DL);
21567 MI->eraseFromParent(); // The pseudo instruction is gone now.
21571 MachineBasicBlock *
21572 X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI,
21573 MachineBasicBlock *BB) const {
21574 // This is pretty easy. We're taking the value that we received from
21575 // our load from the relocation, sticking it in either RDI (x86-64)
21576 // or EAX and doing an indirect call. The return value will then
21577 // be in the normal return register.
21578 MachineFunction *F = BB->getParent();
21579 const X86InstrInfo *TII = Subtarget->getInstrInfo();
21580 DebugLoc DL = MI->getDebugLoc();
21582 assert(Subtarget->isTargetDarwin() && "Darwin only instr emitted?");
21583 assert(MI->getOperand(3).isGlobal() && "This should be a global");
21585 // Get a register mask for the lowered call.
21586 // FIXME: The 32-bit calls have non-standard calling conventions. Use a
21587 // proper register mask.
21588 const uint32_t *RegMask =
21589 Subtarget->getRegisterInfo()->getCallPreservedMask(CallingConv::C);
21590 if (Subtarget->is64Bit()) {
21591 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
21592 TII->get(X86::MOV64rm), X86::RDI)
21594 .addImm(0).addReg(0)
21595 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
21596 MI->getOperand(3).getTargetFlags())
21598 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m));
21599 addDirectMem(MIB, X86::RDI);
21600 MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
21601 } else if (F->getTarget().getRelocationModel() != Reloc::PIC_) {
21602 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
21603 TII->get(X86::MOV32rm), X86::EAX)
21605 .addImm(0).addReg(0)
21606 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
21607 MI->getOperand(3).getTargetFlags())
21609 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
21610 addDirectMem(MIB, X86::EAX);
21611 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
21613 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
21614 TII->get(X86::MOV32rm), X86::EAX)
21615 .addReg(TII->getGlobalBaseReg(F))
21616 .addImm(0).addReg(0)
21617 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
21618 MI->getOperand(3).getTargetFlags())
21620 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
21621 addDirectMem(MIB, X86::EAX);
21622 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
21625 MI->eraseFromParent(); // The pseudo instruction is gone now.
21629 MachineBasicBlock *
21630 X86TargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
21631 MachineBasicBlock *MBB) const {
21632 DebugLoc DL = MI->getDebugLoc();
21633 MachineFunction *MF = MBB->getParent();
21634 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21635 MachineRegisterInfo &MRI = MF->getRegInfo();
21637 const BasicBlock *BB = MBB->getBasicBlock();
21638 MachineFunction::iterator I = MBB;
21641 // Memory Reference
21642 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
21643 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
21646 unsigned MemOpndSlot = 0;
21648 unsigned CurOp = 0;
21650 DstReg = MI->getOperand(CurOp++).getReg();
21651 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
21652 assert(RC->hasType(MVT::i32) && "Invalid destination!");
21653 unsigned mainDstReg = MRI.createVirtualRegister(RC);
21654 unsigned restoreDstReg = MRI.createVirtualRegister(RC);
21656 MemOpndSlot = CurOp;
21658 MVT PVT = getPointerTy();
21659 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
21660 "Invalid Pointer Size!");
21662 // For v = setjmp(buf), we generate
21665 // buf[LabelOffset] = restoreMBB
21666 // SjLjSetup restoreMBB
21672 // v = phi(main, restore)
21675 // if base pointer being used, load it from frame
21678 MachineBasicBlock *thisMBB = MBB;
21679 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
21680 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
21681 MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
21682 MF->insert(I, mainMBB);
21683 MF->insert(I, sinkMBB);
21684 MF->push_back(restoreMBB);
21686 MachineInstrBuilder MIB;
21688 // Transfer the remainder of BB and its successor edges to sinkMBB.
21689 sinkMBB->splice(sinkMBB->begin(), MBB,
21690 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
21691 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
21694 unsigned PtrStoreOpc = 0;
21695 unsigned LabelReg = 0;
21696 const int64_t LabelOffset = 1 * PVT.getStoreSize();
21697 Reloc::Model RM = MF->getTarget().getRelocationModel();
21698 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
21699 (RM == Reloc::Static || RM == Reloc::DynamicNoPIC);
21701 // Prepare IP either in reg or imm.
21702 if (!UseImmLabel) {
21703 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
21704 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
21705 LabelReg = MRI.createVirtualRegister(PtrRC);
21706 if (Subtarget->is64Bit()) {
21707 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA64r), LabelReg)
21711 .addMBB(restoreMBB)
21714 const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
21715 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA32r), LabelReg)
21716 .addReg(XII->getGlobalBaseReg(MF))
21719 .addMBB(restoreMBB, Subtarget->ClassifyBlockAddressReference())
21723 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
21725 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrStoreOpc));
21726 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
21727 if (i == X86::AddrDisp)
21728 MIB.addDisp(MI->getOperand(MemOpndSlot + i), LabelOffset);
21730 MIB.addOperand(MI->getOperand(MemOpndSlot + i));
21733 MIB.addReg(LabelReg);
21735 MIB.addMBB(restoreMBB);
21736 MIB.setMemRefs(MMOBegin, MMOEnd);
21738 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::EH_SjLj_Setup))
21739 .addMBB(restoreMBB);
21741 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
21742 MIB.addRegMask(RegInfo->getNoPreservedMask());
21743 thisMBB->addSuccessor(mainMBB);
21744 thisMBB->addSuccessor(restoreMBB);
21748 BuildMI(mainMBB, DL, TII->get(X86::MOV32r0), mainDstReg);
21749 mainMBB->addSuccessor(sinkMBB);
21752 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
21753 TII->get(X86::PHI), DstReg)
21754 .addReg(mainDstReg).addMBB(mainMBB)
21755 .addReg(restoreDstReg).addMBB(restoreMBB);
21758 if (RegInfo->hasBasePointer(*MF)) {
21759 const bool Uses64BitFramePtr =
21760 Subtarget->isTarget64BitLP64() || Subtarget->isTargetNaCl64();
21761 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
21762 X86FI->setRestoreBasePointer(MF);
21763 unsigned FramePtr = RegInfo->getFrameRegister(*MF);
21764 unsigned BasePtr = RegInfo->getBaseRegister();
21765 unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
21766 addRegOffset(BuildMI(restoreMBB, DL, TII->get(Opm), BasePtr),
21767 FramePtr, true, X86FI->getRestoreBasePointerOffset())
21768 .setMIFlag(MachineInstr::FrameSetup);
21770 BuildMI(restoreMBB, DL, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
21771 BuildMI(restoreMBB, DL, TII->get(X86::JMP_1)).addMBB(sinkMBB);
21772 restoreMBB->addSuccessor(sinkMBB);
21774 MI->eraseFromParent();
21778 MachineBasicBlock *
21779 X86TargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
21780 MachineBasicBlock *MBB) const {
21781 DebugLoc DL = MI->getDebugLoc();
21782 MachineFunction *MF = MBB->getParent();
21783 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21784 MachineRegisterInfo &MRI = MF->getRegInfo();
21786 // Memory Reference
21787 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
21788 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
21790 MVT PVT = getPointerTy();
21791 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
21792 "Invalid Pointer Size!");
21794 const TargetRegisterClass *RC =
21795 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
21796 unsigned Tmp = MRI.createVirtualRegister(RC);
21797 // Since FP is only updated here but NOT referenced, it's treated as GPR.
21798 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
21799 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
21800 unsigned SP = RegInfo->getStackRegister();
21802 MachineInstrBuilder MIB;
21804 const int64_t LabelOffset = 1 * PVT.getStoreSize();
21805 const int64_t SPOffset = 2 * PVT.getStoreSize();
21807 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
21808 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
21811 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), FP);
21812 for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
21813 MIB.addOperand(MI->getOperand(i));
21814 MIB.setMemRefs(MMOBegin, MMOEnd);
21816 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), Tmp);
21817 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
21818 if (i == X86::AddrDisp)
21819 MIB.addDisp(MI->getOperand(i), LabelOffset);
21821 MIB.addOperand(MI->getOperand(i));
21823 MIB.setMemRefs(MMOBegin, MMOEnd);
21825 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), SP);
21826 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
21827 if (i == X86::AddrDisp)
21828 MIB.addDisp(MI->getOperand(i), SPOffset);
21830 MIB.addOperand(MI->getOperand(i));
21832 MIB.setMemRefs(MMOBegin, MMOEnd);
21834 BuildMI(*MBB, MI, DL, TII->get(IJmpOpc)).addReg(Tmp);
21836 MI->eraseFromParent();
21840 // Replace 213-type (isel default) FMA3 instructions with 231-type for
21841 // accumulator loops. Writing back to the accumulator allows the coalescer
21842 // to remove extra copies in the loop.
21843 MachineBasicBlock *
21844 X86TargetLowering::emitFMA3Instr(MachineInstr *MI,
21845 MachineBasicBlock *MBB) const {
21846 MachineOperand &AddendOp = MI->getOperand(3);
21848 // Bail out early if the addend isn't a register - we can't switch these.
21849 if (!AddendOp.isReg())
21852 MachineFunction &MF = *MBB->getParent();
21853 MachineRegisterInfo &MRI = MF.getRegInfo();
21855 // Check whether the addend is defined by a PHI:
21856 assert(MRI.hasOneDef(AddendOp.getReg()) && "Multiple defs in SSA?");
21857 MachineInstr &AddendDef = *MRI.def_instr_begin(AddendOp.getReg());
21858 if (!AddendDef.isPHI())
21861 // Look for the following pattern:
21863 // %addend = phi [%entry, 0], [%loop, %result]
21865 // %result<tied1> = FMA213 %m2<tied0>, %m1, %addend
21869 // %addend = phi [%entry, 0], [%loop, %result]
21871 // %result<tied1> = FMA231 %addend<tied0>, %m1, %m2
21873 for (unsigned i = 1, e = AddendDef.getNumOperands(); i < e; i += 2) {
21874 assert(AddendDef.getOperand(i).isReg());
21875 MachineOperand PHISrcOp = AddendDef.getOperand(i);
21876 MachineInstr &PHISrcInst = *MRI.def_instr_begin(PHISrcOp.getReg());
21877 if (&PHISrcInst == MI) {
21878 // Found a matching instruction.
21879 unsigned NewFMAOpc = 0;
21880 switch (MI->getOpcode()) {
21881 case X86::VFMADDPDr213r: NewFMAOpc = X86::VFMADDPDr231r; break;
21882 case X86::VFMADDPSr213r: NewFMAOpc = X86::VFMADDPSr231r; break;
21883 case X86::VFMADDSDr213r: NewFMAOpc = X86::VFMADDSDr231r; break;
21884 case X86::VFMADDSSr213r: NewFMAOpc = X86::VFMADDSSr231r; break;
21885 case X86::VFMSUBPDr213r: NewFMAOpc = X86::VFMSUBPDr231r; break;
21886 case X86::VFMSUBPSr213r: NewFMAOpc = X86::VFMSUBPSr231r; break;
21887 case X86::VFMSUBSDr213r: NewFMAOpc = X86::VFMSUBSDr231r; break;
21888 case X86::VFMSUBSSr213r: NewFMAOpc = X86::VFMSUBSSr231r; break;
21889 case X86::VFNMADDPDr213r: NewFMAOpc = X86::VFNMADDPDr231r; break;
21890 case X86::VFNMADDPSr213r: NewFMAOpc = X86::VFNMADDPSr231r; break;
21891 case X86::VFNMADDSDr213r: NewFMAOpc = X86::VFNMADDSDr231r; break;
21892 case X86::VFNMADDSSr213r: NewFMAOpc = X86::VFNMADDSSr231r; break;
21893 case X86::VFNMSUBPDr213r: NewFMAOpc = X86::VFNMSUBPDr231r; break;
21894 case X86::VFNMSUBPSr213r: NewFMAOpc = X86::VFNMSUBPSr231r; break;
21895 case X86::VFNMSUBSDr213r: NewFMAOpc = X86::VFNMSUBSDr231r; break;
21896 case X86::VFNMSUBSSr213r: NewFMAOpc = X86::VFNMSUBSSr231r; break;
21897 case X86::VFMADDSUBPDr213r: NewFMAOpc = X86::VFMADDSUBPDr231r; break;
21898 case X86::VFMADDSUBPSr213r: NewFMAOpc = X86::VFMADDSUBPSr231r; break;
21899 case X86::VFMSUBADDPDr213r: NewFMAOpc = X86::VFMSUBADDPDr231r; break;
21900 case X86::VFMSUBADDPSr213r: NewFMAOpc = X86::VFMSUBADDPSr231r; break;
21902 case X86::VFMADDPDr213rY: NewFMAOpc = X86::VFMADDPDr231rY; break;
21903 case X86::VFMADDPSr213rY: NewFMAOpc = X86::VFMADDPSr231rY; break;
21904 case X86::VFMSUBPDr213rY: NewFMAOpc = X86::VFMSUBPDr231rY; break;
21905 case X86::VFMSUBPSr213rY: NewFMAOpc = X86::VFMSUBPSr231rY; break;
21906 case X86::VFNMADDPDr213rY: NewFMAOpc = X86::VFNMADDPDr231rY; break;
21907 case X86::VFNMADDPSr213rY: NewFMAOpc = X86::VFNMADDPSr231rY; break;
21908 case X86::VFNMSUBPDr213rY: NewFMAOpc = X86::VFNMSUBPDr231rY; break;
21909 case X86::VFNMSUBPSr213rY: NewFMAOpc = X86::VFNMSUBPSr231rY; break;
21910 case X86::VFMADDSUBPDr213rY: NewFMAOpc = X86::VFMADDSUBPDr231rY; break;
21911 case X86::VFMADDSUBPSr213rY: NewFMAOpc = X86::VFMADDSUBPSr231rY; break;
21912 case X86::VFMSUBADDPDr213rY: NewFMAOpc = X86::VFMSUBADDPDr231rY; break;
21913 case X86::VFMSUBADDPSr213rY: NewFMAOpc = X86::VFMSUBADDPSr231rY; break;
21914 default: llvm_unreachable("Unrecognized FMA variant.");
21917 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
21918 MachineInstrBuilder MIB =
21919 BuildMI(MF, MI->getDebugLoc(), TII.get(NewFMAOpc))
21920 .addOperand(MI->getOperand(0))
21921 .addOperand(MI->getOperand(3))
21922 .addOperand(MI->getOperand(2))
21923 .addOperand(MI->getOperand(1));
21924 MBB->insert(MachineBasicBlock::iterator(MI), MIB);
21925 MI->eraseFromParent();
21932 MachineBasicBlock *
21933 X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
21934 MachineBasicBlock *BB) const {
21935 switch (MI->getOpcode()) {
21936 default: llvm_unreachable("Unexpected instr type to insert");
21937 case X86::TAILJMPd64:
21938 case X86::TAILJMPr64:
21939 case X86::TAILJMPm64:
21940 case X86::TAILJMPd64_REX:
21941 case X86::TAILJMPr64_REX:
21942 case X86::TAILJMPm64_REX:
21943 llvm_unreachable("TAILJMP64 would not be touched here.");
21944 case X86::TCRETURNdi64:
21945 case X86::TCRETURNri64:
21946 case X86::TCRETURNmi64:
21948 case X86::WIN_ALLOCA:
21949 return EmitLoweredWinAlloca(MI, BB);
21950 case X86::SEG_ALLOCA_32:
21951 case X86::SEG_ALLOCA_64:
21952 return EmitLoweredSegAlloca(MI, BB);
21953 case X86::TLSCall_32:
21954 case X86::TLSCall_64:
21955 return EmitLoweredTLSCall(MI, BB);
21956 case X86::CMOV_GR8:
21957 case X86::CMOV_FR32:
21958 case X86::CMOV_FR64:
21959 case X86::CMOV_V4F32:
21960 case X86::CMOV_V2F64:
21961 case X86::CMOV_V2I64:
21962 case X86::CMOV_V8F32:
21963 case X86::CMOV_V4F64:
21964 case X86::CMOV_V4I64:
21965 case X86::CMOV_V16F32:
21966 case X86::CMOV_V8F64:
21967 case X86::CMOV_V8I64:
21968 case X86::CMOV_GR16:
21969 case X86::CMOV_GR32:
21970 case X86::CMOV_RFP32:
21971 case X86::CMOV_RFP64:
21972 case X86::CMOV_RFP80:
21973 return EmitLoweredSelect(MI, BB);
21975 case X86::FP32_TO_INT16_IN_MEM:
21976 case X86::FP32_TO_INT32_IN_MEM:
21977 case X86::FP32_TO_INT64_IN_MEM:
21978 case X86::FP64_TO_INT16_IN_MEM:
21979 case X86::FP64_TO_INT32_IN_MEM:
21980 case X86::FP64_TO_INT64_IN_MEM:
21981 case X86::FP80_TO_INT16_IN_MEM:
21982 case X86::FP80_TO_INT32_IN_MEM:
21983 case X86::FP80_TO_INT64_IN_MEM: {
21984 MachineFunction *F = BB->getParent();
21985 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21986 DebugLoc DL = MI->getDebugLoc();
21988 // Change the floating point control register to use "round towards zero"
21989 // mode when truncating to an integer value.
21990 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2, false);
21991 addFrameReference(BuildMI(*BB, MI, DL,
21992 TII->get(X86::FNSTCW16m)), CWFrameIdx);
21994 // Load the old value of the high byte of the control word...
21996 F->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
21997 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16rm), OldCW),
22000 // Set the high part to be round to zero...
22001 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mi)), CWFrameIdx)
22004 // Reload the modified control word now...
22005 addFrameReference(BuildMI(*BB, MI, DL,
22006 TII->get(X86::FLDCW16m)), CWFrameIdx);
22008 // Restore the memory image of control word to original value
22009 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)), CWFrameIdx)
22012 // Get the X86 opcode to use.
22014 switch (MI->getOpcode()) {
22015 default: llvm_unreachable("illegal opcode!");
22016 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
22017 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
22018 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
22019 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
22020 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
22021 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
22022 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
22023 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
22024 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
22028 MachineOperand &Op = MI->getOperand(0);
22030 AM.BaseType = X86AddressMode::RegBase;
22031 AM.Base.Reg = Op.getReg();
22033 AM.BaseType = X86AddressMode::FrameIndexBase;
22034 AM.Base.FrameIndex = Op.getIndex();
22036 Op = MI->getOperand(1);
22038 AM.Scale = Op.getImm();
22039 Op = MI->getOperand(2);
22041 AM.IndexReg = Op.getImm();
22042 Op = MI->getOperand(3);
22043 if (Op.isGlobal()) {
22044 AM.GV = Op.getGlobal();
22046 AM.Disp = Op.getImm();
22048 addFullAddress(BuildMI(*BB, MI, DL, TII->get(Opc)), AM)
22049 .addReg(MI->getOperand(X86::AddrNumOperands).getReg());
22051 // Reload the original control word now.
22052 addFrameReference(BuildMI(*BB, MI, DL,
22053 TII->get(X86::FLDCW16m)), CWFrameIdx);
22055 MI->eraseFromParent(); // The pseudo instruction is gone now.
22058 // String/text processing lowering.
22059 case X86::PCMPISTRM128REG:
22060 case X86::VPCMPISTRM128REG:
22061 case X86::PCMPISTRM128MEM:
22062 case X86::VPCMPISTRM128MEM:
22063 case X86::PCMPESTRM128REG:
22064 case X86::VPCMPESTRM128REG:
22065 case X86::PCMPESTRM128MEM:
22066 case X86::VPCMPESTRM128MEM:
22067 assert(Subtarget->hasSSE42() &&
22068 "Target must have SSE4.2 or AVX features enabled");
22069 return EmitPCMPSTRM(MI, BB, Subtarget->getInstrInfo());
22071 // String/text processing lowering.
22072 case X86::PCMPISTRIREG:
22073 case X86::VPCMPISTRIREG:
22074 case X86::PCMPISTRIMEM:
22075 case X86::VPCMPISTRIMEM:
22076 case X86::PCMPESTRIREG:
22077 case X86::VPCMPESTRIREG:
22078 case X86::PCMPESTRIMEM:
22079 case X86::VPCMPESTRIMEM:
22080 assert(Subtarget->hasSSE42() &&
22081 "Target must have SSE4.2 or AVX features enabled");
22082 return EmitPCMPSTRI(MI, BB, Subtarget->getInstrInfo());
22084 // Thread synchronization.
22086 return EmitMonitor(MI, BB, Subtarget);
22090 return EmitXBegin(MI, BB, Subtarget->getInstrInfo());
22092 case X86::VASTART_SAVE_XMM_REGS:
22093 return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB);
22095 case X86::VAARG_64:
22096 return EmitVAARG64WithCustomInserter(MI, BB);
22098 case X86::EH_SjLj_SetJmp32:
22099 case X86::EH_SjLj_SetJmp64:
22100 return emitEHSjLjSetJmp(MI, BB);
22102 case X86::EH_SjLj_LongJmp32:
22103 case X86::EH_SjLj_LongJmp64:
22104 return emitEHSjLjLongJmp(MI, BB);
22106 case TargetOpcode::STATEPOINT:
22107 // As an implementation detail, STATEPOINT shares the STACKMAP format at
22108 // this point in the process. We diverge later.
22109 return emitPatchPoint(MI, BB);
22111 case TargetOpcode::STACKMAP:
22112 case TargetOpcode::PATCHPOINT:
22113 return emitPatchPoint(MI, BB);
22115 case X86::VFMADDPDr213r:
22116 case X86::VFMADDPSr213r:
22117 case X86::VFMADDSDr213r:
22118 case X86::VFMADDSSr213r:
22119 case X86::VFMSUBPDr213r:
22120 case X86::VFMSUBPSr213r:
22121 case X86::VFMSUBSDr213r:
22122 case X86::VFMSUBSSr213r:
22123 case X86::VFNMADDPDr213r:
22124 case X86::VFNMADDPSr213r:
22125 case X86::VFNMADDSDr213r:
22126 case X86::VFNMADDSSr213r:
22127 case X86::VFNMSUBPDr213r:
22128 case X86::VFNMSUBPSr213r:
22129 case X86::VFNMSUBSDr213r:
22130 case X86::VFNMSUBSSr213r:
22131 case X86::VFMADDSUBPDr213r:
22132 case X86::VFMADDSUBPSr213r:
22133 case X86::VFMSUBADDPDr213r:
22134 case X86::VFMSUBADDPSr213r:
22135 case X86::VFMADDPDr213rY:
22136 case X86::VFMADDPSr213rY:
22137 case X86::VFMSUBPDr213rY:
22138 case X86::VFMSUBPSr213rY:
22139 case X86::VFNMADDPDr213rY:
22140 case X86::VFNMADDPSr213rY:
22141 case X86::VFNMSUBPDr213rY:
22142 case X86::VFNMSUBPSr213rY:
22143 case X86::VFMADDSUBPDr213rY:
22144 case X86::VFMADDSUBPSr213rY:
22145 case X86::VFMSUBADDPDr213rY:
22146 case X86::VFMSUBADDPSr213rY:
22147 return emitFMA3Instr(MI, BB);
22151 //===----------------------------------------------------------------------===//
22152 // X86 Optimization Hooks
22153 //===----------------------------------------------------------------------===//
22155 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
22158 const SelectionDAG &DAG,
22159 unsigned Depth) const {
22160 unsigned BitWidth = KnownZero.getBitWidth();
22161 unsigned Opc = Op.getOpcode();
22162 assert((Opc >= ISD::BUILTIN_OP_END ||
22163 Opc == ISD::INTRINSIC_WO_CHAIN ||
22164 Opc == ISD::INTRINSIC_W_CHAIN ||
22165 Opc == ISD::INTRINSIC_VOID) &&
22166 "Should use MaskedValueIsZero if you don't know whether Op"
22167 " is a target node!");
22169 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
22183 // These nodes' second result is a boolean.
22184 if (Op.getResNo() == 0)
22187 case X86ISD::SETCC:
22188 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
22190 case ISD::INTRINSIC_WO_CHAIN: {
22191 unsigned IntId = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
22192 unsigned NumLoBits = 0;
22195 case Intrinsic::x86_sse_movmsk_ps:
22196 case Intrinsic::x86_avx_movmsk_ps_256:
22197 case Intrinsic::x86_sse2_movmsk_pd:
22198 case Intrinsic::x86_avx_movmsk_pd_256:
22199 case Intrinsic::x86_mmx_pmovmskb:
22200 case Intrinsic::x86_sse2_pmovmskb_128:
22201 case Intrinsic::x86_avx2_pmovmskb: {
22202 // High bits of movmskp{s|d}, pmovmskb are known zero.
22204 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
22205 case Intrinsic::x86_sse_movmsk_ps: NumLoBits = 4; break;
22206 case Intrinsic::x86_avx_movmsk_ps_256: NumLoBits = 8; break;
22207 case Intrinsic::x86_sse2_movmsk_pd: NumLoBits = 2; break;
22208 case Intrinsic::x86_avx_movmsk_pd_256: NumLoBits = 4; break;
22209 case Intrinsic::x86_mmx_pmovmskb: NumLoBits = 8; break;
22210 case Intrinsic::x86_sse2_pmovmskb_128: NumLoBits = 16; break;
22211 case Intrinsic::x86_avx2_pmovmskb: NumLoBits = 32; break;
22213 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - NumLoBits);
22222 unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
22224 const SelectionDAG &,
22225 unsigned Depth) const {
22226 // SETCC_CARRY sets the dest to ~0 for true or 0 for false.
22227 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
22228 return Op.getValueType().getScalarType().getSizeInBits();
22234 /// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
22235 /// node is a GlobalAddress + offset.
22236 bool X86TargetLowering::isGAPlusOffset(SDNode *N,
22237 const GlobalValue* &GA,
22238 int64_t &Offset) const {
22239 if (N->getOpcode() == X86ISD::Wrapper) {
22240 if (isa<GlobalAddressSDNode>(N->getOperand(0))) {
22241 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
22242 Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset();
22246 return TargetLowering::isGAPlusOffset(N, GA, Offset);
22249 /// isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the
22250 /// same as extracting the high 128-bit part of 256-bit vector and then
22251 /// inserting the result into the low part of a new 256-bit vector
22252 static bool isShuffleHigh128VectorInsertLow(ShuffleVectorSDNode *SVOp) {
22253 EVT VT = SVOp->getValueType(0);
22254 unsigned NumElems = VT.getVectorNumElements();
22256 // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
22257 for (unsigned i = 0, j = NumElems/2; i != NumElems/2; ++i, ++j)
22258 if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
22259 SVOp->getMaskElt(j) >= 0)
22265 /// isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the
22266 /// same as extracting the low 128-bit part of 256-bit vector and then
22267 /// inserting the result into the high part of a new 256-bit vector
22268 static bool isShuffleLow128VectorInsertHigh(ShuffleVectorSDNode *SVOp) {
22269 EVT VT = SVOp->getValueType(0);
22270 unsigned NumElems = VT.getVectorNumElements();
22272 // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
22273 for (unsigned i = NumElems/2, j = 0; i != NumElems; ++i, ++j)
22274 if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
22275 SVOp->getMaskElt(j) >= 0)
22281 /// PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors.
22282 static SDValue PerformShuffleCombine256(SDNode *N, SelectionDAG &DAG,
22283 TargetLowering::DAGCombinerInfo &DCI,
22284 const X86Subtarget* Subtarget) {
22286 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
22287 SDValue V1 = SVOp->getOperand(0);
22288 SDValue V2 = SVOp->getOperand(1);
22289 EVT VT = SVOp->getValueType(0);
22290 unsigned NumElems = VT.getVectorNumElements();
22292 if (V1.getOpcode() == ISD::CONCAT_VECTORS &&
22293 V2.getOpcode() == ISD::CONCAT_VECTORS) {
22297 // V UNDEF BUILD_VECTOR UNDEF
22299 // CONCAT_VECTOR CONCAT_VECTOR
22302 // RESULT: V + zero extended
22304 if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR ||
22305 V2.getOperand(1).getOpcode() != ISD::UNDEF ||
22306 V1.getOperand(1).getOpcode() != ISD::UNDEF)
22309 if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()))
22312 // To match the shuffle mask, the first half of the mask should
22313 // be exactly the first vector, and all the rest a splat with the
22314 // first element of the second one.
22315 for (unsigned i = 0; i != NumElems/2; ++i)
22316 if (!isUndefOrEqual(SVOp->getMaskElt(i), i) ||
22317 !isUndefOrEqual(SVOp->getMaskElt(i+NumElems/2), NumElems))
22320 // If V1 is coming from a vector load then just fold to a VZEXT_LOAD.
22321 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(V1.getOperand(0))) {
22322 if (Ld->hasNUsesOfValue(1, 0)) {
22323 SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other);
22324 SDValue Ops[] = { Ld->getChain(), Ld->getBasePtr() };
22326 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
22328 Ld->getPointerInfo(),
22329 Ld->getAlignment(),
22330 false/*isVolatile*/, true/*ReadMem*/,
22331 false/*WriteMem*/);
22333 // Make sure the newly-created LOAD is in the same position as Ld in
22334 // terms of dependency. We create a TokenFactor for Ld and ResNode,
22335 // and update uses of Ld's output chain to use the TokenFactor.
22336 if (Ld->hasAnyUseOfValue(1)) {
22337 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
22338 SDValue(Ld, 1), SDValue(ResNode.getNode(), 1));
22339 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), NewChain);
22340 DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(Ld, 1),
22341 SDValue(ResNode.getNode(), 1));
22344 return DAG.getNode(ISD::BITCAST, dl, VT, ResNode);
22348 // Emit a zeroed vector and insert the desired subvector on its
22350 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
22351 SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0), 0, DAG, dl);
22352 return DCI.CombineTo(N, InsV);
22355 //===--------------------------------------------------------------------===//
22356 // Combine some shuffles into subvector extracts and inserts:
22359 // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
22360 if (isShuffleHigh128VectorInsertLow(SVOp)) {
22361 SDValue V = Extract128BitVector(V1, NumElems/2, DAG, dl);
22362 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, 0, DAG, dl);
22363 return DCI.CombineTo(N, InsV);
22366 // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
22367 if (isShuffleLow128VectorInsertHigh(SVOp)) {
22368 SDValue V = Extract128BitVector(V1, 0, DAG, dl);
22369 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, NumElems/2, DAG, dl);
22370 return DCI.CombineTo(N, InsV);
22376 /// \brief Combine an arbitrary chain of shuffles into a single instruction if
22379 /// This is the leaf of the recursive combinine below. When we have found some
22380 /// chain of single-use x86 shuffle instructions and accumulated the combined
22381 /// shuffle mask represented by them, this will try to pattern match that mask
22382 /// into either a single instruction if there is a special purpose instruction
22383 /// for this operation, or into a PSHUFB instruction which is a fully general
22384 /// instruction but should only be used to replace chains over a certain depth.
22385 static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
22386 int Depth, bool HasPSHUFB, SelectionDAG &DAG,
22387 TargetLowering::DAGCombinerInfo &DCI,
22388 const X86Subtarget *Subtarget) {
22389 assert(!Mask.empty() && "Cannot combine an empty shuffle mask!");
22391 // Find the operand that enters the chain. Note that multiple uses are OK
22392 // here, we're not going to remove the operand we find.
22393 SDValue Input = Op.getOperand(0);
22394 while (Input.getOpcode() == ISD::BITCAST)
22395 Input = Input.getOperand(0);
22397 MVT VT = Input.getSimpleValueType();
22398 MVT RootVT = Root.getSimpleValueType();
22401 // Just remove no-op shuffle masks.
22402 if (Mask.size() == 1) {
22403 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Input),
22408 // Use the float domain if the operand type is a floating point type.
22409 bool FloatDomain = VT.isFloatingPoint();
22411 // For floating point shuffles, we don't have free copies in the shuffle
22412 // instructions or the ability to load as part of the instruction, so
22413 // canonicalize their shuffles to UNPCK or MOV variants.
22415 // Note that even with AVX we prefer the PSHUFD form of shuffle for integer
22416 // vectors because it can have a load folded into it that UNPCK cannot. This
22417 // doesn't preclude something switching to the shorter encoding post-RA.
22419 if (Mask.equals(0, 0) || Mask.equals(1, 1)) {
22420 bool Lo = Mask.equals(0, 0);
22423 // Check if we have SSE3 which will let us use MOVDDUP. That instruction
22424 // is no slower than UNPCKLPD but has the option to fold the input operand
22425 // into even an unaligned memory load.
22426 if (Lo && Subtarget->hasSSE3()) {
22427 Shuffle = X86ISD::MOVDDUP;
22428 ShuffleVT = MVT::v2f64;
22430 // We have MOVLHPS and MOVHLPS throughout SSE and they encode smaller
22431 // than the UNPCK variants.
22432 Shuffle = Lo ? X86ISD::MOVLHPS : X86ISD::MOVHLPS;
22433 ShuffleVT = MVT::v4f32;
22435 if (Depth == 1 && Root->getOpcode() == Shuffle)
22436 return false; // Nothing to do!
22437 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22438 DCI.AddToWorklist(Op.getNode());
22439 if (Shuffle == X86ISD::MOVDDUP)
22440 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op);
22442 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
22443 DCI.AddToWorklist(Op.getNode());
22444 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22448 if (Subtarget->hasSSE3() &&
22449 (Mask.equals(0, 0, 2, 2) || Mask.equals(1, 1, 3, 3))) {
22450 bool Lo = Mask.equals(0, 0, 2, 2);
22451 unsigned Shuffle = Lo ? X86ISD::MOVSLDUP : X86ISD::MOVSHDUP;
22452 MVT ShuffleVT = MVT::v4f32;
22453 if (Depth == 1 && Root->getOpcode() == Shuffle)
22454 return false; // Nothing to do!
22455 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22456 DCI.AddToWorklist(Op.getNode());
22457 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op);
22458 DCI.AddToWorklist(Op.getNode());
22459 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22463 if (Mask.equals(0, 0, 1, 1) || Mask.equals(2, 2, 3, 3)) {
22464 bool Lo = Mask.equals(0, 0, 1, 1);
22465 unsigned Shuffle = Lo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
22466 MVT ShuffleVT = MVT::v4f32;
22467 if (Depth == 1 && Root->getOpcode() == Shuffle)
22468 return false; // Nothing to do!
22469 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22470 DCI.AddToWorklist(Op.getNode());
22471 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
22472 DCI.AddToWorklist(Op.getNode());
22473 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22479 // We always canonicalize the 8 x i16 and 16 x i8 shuffles into their UNPCK
22480 // variants as none of these have single-instruction variants that are
22481 // superior to the UNPCK formulation.
22482 if (!FloatDomain &&
22483 (Mask.equals(0, 0, 1, 1, 2, 2, 3, 3) ||
22484 Mask.equals(4, 4, 5, 5, 6, 6, 7, 7) ||
22485 Mask.equals(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7) ||
22486 Mask.equals(8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15,
22488 bool Lo = Mask[0] == 0;
22489 unsigned Shuffle = Lo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
22490 if (Depth == 1 && Root->getOpcode() == Shuffle)
22491 return false; // Nothing to do!
22493 switch (Mask.size()) {
22495 ShuffleVT = MVT::v8i16;
22498 ShuffleVT = MVT::v16i8;
22501 llvm_unreachable("Impossible mask size!");
22503 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22504 DCI.AddToWorklist(Op.getNode());
22505 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
22506 DCI.AddToWorklist(Op.getNode());
22507 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22512 // Don't try to re-form single instruction chains under any circumstances now
22513 // that we've done encoding canonicalization for them.
22517 // If we have 3 or more shuffle instructions or a chain involving PSHUFB, we
22518 // can replace them with a single PSHUFB instruction profitably. Intel's
22519 // manuals suggest only using PSHUFB if doing so replacing 5 instructions, but
22520 // in practice PSHUFB tends to be *very* fast so we're more aggressive.
22521 if ((Depth >= 3 || HasPSHUFB) && Subtarget->hasSSSE3()) {
22522 SmallVector<SDValue, 16> PSHUFBMask;
22523 assert(Mask.size() <= 16 && "Can't shuffle elements smaller than bytes!");
22524 int Ratio = 16 / Mask.size();
22525 for (unsigned i = 0; i < 16; ++i) {
22526 if (Mask[i / Ratio] == SM_SentinelUndef) {
22527 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
22530 int M = Mask[i / Ratio] != SM_SentinelZero
22531 ? Ratio * Mask[i / Ratio] + i % Ratio
22533 PSHUFBMask.push_back(DAG.getConstant(M, MVT::i8));
22535 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Input);
22536 DCI.AddToWorklist(Op.getNode());
22537 SDValue PSHUFBMaskOp =
22538 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, PSHUFBMask);
22539 DCI.AddToWorklist(PSHUFBMaskOp.getNode());
22540 Op = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, Op, PSHUFBMaskOp);
22541 DCI.AddToWorklist(Op.getNode());
22542 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22547 // Failed to find any combines.
22551 /// \brief Fully generic combining of x86 shuffle instructions.
22553 /// This should be the last combine run over the x86 shuffle instructions. Once
22554 /// they have been fully optimized, this will recursively consider all chains
22555 /// of single-use shuffle instructions, build a generic model of the cumulative
22556 /// shuffle operation, and check for simpler instructions which implement this
22557 /// operation. We use this primarily for two purposes:
22559 /// 1) Collapse generic shuffles to specialized single instructions when
22560 /// equivalent. In most cases, this is just an encoding size win, but
22561 /// sometimes we will collapse multiple generic shuffles into a single
22562 /// special-purpose shuffle.
22563 /// 2) Look for sequences of shuffle instructions with 3 or more total
22564 /// instructions, and replace them with the slightly more expensive SSSE3
22565 /// PSHUFB instruction if available. We do this as the last combining step
22566 /// to ensure we avoid using PSHUFB if we can implement the shuffle with
22567 /// a suitable short sequence of other instructions. The PHUFB will either
22568 /// use a register or have to read from memory and so is slightly (but only
22569 /// slightly) more expensive than the other shuffle instructions.
22571 /// Because this is inherently a quadratic operation (for each shuffle in
22572 /// a chain, we recurse up the chain), the depth is limited to 8 instructions.
22573 /// This should never be an issue in practice as the shuffle lowering doesn't
22574 /// produce sequences of more than 8 instructions.
22576 /// FIXME: We will currently miss some cases where the redundant shuffling
22577 /// would simplify under the threshold for PSHUFB formation because of
22578 /// combine-ordering. To fix this, we should do the redundant instruction
22579 /// combining in this recursive walk.
22580 static bool combineX86ShufflesRecursively(SDValue Op, SDValue Root,
22581 ArrayRef<int> RootMask,
22582 int Depth, bool HasPSHUFB,
22584 TargetLowering::DAGCombinerInfo &DCI,
22585 const X86Subtarget *Subtarget) {
22586 // Bound the depth of our recursive combine because this is ultimately
22587 // quadratic in nature.
22591 // Directly rip through bitcasts to find the underlying operand.
22592 while (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).hasOneUse())
22593 Op = Op.getOperand(0);
22595 MVT VT = Op.getSimpleValueType();
22596 if (!VT.isVector())
22597 return false; // Bail if we hit a non-vector.
22598 // FIXME: This routine should be taught about 256-bit shuffles, or a 256-bit
22599 // version should be added.
22600 if (VT.getSizeInBits() != 128)
22603 assert(Root.getSimpleValueType().isVector() &&
22604 "Shuffles operate on vector types!");
22605 assert(VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits() &&
22606 "Can only combine shuffles of the same vector register size.");
22608 if (!isTargetShuffle(Op.getOpcode()))
22610 SmallVector<int, 16> OpMask;
22612 bool HaveMask = getTargetShuffleMask(Op.getNode(), VT, OpMask, IsUnary);
22613 // We only can combine unary shuffles which we can decode the mask for.
22614 if (!HaveMask || !IsUnary)
22617 assert(VT.getVectorNumElements() == OpMask.size() &&
22618 "Different mask size from vector size!");
22619 assert(((RootMask.size() > OpMask.size() &&
22620 RootMask.size() % OpMask.size() == 0) ||
22621 (OpMask.size() > RootMask.size() &&
22622 OpMask.size() % RootMask.size() == 0) ||
22623 OpMask.size() == RootMask.size()) &&
22624 "The smaller number of elements must divide the larger.");
22625 int RootRatio = std::max<int>(1, OpMask.size() / RootMask.size());
22626 int OpRatio = std::max<int>(1, RootMask.size() / OpMask.size());
22627 assert(((RootRatio == 1 && OpRatio == 1) ||
22628 (RootRatio == 1) != (OpRatio == 1)) &&
22629 "Must not have a ratio for both incoming and op masks!");
22631 SmallVector<int, 16> Mask;
22632 Mask.reserve(std::max(OpMask.size(), RootMask.size()));
22634 // Merge this shuffle operation's mask into our accumulated mask. Note that
22635 // this shuffle's mask will be the first applied to the input, followed by the
22636 // root mask to get us all the way to the root value arrangement. The reason
22637 // for this order is that we are recursing up the operation chain.
22638 for (int i = 0, e = std::max(OpMask.size(), RootMask.size()); i < e; ++i) {
22639 int RootIdx = i / RootRatio;
22640 if (RootMask[RootIdx] < 0) {
22641 // This is a zero or undef lane, we're done.
22642 Mask.push_back(RootMask[RootIdx]);
22646 int RootMaskedIdx = RootMask[RootIdx] * RootRatio + i % RootRatio;
22647 int OpIdx = RootMaskedIdx / OpRatio;
22648 if (OpMask[OpIdx] < 0) {
22649 // The incoming lanes are zero or undef, it doesn't matter which ones we
22651 Mask.push_back(OpMask[OpIdx]);
22655 // Ok, we have non-zero lanes, map them through.
22656 Mask.push_back(OpMask[OpIdx] * OpRatio +
22657 RootMaskedIdx % OpRatio);
22660 // See if we can recurse into the operand to combine more things.
22661 switch (Op.getOpcode()) {
22662 case X86ISD::PSHUFB:
22664 case X86ISD::PSHUFD:
22665 case X86ISD::PSHUFHW:
22666 case X86ISD::PSHUFLW:
22667 if (Op.getOperand(0).hasOneUse() &&
22668 combineX86ShufflesRecursively(Op.getOperand(0), Root, Mask, Depth + 1,
22669 HasPSHUFB, DAG, DCI, Subtarget))
22673 case X86ISD::UNPCKL:
22674 case X86ISD::UNPCKH:
22675 assert(Op.getOperand(0) == Op.getOperand(1) && "We only combine unary shuffles!");
22676 // We can't check for single use, we have to check that this shuffle is the only user.
22677 if (Op->isOnlyUserOf(Op.getOperand(0).getNode()) &&
22678 combineX86ShufflesRecursively(Op.getOperand(0), Root, Mask, Depth + 1,
22679 HasPSHUFB, DAG, DCI, Subtarget))
22684 // Minor canonicalization of the accumulated shuffle mask to make it easier
22685 // to match below. All this does is detect masks with squential pairs of
22686 // elements, and shrink them to the half-width mask. It does this in a loop
22687 // so it will reduce the size of the mask to the minimal width mask which
22688 // performs an equivalent shuffle.
22689 SmallVector<int, 16> WidenedMask;
22690 while (Mask.size() > 1 && canWidenShuffleElements(Mask, WidenedMask)) {
22691 Mask = std::move(WidenedMask);
22692 WidenedMask.clear();
22695 return combineX86ShuffleChain(Op, Root, Mask, Depth, HasPSHUFB, DAG, DCI,
22699 /// \brief Get the PSHUF-style mask from PSHUF node.
22701 /// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
22702 /// PSHUF-style masks that can be reused with such instructions.
22703 static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
22704 SmallVector<int, 4> Mask;
22706 bool HaveMask = getTargetShuffleMask(N.getNode(), N.getSimpleValueType(), Mask, IsUnary);
22710 switch (N.getOpcode()) {
22711 case X86ISD::PSHUFD:
22713 case X86ISD::PSHUFLW:
22716 case X86ISD::PSHUFHW:
22717 Mask.erase(Mask.begin(), Mask.begin() + 4);
22718 for (int &M : Mask)
22722 llvm_unreachable("No valid shuffle instruction found!");
22726 /// \brief Search for a combinable shuffle across a chain ending in pshufd.
22728 /// We walk up the chain and look for a combinable shuffle, skipping over
22729 /// shuffles that we could hoist this shuffle's transformation past without
22730 /// altering anything.
22732 combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
22734 TargetLowering::DAGCombinerInfo &DCI) {
22735 assert(N.getOpcode() == X86ISD::PSHUFD &&
22736 "Called with something other than an x86 128-bit half shuffle!");
22739 // Walk up a single-use chain looking for a combinable shuffle. Keep a stack
22740 // of the shuffles in the chain so that we can form a fresh chain to replace
22742 SmallVector<SDValue, 8> Chain;
22743 SDValue V = N.getOperand(0);
22744 for (; V.hasOneUse(); V = V.getOperand(0)) {
22745 switch (V.getOpcode()) {
22747 return SDValue(); // Nothing combined!
22750 // Skip bitcasts as we always know the type for the target specific
22754 case X86ISD::PSHUFD:
22755 // Found another dword shuffle.
22758 case X86ISD::PSHUFLW:
22759 // Check that the low words (being shuffled) are the identity in the
22760 // dword shuffle, and the high words are self-contained.
22761 if (Mask[0] != 0 || Mask[1] != 1 ||
22762 !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
22765 Chain.push_back(V);
22768 case X86ISD::PSHUFHW:
22769 // Check that the high words (being shuffled) are the identity in the
22770 // dword shuffle, and the low words are self-contained.
22771 if (Mask[2] != 2 || Mask[3] != 3 ||
22772 !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
22775 Chain.push_back(V);
22778 case X86ISD::UNPCKL:
22779 case X86ISD::UNPCKH:
22780 // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
22781 // shuffle into a preceding word shuffle.
22782 if (V.getValueType() != MVT::v16i8 && V.getValueType() != MVT::v8i16)
22785 // Search for a half-shuffle which we can combine with.
22786 unsigned CombineOp =
22787 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
22788 if (V.getOperand(0) != V.getOperand(1) ||
22789 !V->isOnlyUserOf(V.getOperand(0).getNode()))
22791 Chain.push_back(V);
22792 V = V.getOperand(0);
22794 switch (V.getOpcode()) {
22796 return SDValue(); // Nothing to combine.
22798 case X86ISD::PSHUFLW:
22799 case X86ISD::PSHUFHW:
22800 if (V.getOpcode() == CombineOp)
22803 Chain.push_back(V);
22807 V = V.getOperand(0);
22811 } while (V.hasOneUse());
22814 // Break out of the loop if we break out of the switch.
22818 if (!V.hasOneUse())
22819 // We fell out of the loop without finding a viable combining instruction.
22822 // Merge this node's mask and our incoming mask.
22823 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
22824 for (int &M : Mask)
22826 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
22827 getV4X86ShuffleImm8ForMask(Mask, DAG));
22829 // Rebuild the chain around this new shuffle.
22830 while (!Chain.empty()) {
22831 SDValue W = Chain.pop_back_val();
22833 if (V.getValueType() != W.getOperand(0).getValueType())
22834 V = DAG.getNode(ISD::BITCAST, DL, W.getOperand(0).getValueType(), V);
22836 switch (W.getOpcode()) {
22838 llvm_unreachable("Only PSHUF and UNPCK instructions get here!");
22840 case X86ISD::UNPCKL:
22841 case X86ISD::UNPCKH:
22842 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
22845 case X86ISD::PSHUFD:
22846 case X86ISD::PSHUFLW:
22847 case X86ISD::PSHUFHW:
22848 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
22852 if (V.getValueType() != N.getValueType())
22853 V = DAG.getNode(ISD::BITCAST, DL, N.getValueType(), V);
22855 // Return the new chain to replace N.
22859 /// \brief Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.
22861 /// We walk up the chain, skipping shuffles of the other half and looking
22862 /// through shuffles which switch halves trying to find a shuffle of the same
22863 /// pair of dwords.
22864 static bool combineRedundantHalfShuffle(SDValue N, MutableArrayRef<int> Mask,
22866 TargetLowering::DAGCombinerInfo &DCI) {
22868 (N.getOpcode() == X86ISD::PSHUFLW || N.getOpcode() == X86ISD::PSHUFHW) &&
22869 "Called with something other than an x86 128-bit half shuffle!");
22871 unsigned CombineOpcode = N.getOpcode();
22873 // Walk up a single-use chain looking for a combinable shuffle.
22874 SDValue V = N.getOperand(0);
22875 for (; V.hasOneUse(); V = V.getOperand(0)) {
22876 switch (V.getOpcode()) {
22878 return false; // Nothing combined!
22881 // Skip bitcasts as we always know the type for the target specific
22885 case X86ISD::PSHUFLW:
22886 case X86ISD::PSHUFHW:
22887 if (V.getOpcode() == CombineOpcode)
22890 // Other-half shuffles are no-ops.
22893 // Break out of the loop if we break out of the switch.
22897 if (!V.hasOneUse())
22898 // We fell out of the loop without finding a viable combining instruction.
22901 // Combine away the bottom node as its shuffle will be accumulated into
22902 // a preceding shuffle.
22903 DCI.CombineTo(N.getNode(), N.getOperand(0), /*AddTo*/ true);
22905 // Record the old value.
22908 // Merge this node's mask and our incoming mask (adjusted to account for all
22909 // the pshufd instructions encountered).
22910 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
22911 for (int &M : Mask)
22913 V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0),
22914 getV4X86ShuffleImm8ForMask(Mask, DAG));
22916 // Check that the shuffles didn't cancel each other out. If not, we need to
22917 // combine to the new one.
22919 // Replace the combinable shuffle with the combined one, updating all users
22920 // so that we re-evaluate the chain here.
22921 DCI.CombineTo(Old.getNode(), V, /*AddTo*/ true);
22926 /// \brief Try to combine x86 target specific shuffles.
22927 static SDValue PerformTargetShuffleCombine(SDValue N, SelectionDAG &DAG,
22928 TargetLowering::DAGCombinerInfo &DCI,
22929 const X86Subtarget *Subtarget) {
22931 MVT VT = N.getSimpleValueType();
22932 SmallVector<int, 4> Mask;
22934 switch (N.getOpcode()) {
22935 case X86ISD::PSHUFD:
22936 case X86ISD::PSHUFLW:
22937 case X86ISD::PSHUFHW:
22938 Mask = getPSHUFShuffleMask(N);
22939 assert(Mask.size() == 4);
22945 // Nuke no-op shuffles that show up after combining.
22946 if (isNoopShuffleMask(Mask))
22947 return DCI.CombineTo(N.getNode(), N.getOperand(0), /*AddTo*/ true);
22949 // Look for simplifications involving one or two shuffle instructions.
22950 SDValue V = N.getOperand(0);
22951 switch (N.getOpcode()) {
22954 case X86ISD::PSHUFLW:
22955 case X86ISD::PSHUFHW:
22956 assert(VT == MVT::v8i16);
22959 if (combineRedundantHalfShuffle(N, Mask, DAG, DCI))
22960 return SDValue(); // We combined away this shuffle, so we're done.
22962 // See if this reduces to a PSHUFD which is no more expensive and can
22963 // combine with more operations. Note that it has to at least flip the
22964 // dwords as otherwise it would have been removed as a no-op.
22965 if (Mask[0] == 2 && Mask[1] == 3 && Mask[2] == 0 && Mask[3] == 1) {
22966 int DMask[] = {0, 1, 2, 3};
22967 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
22968 DMask[DOffset + 0] = DOffset + 1;
22969 DMask[DOffset + 1] = DOffset + 0;
22970 V = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V);
22971 DCI.AddToWorklist(V.getNode());
22972 V = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V,
22973 getV4X86ShuffleImm8ForMask(DMask, DAG));
22974 DCI.AddToWorklist(V.getNode());
22975 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
22978 // Look for shuffle patterns which can be implemented as a single unpack.
22979 // FIXME: This doesn't handle the location of the PSHUFD generically, and
22980 // only works when we have a PSHUFD followed by two half-shuffles.
22981 if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
22982 (V.getOpcode() == X86ISD::PSHUFLW ||
22983 V.getOpcode() == X86ISD::PSHUFHW) &&
22984 V.getOpcode() != N.getOpcode() &&
22986 SDValue D = V.getOperand(0);
22987 while (D.getOpcode() == ISD::BITCAST && D.hasOneUse())
22988 D = D.getOperand(0);
22989 if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
22990 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
22991 SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
22992 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
22993 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
22995 for (int i = 0; i < 4; ++i) {
22996 WordMask[i + NOffset] = Mask[i] + NOffset;
22997 WordMask[i + VOffset] = VMask[i] + VOffset;
22999 // Map the word mask through the DWord mask.
23001 for (int i = 0; i < 8; ++i)
23002 MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
23003 const int UnpackLoMask[] = {0, 0, 1, 1, 2, 2, 3, 3};
23004 const int UnpackHiMask[] = {4, 4, 5, 5, 6, 6, 7, 7};
23005 if (std::equal(std::begin(MappedMask), std::end(MappedMask),
23006 std::begin(UnpackLoMask)) ||
23007 std::equal(std::begin(MappedMask), std::end(MappedMask),
23008 std::begin(UnpackHiMask))) {
23009 // We can replace all three shuffles with an unpack.
23010 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, D.getOperand(0));
23011 DCI.AddToWorklist(V.getNode());
23012 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
23014 DL, MVT::v8i16, V, V);
23021 case X86ISD::PSHUFD:
23022 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG, DCI))
23031 /// \brief Try to combine a shuffle into a target-specific add-sub node.
23033 /// We combine this directly on the abstract vector shuffle nodes so it is
23034 /// easier to generically match. We also insert dummy vector shuffle nodes for
23035 /// the operands which explicitly discard the lanes which are unused by this
23036 /// operation to try to flow through the rest of the combiner the fact that
23037 /// they're unused.
23038 static SDValue combineShuffleToAddSub(SDNode *N, SelectionDAG &DAG) {
23040 EVT VT = N->getValueType(0);
23042 // We only handle target-independent shuffles.
23043 // FIXME: It would be easy and harmless to use the target shuffle mask
23044 // extraction tool to support more.
23045 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
23048 auto *SVN = cast<ShuffleVectorSDNode>(N);
23049 ArrayRef<int> Mask = SVN->getMask();
23050 SDValue V1 = N->getOperand(0);
23051 SDValue V2 = N->getOperand(1);
23053 // We require the first shuffle operand to be the SUB node, and the second to
23054 // be the ADD node.
23055 // FIXME: We should support the commuted patterns.
23056 if (V1->getOpcode() != ISD::FSUB || V2->getOpcode() != ISD::FADD)
23059 // If there are other uses of these operations we can't fold them.
23060 if (!V1->hasOneUse() || !V2->hasOneUse())
23063 // Ensure that both operations have the same operands. Note that we can
23064 // commute the FADD operands.
23065 SDValue LHS = V1->getOperand(0), RHS = V1->getOperand(1);
23066 if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
23067 (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
23070 // We're looking for blends between FADD and FSUB nodes. We insist on these
23071 // nodes being lined up in a specific expected pattern.
23072 if (!(isShuffleEquivalent(V1, V2, Mask, 0, 3) ||
23073 isShuffleEquivalent(V1, V2, Mask, 0, 5, 2, 7) ||
23074 isShuffleEquivalent(V1, V2, Mask, 0, 9, 2, 11, 4, 13, 6, 15)))
23077 // Only specific types are legal at this point, assert so we notice if and
23078 // when these change.
23079 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v8f32 ||
23080 VT == MVT::v4f64) &&
23081 "Unknown vector type encountered!");
23083 return DAG.getNode(X86ISD::ADDSUB, DL, VT, LHS, RHS);
23086 /// PerformShuffleCombine - Performs several different shuffle combines.
23087 static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
23088 TargetLowering::DAGCombinerInfo &DCI,
23089 const X86Subtarget *Subtarget) {
23091 SDValue N0 = N->getOperand(0);
23092 SDValue N1 = N->getOperand(1);
23093 EVT VT = N->getValueType(0);
23095 // Don't create instructions with illegal types after legalize types has run.
23096 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23097 if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType()))
23100 // If we have legalized the vector types, look for blends of FADD and FSUB
23101 // nodes that we can fuse into an ADDSUB node.
23102 if (TLI.isTypeLegal(VT) && Subtarget->hasSSE3())
23103 if (SDValue AddSub = combineShuffleToAddSub(N, DAG))
23106 // Combine 256-bit vector shuffles. This is only profitable when in AVX mode
23107 if (Subtarget->hasFp256() && VT.is256BitVector() &&
23108 N->getOpcode() == ISD::VECTOR_SHUFFLE)
23109 return PerformShuffleCombine256(N, DAG, DCI, Subtarget);
23111 // During Type Legalization, when promoting illegal vector types,
23112 // the backend might introduce new shuffle dag nodes and bitcasts.
23114 // This code performs the following transformation:
23115 // fold: (shuffle (bitcast (BINOP A, B)), Undef, <Mask>) ->
23116 // (shuffle (BINOP (bitcast A), (bitcast B)), Undef, <Mask>)
23118 // We do this only if both the bitcast and the BINOP dag nodes have
23119 // one use. Also, perform this transformation only if the new binary
23120 // operation is legal. This is to avoid introducing dag nodes that
23121 // potentially need to be further expanded (or custom lowered) into a
23122 // less optimal sequence of dag nodes.
23123 if (!DCI.isBeforeLegalize() && DCI.isBeforeLegalizeOps() &&
23124 N1.getOpcode() == ISD::UNDEF && N0.hasOneUse() &&
23125 N0.getOpcode() == ISD::BITCAST) {
23126 SDValue BC0 = N0.getOperand(0);
23127 EVT SVT = BC0.getValueType();
23128 unsigned Opcode = BC0.getOpcode();
23129 unsigned NumElts = VT.getVectorNumElements();
23131 if (BC0.hasOneUse() && SVT.isVector() &&
23132 SVT.getVectorNumElements() * 2 == NumElts &&
23133 TLI.isOperationLegal(Opcode, VT)) {
23134 bool CanFold = false;
23146 unsigned SVTNumElts = SVT.getVectorNumElements();
23147 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
23148 for (unsigned i = 0, e = SVTNumElts; i != e && CanFold; ++i)
23149 CanFold = SVOp->getMaskElt(i) == (int)(i * 2);
23150 for (unsigned i = SVTNumElts, e = NumElts; i != e && CanFold; ++i)
23151 CanFold = SVOp->getMaskElt(i) < 0;
23154 SDValue BC00 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(0));
23155 SDValue BC01 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(1));
23156 SDValue NewBinOp = DAG.getNode(BC0.getOpcode(), dl, VT, BC00, BC01);
23157 return DAG.getVectorShuffle(VT, dl, NewBinOp, N1, &SVOp->getMask()[0]);
23162 // Only handle 128 wide vector from here on.
23163 if (!VT.is128BitVector())
23166 // Combine a vector_shuffle that is equal to build_vector load1, load2, load3,
23167 // load4, <0, 1, 2, 3> into a 128-bit load if the load addresses are
23168 // consecutive, non-overlapping, and in the right order.
23169 SmallVector<SDValue, 16> Elts;
23170 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
23171 Elts.push_back(getShuffleScalarElt(N, i, DAG, 0));
23173 SDValue LD = EltsFromConsecutiveLoads(VT, Elts, dl, DAG, true);
23177 if (isTargetShuffle(N->getOpcode())) {
23179 PerformTargetShuffleCombine(SDValue(N, 0), DAG, DCI, Subtarget);
23180 if (Shuffle.getNode())
23183 // Try recursively combining arbitrary sequences of x86 shuffle
23184 // instructions into higher-order shuffles. We do this after combining
23185 // specific PSHUF instruction sequences into their minimal form so that we
23186 // can evaluate how many specialized shuffle instructions are involved in
23187 // a particular chain.
23188 SmallVector<int, 1> NonceMask; // Just a placeholder.
23189 NonceMask.push_back(0);
23190 if (combineX86ShufflesRecursively(SDValue(N, 0), SDValue(N, 0), NonceMask,
23191 /*Depth*/ 1, /*HasPSHUFB*/ false, DAG,
23193 return SDValue(); // This routine will use CombineTo to replace N.
23199 /// PerformTruncateCombine - Converts truncate operation to
23200 /// a sequence of vector shuffle operations.
23201 /// It is possible when we truncate 256-bit vector to 128-bit vector
23202 static SDValue PerformTruncateCombine(SDNode *N, SelectionDAG &DAG,
23203 TargetLowering::DAGCombinerInfo &DCI,
23204 const X86Subtarget *Subtarget) {
23208 /// XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target
23209 /// specific shuffle of a load can be folded into a single element load.
23210 /// Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but
23211 /// shuffles have been custom lowered so we need to handle those here.
23212 static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
23213 TargetLowering::DAGCombinerInfo &DCI) {
23214 if (DCI.isBeforeLegalizeOps())
23217 SDValue InVec = N->getOperand(0);
23218 SDValue EltNo = N->getOperand(1);
23220 if (!isa<ConstantSDNode>(EltNo))
23223 EVT OriginalVT = InVec.getValueType();
23225 if (InVec.getOpcode() == ISD::BITCAST) {
23226 // Don't duplicate a load with other uses.
23227 if (!InVec.hasOneUse())
23229 EVT BCVT = InVec.getOperand(0).getValueType();
23230 if (BCVT.getVectorNumElements() != OriginalVT.getVectorNumElements())
23232 InVec = InVec.getOperand(0);
23235 EVT CurrentVT = InVec.getValueType();
23237 if (!isTargetShuffle(InVec.getOpcode()))
23240 // Don't duplicate a load with other uses.
23241 if (!InVec.hasOneUse())
23244 SmallVector<int, 16> ShuffleMask;
23246 if (!getTargetShuffleMask(InVec.getNode(), CurrentVT.getSimpleVT(),
23247 ShuffleMask, UnaryShuffle))
23250 // Select the input vector, guarding against out of range extract vector.
23251 unsigned NumElems = CurrentVT.getVectorNumElements();
23252 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
23253 int Idx = (Elt > (int)NumElems) ? -1 : ShuffleMask[Elt];
23254 SDValue LdNode = (Idx < (int)NumElems) ? InVec.getOperand(0)
23255 : InVec.getOperand(1);
23257 // If inputs to shuffle are the same for both ops, then allow 2 uses
23258 unsigned AllowedUses = InVec.getNumOperands() > 1 &&
23259 InVec.getOperand(0) == InVec.getOperand(1) ? 2 : 1;
23261 if (LdNode.getOpcode() == ISD::BITCAST) {
23262 // Don't duplicate a load with other uses.
23263 if (!LdNode.getNode()->hasNUsesOfValue(AllowedUses, 0))
23266 AllowedUses = 1; // only allow 1 load use if we have a bitcast
23267 LdNode = LdNode.getOperand(0);
23270 if (!ISD::isNormalLoad(LdNode.getNode()))
23273 LoadSDNode *LN0 = cast<LoadSDNode>(LdNode);
23275 if (!LN0 ||!LN0->hasNUsesOfValue(AllowedUses, 0) || LN0->isVolatile())
23278 EVT EltVT = N->getValueType(0);
23279 // If there's a bitcast before the shuffle, check if the load type and
23280 // alignment is valid.
23281 unsigned Align = LN0->getAlignment();
23282 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23283 unsigned NewAlign = TLI.getDataLayout()->getABITypeAlignment(
23284 EltVT.getTypeForEVT(*DAG.getContext()));
23286 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, EltVT))
23289 // All checks match so transform back to vector_shuffle so that DAG combiner
23290 // can finish the job
23293 // Create shuffle node taking into account the case that its a unary shuffle
23294 SDValue Shuffle = (UnaryShuffle) ? DAG.getUNDEF(CurrentVT)
23295 : InVec.getOperand(1);
23296 Shuffle = DAG.getVectorShuffle(CurrentVT, dl,
23297 InVec.getOperand(0), Shuffle,
23299 Shuffle = DAG.getNode(ISD::BITCAST, dl, OriginalVT, Shuffle);
23300 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), Shuffle,
23304 /// \brief Detect bitcasts between i32 to x86mmx low word. Since MMX types are
23305 /// special and don't usually play with other vector types, it's better to
23306 /// handle them early to be sure we emit efficient code by avoiding
23307 /// store-load conversions.
23308 static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG) {
23309 if (N->getValueType(0) != MVT::x86mmx ||
23310 N->getOperand(0)->getOpcode() != ISD::BUILD_VECTOR ||
23311 N->getOperand(0)->getValueType(0) != MVT::v2i32)
23314 SDValue V = N->getOperand(0);
23315 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
23316 if (C && C->getZExtValue() == 0 && V.getOperand(0).getValueType() == MVT::i32)
23317 return DAG.getNode(X86ISD::MMX_MOVW2D, SDLoc(V.getOperand(0)),
23318 N->getValueType(0), V.getOperand(0));
23323 /// PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index
23324 /// generation and convert it from being a bunch of shuffles and extracts
23325 /// into a somewhat faster sequence. For i686, the best sequence is apparently
23326 /// storing the value and loading scalars back, while for x64 we should
23327 /// use 64-bit extracts and shifts.
23328 static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
23329 TargetLowering::DAGCombinerInfo &DCI) {
23330 SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI);
23331 if (NewOp.getNode())
23334 SDValue InputVector = N->getOperand(0);
23336 // Detect mmx to i32 conversion through a v2i32 elt extract.
23337 if (InputVector.getOpcode() == ISD::BITCAST && InputVector.hasOneUse() &&
23338 N->getValueType(0) == MVT::i32 &&
23339 InputVector.getValueType() == MVT::v2i32) {
23341 // The bitcast source is a direct mmx result.
23342 SDValue MMXSrc = InputVector.getNode()->getOperand(0);
23343 if (MMXSrc.getValueType() == MVT::x86mmx)
23344 return DAG.getNode(X86ISD::MMX_MOVD2W, SDLoc(InputVector),
23345 N->getValueType(0),
23346 InputVector.getNode()->getOperand(0));
23348 // The mmx is indirect: (i64 extract_elt (v1i64 bitcast (x86mmx ...))).
23349 SDValue MMXSrcOp = MMXSrc.getOperand(0);
23350 if (MMXSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT && MMXSrc.hasOneUse() &&
23351 MMXSrc.getValueType() == MVT::i64 && MMXSrcOp.hasOneUse() &&
23352 MMXSrcOp.getOpcode() == ISD::BITCAST &&
23353 MMXSrcOp.getValueType() == MVT::v1i64 &&
23354 MMXSrcOp.getOperand(0).getValueType() == MVT::x86mmx)
23355 return DAG.getNode(X86ISD::MMX_MOVD2W, SDLoc(InputVector),
23356 N->getValueType(0),
23357 MMXSrcOp.getOperand(0));
23360 // Only operate on vectors of 4 elements, where the alternative shuffling
23361 // gets to be more expensive.
23362 if (InputVector.getValueType() != MVT::v4i32)
23365 // Check whether every use of InputVector is an EXTRACT_VECTOR_ELT with a
23366 // single use which is a sign-extend or zero-extend, and all elements are
23368 SmallVector<SDNode *, 4> Uses;
23369 unsigned ExtractedElements = 0;
23370 for (SDNode::use_iterator UI = InputVector.getNode()->use_begin(),
23371 UE = InputVector.getNode()->use_end(); UI != UE; ++UI) {
23372 if (UI.getUse().getResNo() != InputVector.getResNo())
23375 SDNode *Extract = *UI;
23376 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
23379 if (Extract->getValueType(0) != MVT::i32)
23381 if (!Extract->hasOneUse())
23383 if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND &&
23384 Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND)
23386 if (!isa<ConstantSDNode>(Extract->getOperand(1)))
23389 // Record which element was extracted.
23390 ExtractedElements |=
23391 1 << cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
23393 Uses.push_back(Extract);
23396 // If not all the elements were used, this may not be worthwhile.
23397 if (ExtractedElements != 15)
23400 // Ok, we've now decided to do the transformation.
23401 // If 64-bit shifts are legal, use the extract-shift sequence,
23402 // otherwise bounce the vector off the cache.
23403 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23405 SDLoc dl(InputVector);
23407 if (TLI.isOperationLegal(ISD::SRA, MVT::i64)) {
23408 SDValue Cst = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, InputVector);
23409 EVT VecIdxTy = DAG.getTargetLoweringInfo().getVectorIdxTy();
23410 SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst,
23411 DAG.getConstant(0, VecIdxTy));
23412 SDValue TopHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst,
23413 DAG.getConstant(1, VecIdxTy));
23415 SDValue ShAmt = DAG.getConstant(32,
23416 DAG.getTargetLoweringInfo().getShiftAmountTy(MVT::i64));
23417 Vals[0] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BottomHalf);
23418 Vals[1] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
23419 DAG.getNode(ISD::SRA, dl, MVT::i64, BottomHalf, ShAmt));
23420 Vals[2] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, TopHalf);
23421 Vals[3] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
23422 DAG.getNode(ISD::SRA, dl, MVT::i64, TopHalf, ShAmt));
23424 // Store the value to a temporary stack slot.
23425 SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType());
23426 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr,
23427 MachinePointerInfo(), false, false, 0);
23429 EVT ElementType = InputVector.getValueType().getVectorElementType();
23430 unsigned EltSize = ElementType.getSizeInBits() / 8;
23432 // Replace each use (extract) with a load of the appropriate element.
23433 for (unsigned i = 0; i < 4; ++i) {
23434 uint64_t Offset = EltSize * i;
23435 SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy());
23437 SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(),
23438 StackPtr, OffsetVal);
23440 // Load the scalar.
23441 Vals[i] = DAG.getLoad(ElementType, dl, Ch,
23442 ScalarAddr, MachinePointerInfo(),
23443 false, false, false, 0);
23448 // Replace the extracts
23449 for (SmallVectorImpl<SDNode *>::iterator UI = Uses.begin(),
23450 UE = Uses.end(); UI != UE; ++UI) {
23451 SDNode *Extract = *UI;
23453 SDValue Idx = Extract->getOperand(1);
23454 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
23455 DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), Vals[IdxVal]);
23458 // The replacement was made in place; don't return anything.
23462 /// \brief Matches a VSELECT onto min/max or return 0 if the node doesn't match.
23463 static std::pair<unsigned, bool>
23464 matchIntegerMINMAX(SDValue Cond, EVT VT, SDValue LHS, SDValue RHS,
23465 SelectionDAG &DAG, const X86Subtarget *Subtarget) {
23466 if (!VT.isVector())
23467 return std::make_pair(0, false);
23469 bool NeedSplit = false;
23470 switch (VT.getSimpleVT().SimpleTy) {
23471 default: return std::make_pair(0, false);
23474 if (!Subtarget->hasVLX())
23475 return std::make_pair(0, false);
23479 if (!Subtarget->hasBWI())
23480 return std::make_pair(0, false);
23484 if (!Subtarget->hasAVX512())
23485 return std::make_pair(0, false);
23490 if (!Subtarget->hasAVX2())
23492 if (!Subtarget->hasAVX())
23493 return std::make_pair(0, false);
23498 if (!Subtarget->hasSSE2())
23499 return std::make_pair(0, false);
23502 // SSE2 has only a small subset of the operations.
23503 bool hasUnsigned = Subtarget->hasSSE41() ||
23504 (Subtarget->hasSSE2() && VT == MVT::v16i8);
23505 bool hasSigned = Subtarget->hasSSE41() ||
23506 (Subtarget->hasSSE2() && VT == MVT::v8i16);
23508 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23511 // Check for x CC y ? x : y.
23512 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
23513 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
23518 Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
23521 Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
23524 Opc = hasSigned ? X86ISD::SMIN : 0; break;
23527 Opc = hasSigned ? X86ISD::SMAX : 0; break;
23529 // Check for x CC y ? y : x -- a min/max with reversed arms.
23530 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
23531 DAG.isEqualTo(RHS, Cond.getOperand(0))) {
23536 Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
23539 Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
23542 Opc = hasSigned ? X86ISD::SMAX : 0; break;
23545 Opc = hasSigned ? X86ISD::SMIN : 0; break;
23549 return std::make_pair(Opc, NeedSplit);
23553 transformVSELECTtoBlendVECTOR_SHUFFLE(SDNode *N, SelectionDAG &DAG,
23554 const X86Subtarget *Subtarget) {
23556 SDValue Cond = N->getOperand(0);
23557 SDValue LHS = N->getOperand(1);
23558 SDValue RHS = N->getOperand(2);
23560 if (Cond.getOpcode() == ISD::SIGN_EXTEND) {
23561 SDValue CondSrc = Cond->getOperand(0);
23562 if (CondSrc->getOpcode() == ISD::SIGN_EXTEND_INREG)
23563 Cond = CondSrc->getOperand(0);
23566 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
23569 // A vselect where all conditions and data are constants can be optimized into
23570 // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
23571 if (ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) &&
23572 ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()))
23575 unsigned MaskValue = 0;
23576 if (!BUILD_VECTORtoBlendMask(cast<BuildVectorSDNode>(Cond), MaskValue))
23579 MVT VT = N->getSimpleValueType(0);
23580 unsigned NumElems = VT.getVectorNumElements();
23581 SmallVector<int, 8> ShuffleMask(NumElems, -1);
23582 for (unsigned i = 0; i < NumElems; ++i) {
23583 // Be sure we emit undef where we can.
23584 if (Cond.getOperand(i)->getOpcode() == ISD::UNDEF)
23585 ShuffleMask[i] = -1;
23587 ShuffleMask[i] = i + NumElems * ((MaskValue >> i) & 1);
23590 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23591 if (!TLI.isShuffleMaskLegal(ShuffleMask, VT))
23593 return DAG.getVectorShuffle(VT, dl, LHS, RHS, &ShuffleMask[0]);
23596 /// PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT
23598 static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
23599 TargetLowering::DAGCombinerInfo &DCI,
23600 const X86Subtarget *Subtarget) {
23602 SDValue Cond = N->getOperand(0);
23603 // Get the LHS/RHS of the select.
23604 SDValue LHS = N->getOperand(1);
23605 SDValue RHS = N->getOperand(2);
23606 EVT VT = LHS.getValueType();
23607 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23609 // If we have SSE[12] support, try to form min/max nodes. SSE min/max
23610 // instructions match the semantics of the common C idiom x<y?x:y but not
23611 // x<=y?x:y, because of how they handle negative zero (which can be
23612 // ignored in unsafe-math mode).
23613 // We also try to create v2f32 min/max nodes, which we later widen to v4f32.
23614 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
23615 VT != MVT::f80 && (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
23616 (Subtarget->hasSSE2() ||
23617 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) {
23618 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23620 unsigned Opcode = 0;
23621 // Check for x CC y ? x : y.
23622 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
23623 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
23627 // Converting this to a min would handle NaNs incorrectly, and swapping
23628 // the operands would cause it to handle comparisons between positive
23629 // and negative zero incorrectly.
23630 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
23631 if (!DAG.getTarget().Options.UnsafeFPMath &&
23632 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
23634 std::swap(LHS, RHS);
23636 Opcode = X86ISD::FMIN;
23639 // Converting this to a min would handle comparisons between positive
23640 // and negative zero incorrectly.
23641 if (!DAG.getTarget().Options.UnsafeFPMath &&
23642 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
23644 Opcode = X86ISD::FMIN;
23647 // Converting this to a min would handle both negative zeros and NaNs
23648 // incorrectly, but we can swap the operands to fix both.
23649 std::swap(LHS, RHS);
23653 Opcode = X86ISD::FMIN;
23657 // Converting this to a max would handle comparisons between positive
23658 // and negative zero incorrectly.
23659 if (!DAG.getTarget().Options.UnsafeFPMath &&
23660 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
23662 Opcode = X86ISD::FMAX;
23665 // Converting this to a max would handle NaNs incorrectly, and swapping
23666 // the operands would cause it to handle comparisons between positive
23667 // and negative zero incorrectly.
23668 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
23669 if (!DAG.getTarget().Options.UnsafeFPMath &&
23670 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
23672 std::swap(LHS, RHS);
23674 Opcode = X86ISD::FMAX;
23677 // Converting this to a max would handle both negative zeros and NaNs
23678 // incorrectly, but we can swap the operands to fix both.
23679 std::swap(LHS, RHS);
23683 Opcode = X86ISD::FMAX;
23686 // Check for x CC y ? y : x -- a min/max with reversed arms.
23687 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
23688 DAG.isEqualTo(RHS, Cond.getOperand(0))) {
23692 // Converting this to a min would handle comparisons between positive
23693 // and negative zero incorrectly, and swapping the operands would
23694 // cause it to handle NaNs incorrectly.
23695 if (!DAG.getTarget().Options.UnsafeFPMath &&
23696 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) {
23697 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
23699 std::swap(LHS, RHS);
23701 Opcode = X86ISD::FMIN;
23704 // Converting this to a min would handle NaNs incorrectly.
23705 if (!DAG.getTarget().Options.UnsafeFPMath &&
23706 (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
23708 Opcode = X86ISD::FMIN;
23711 // Converting this to a min would handle both negative zeros and NaNs
23712 // incorrectly, but we can swap the operands to fix both.
23713 std::swap(LHS, RHS);
23717 Opcode = X86ISD::FMIN;
23721 // Converting this to a max would handle NaNs incorrectly.
23722 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
23724 Opcode = X86ISD::FMAX;
23727 // Converting this to a max would handle comparisons between positive
23728 // and negative zero incorrectly, and swapping the operands would
23729 // cause it to handle NaNs incorrectly.
23730 if (!DAG.getTarget().Options.UnsafeFPMath &&
23731 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) {
23732 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
23734 std::swap(LHS, RHS);
23736 Opcode = X86ISD::FMAX;
23739 // Converting this to a max would handle both negative zeros and NaNs
23740 // incorrectly, but we can swap the operands to fix both.
23741 std::swap(LHS, RHS);
23745 Opcode = X86ISD::FMAX;
23751 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
23754 EVT CondVT = Cond.getValueType();
23755 if (Subtarget->hasAVX512() && VT.isVector() && CondVT.isVector() &&
23756 CondVT.getVectorElementType() == MVT::i1) {
23757 // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
23758 // lowering on KNL. In this case we convert it to
23759 // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
23760 // The same situation for all 128 and 256-bit vectors of i8 and i16.
23761 // Since SKX these selects have a proper lowering.
23762 EVT OpVT = LHS.getValueType();
23763 if ((OpVT.is128BitVector() || OpVT.is256BitVector()) &&
23764 (OpVT.getVectorElementType() == MVT::i8 ||
23765 OpVT.getVectorElementType() == MVT::i16) &&
23766 !(Subtarget->hasBWI() && Subtarget->hasVLX())) {
23767 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, OpVT, Cond);
23768 DCI.AddToWorklist(Cond.getNode());
23769 return DAG.getNode(N->getOpcode(), DL, OpVT, Cond, LHS, RHS);
23772 // If this is a select between two integer constants, try to do some
23774 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) {
23775 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS))
23776 // Don't do this for crazy integer types.
23777 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) {
23778 // If this is efficiently invertible, canonicalize the LHSC/RHSC values
23779 // so that TrueC (the true value) is larger than FalseC.
23780 bool NeedsCondInvert = false;
23782 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) &&
23783 // Efficiently invertible.
23784 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible.
23785 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible.
23786 isa<ConstantSDNode>(Cond.getOperand(1))))) {
23787 NeedsCondInvert = true;
23788 std::swap(TrueC, FalseC);
23791 // Optimize C ? 8 : 0 -> zext(C) << 3. Likewise for any pow2/0.
23792 if (FalseC->getAPIntValue() == 0 &&
23793 TrueC->getAPIntValue().isPowerOf2()) {
23794 if (NeedsCondInvert) // Invert the condition if needed.
23795 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
23796 DAG.getConstant(1, Cond.getValueType()));
23798 // Zero extend the condition if needed.
23799 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond);
23801 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
23802 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond,
23803 DAG.getConstant(ShAmt, MVT::i8));
23806 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.
23807 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
23808 if (NeedsCondInvert) // Invert the condition if needed.
23809 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
23810 DAG.getConstant(1, Cond.getValueType()));
23812 // Zero extend the condition if needed.
23813 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
23814 FalseC->getValueType(0), Cond);
23815 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
23816 SDValue(FalseC, 0));
23819 // Optimize cases that will turn into an LEA instruction. This requires
23820 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
23821 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
23822 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
23823 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
23825 bool isFastMultiplier = false;
23827 switch ((unsigned char)Diff) {
23829 case 1: // result = add base, cond
23830 case 2: // result = lea base( , cond*2)
23831 case 3: // result = lea base(cond, cond*2)
23832 case 4: // result = lea base( , cond*4)
23833 case 5: // result = lea base(cond, cond*4)
23834 case 8: // result = lea base( , cond*8)
23835 case 9: // result = lea base(cond, cond*8)
23836 isFastMultiplier = true;
23841 if (isFastMultiplier) {
23842 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
23843 if (NeedsCondInvert) // Invert the condition if needed.
23844 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
23845 DAG.getConstant(1, Cond.getValueType()));
23847 // Zero extend the condition if needed.
23848 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
23850 // Scale the condition by the difference.
23852 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
23853 DAG.getConstant(Diff, Cond.getValueType()));
23855 // Add the base if non-zero.
23856 if (FalseC->getAPIntValue() != 0)
23857 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
23858 SDValue(FalseC, 0));
23865 // Canonicalize max and min:
23866 // (x > y) ? x : y -> (x >= y) ? x : y
23867 // (x < y) ? x : y -> (x <= y) ? x : y
23868 // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
23869 // the need for an extra compare
23870 // against zero. e.g.
23871 // (x - y) > 0 : (x - y) ? 0 -> (x - y) >= 0 : (x - y) ? 0
23873 // testl %edi, %edi
23875 // cmovgl %edi, %eax
23879 // cmovsl %eax, %edi
23880 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
23881 DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
23882 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
23883 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23888 ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE;
23889 Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(),
23890 Cond.getOperand(0), Cond.getOperand(1), NewCC);
23891 return DAG.getNode(ISD::SELECT, DL, VT, Cond, LHS, RHS);
23896 // Early exit check
23897 if (!TLI.isTypeLegal(VT))
23900 // Match VSELECTs into subs with unsigned saturation.
23901 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
23902 // psubus is available in SSE2 and AVX2 for i8 and i16 vectors.
23903 ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) ||
23904 (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) {
23905 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23907 // Check if one of the arms of the VSELECT is a zero vector. If it's on the
23908 // left side invert the predicate to simplify logic below.
23910 if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
23912 CC = ISD::getSetCCInverse(CC, true);
23913 } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) {
23917 if (Other.getNode() && Other->getNumOperands() == 2 &&
23918 DAG.isEqualTo(Other->getOperand(0), Cond.getOperand(0))) {
23919 SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
23920 SDValue CondRHS = Cond->getOperand(1);
23922 // Look for a general sub with unsigned saturation first.
23923 // x >= y ? x-y : 0 --> subus x, y
23924 // x > y ? x-y : 0 --> subus x, y
23925 if ((CC == ISD::SETUGE || CC == ISD::SETUGT) &&
23926 Other->getOpcode() == ISD::SUB && DAG.isEqualTo(OpRHS, CondRHS))
23927 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
23929 if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS))
23930 if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
23931 if (auto *CondRHSBV = dyn_cast<BuildVectorSDNode>(CondRHS))
23932 if (auto *CondRHSConst = CondRHSBV->getConstantSplatNode())
23933 // If the RHS is a constant we have to reverse the const
23934 // canonicalization.
23935 // x > C-1 ? x+-C : 0 --> subus x, C
23936 if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD &&
23937 CondRHSConst->getAPIntValue() ==
23938 (-OpRHSConst->getAPIntValue() - 1))
23939 return DAG.getNode(
23940 X86ISD::SUBUS, DL, VT, OpLHS,
23941 DAG.getConstant(-OpRHSConst->getAPIntValue(), VT));
23943 // Another special case: If C was a sign bit, the sub has been
23944 // canonicalized into a xor.
23945 // FIXME: Would it be better to use computeKnownBits to determine
23946 // whether it's safe to decanonicalize the xor?
23947 // x s< 0 ? x^C : 0 --> subus x, C
23948 if (CC == ISD::SETLT && Other->getOpcode() == ISD::XOR &&
23949 ISD::isBuildVectorAllZeros(CondRHS.getNode()) &&
23950 OpRHSConst->getAPIntValue().isSignBit())
23951 // Note that we have to rebuild the RHS constant here to ensure we
23952 // don't rely on particular values of undef lanes.
23953 return DAG.getNode(
23954 X86ISD::SUBUS, DL, VT, OpLHS,
23955 DAG.getConstant(OpRHSConst->getAPIntValue(), VT));
23960 // Try to match a min/max vector operation.
23961 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC) {
23962 std::pair<unsigned, bool> ret = matchIntegerMINMAX(Cond, VT, LHS, RHS, DAG, Subtarget);
23963 unsigned Opc = ret.first;
23964 bool NeedSplit = ret.second;
23966 if (Opc && NeedSplit) {
23967 unsigned NumElems = VT.getVectorNumElements();
23968 // Extract the LHS vectors
23969 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, DL);
23970 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, DL);
23972 // Extract the RHS vectors
23973 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, DL);
23974 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, DL);
23976 // Create min/max for each subvector
23977 LHS = DAG.getNode(Opc, DL, LHS1.getValueType(), LHS1, RHS1);
23978 RHS = DAG.getNode(Opc, DL, LHS2.getValueType(), LHS2, RHS2);
23980 // Merge the result
23981 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS, RHS);
23983 return DAG.getNode(Opc, DL, VT, LHS, RHS);
23986 // Simplify vector selection if condition value type matches vselect
23988 if (N->getOpcode() == ISD::VSELECT && CondVT == VT) {
23989 assert(Cond.getValueType().isVector() &&
23990 "vector select expects a vector selector!");
23992 bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
23993 bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
23995 // Try invert the condition if true value is not all 1s and false value
23997 if (!TValIsAllOnes && !FValIsAllZeros &&
23998 // Check if the selector will be produced by CMPP*/PCMP*
23999 Cond.getOpcode() == ISD::SETCC &&
24000 // Check if SETCC has already been promoted
24001 TLI.getSetCCResultType(*DAG.getContext(), VT) == CondVT) {
24002 bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
24003 bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
24005 if (TValIsAllZeros || FValIsAllOnes) {
24006 SDValue CC = Cond.getOperand(2);
24007 ISD::CondCode NewCC =
24008 ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
24009 Cond.getOperand(0).getValueType().isInteger());
24010 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1), NewCC);
24011 std::swap(LHS, RHS);
24012 TValIsAllOnes = FValIsAllOnes;
24013 FValIsAllZeros = TValIsAllZeros;
24017 if (TValIsAllOnes || FValIsAllZeros) {
24020 if (TValIsAllOnes && FValIsAllZeros)
24022 else if (TValIsAllOnes)
24023 Ret = DAG.getNode(ISD::OR, DL, CondVT, Cond,
24024 DAG.getNode(ISD::BITCAST, DL, CondVT, RHS));
24025 else if (FValIsAllZeros)
24026 Ret = DAG.getNode(ISD::AND, DL, CondVT, Cond,
24027 DAG.getNode(ISD::BITCAST, DL, CondVT, LHS));
24029 return DAG.getNode(ISD::BITCAST, DL, VT, Ret);
24033 // If we know that this node is legal then we know that it is going to be
24034 // matched by one of the SSE/AVX BLEND instructions. These instructions only
24035 // depend on the highest bit in each word. Try to use SimplifyDemandedBits
24036 // to simplify previous instructions.
24037 if (N->getOpcode() == ISD::VSELECT && DCI.isBeforeLegalizeOps() &&
24038 !DCI.isBeforeLegalize() &&
24039 // We explicitly check against v8i16 and v16i16 because, although
24040 // they're marked as Custom, they might only be legal when Cond is a
24041 // build_vector of constants. This will be taken care in a later
24043 (TLI.isOperationLegalOrCustom(ISD::VSELECT, VT) && VT != MVT::v16i16 &&
24044 VT != MVT::v8i16) &&
24045 // Don't optimize vector of constants. Those are handled by
24046 // the generic code and all the bits must be properly set for
24047 // the generic optimizer.
24048 !ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) {
24049 unsigned BitWidth = Cond.getValueType().getScalarType().getSizeInBits();
24051 // Don't optimize vector selects that map to mask-registers.
24055 assert(BitWidth >= 8 && BitWidth <= 64 && "Invalid mask size");
24056 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 1);
24058 APInt KnownZero, KnownOne;
24059 TargetLowering::TargetLoweringOpt TLO(DAG, DCI.isBeforeLegalize(),
24060 DCI.isBeforeLegalizeOps());
24061 if (TLO.ShrinkDemandedConstant(Cond, DemandedMask) ||
24062 TLI.SimplifyDemandedBits(Cond, DemandedMask, KnownZero, KnownOne,
24064 // If we changed the computation somewhere in the DAG, this change
24065 // will affect all users of Cond.
24066 // Make sure it is fine and update all the nodes so that we do not
24067 // use the generic VSELECT anymore. Otherwise, we may perform
24068 // wrong optimizations as we messed up with the actual expectation
24069 // for the vector boolean values.
24070 if (Cond != TLO.Old) {
24071 // Check all uses of that condition operand to check whether it will be
24072 // consumed by non-BLEND instructions, which may depend on all bits are
24074 for (SDNode::use_iterator I = Cond->use_begin(), E = Cond->use_end();
24076 if (I->getOpcode() != ISD::VSELECT)
24077 // TODO: Add other opcodes eventually lowered into BLEND.
24080 // Update all the users of the condition, before committing the change,
24081 // so that the VSELECT optimizations that expect the correct vector
24082 // boolean value will not be triggered.
24083 for (SDNode::use_iterator I = Cond->use_begin(), E = Cond->use_end();
24085 DAG.ReplaceAllUsesOfValueWith(
24087 DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(*I), I->getValueType(0),
24088 Cond, I->getOperand(1), I->getOperand(2)));
24089 DCI.CommitTargetLoweringOpt(TLO);
24092 // At this point, only Cond is changed. Change the condition
24093 // just for N to keep the opportunity to optimize all other
24094 // users their own way.
24095 DAG.ReplaceAllUsesOfValueWith(
24097 DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(N), N->getValueType(0),
24098 TLO.New, N->getOperand(1), N->getOperand(2)));
24103 // We should generate an X86ISD::BLENDI from a vselect if its argument
24104 // is a sign_extend_inreg of an any_extend of a BUILD_VECTOR of
24105 // constants. This specific pattern gets generated when we split a
24106 // selector for a 512 bit vector in a machine without AVX512 (but with
24107 // 256-bit vectors), during legalization:
24109 // (vselect (sign_extend (any_extend (BUILD_VECTOR)) i1) LHS RHS)
24111 // Iff we find this pattern and the build_vectors are built from
24112 // constants, we translate the vselect into a shuffle_vector that we
24113 // know will be matched by LowerVECTOR_SHUFFLEtoBlend.
24114 if ((N->getOpcode() == ISD::VSELECT ||
24115 N->getOpcode() == X86ISD::SHRUNKBLEND) &&
24116 !DCI.isBeforeLegalize()) {
24117 SDValue Shuffle = transformVSELECTtoBlendVECTOR_SHUFFLE(N, DAG, Subtarget);
24118 if (Shuffle.getNode())
24125 // Check whether a boolean test is testing a boolean value generated by
24126 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
24129 // Simplify the following patterns:
24130 // (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
24131 // (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
24132 // to (Op EFLAGS Cond)
24134 // (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
24135 // (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
24136 // to (Op EFLAGS !Cond)
24138 // where Op could be BRCOND or CMOV.
24140 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
24141 // Quit if not CMP and SUB with its value result used.
24142 if (Cmp.getOpcode() != X86ISD::CMP &&
24143 (Cmp.getOpcode() != X86ISD::SUB || Cmp.getNode()->hasAnyUseOfValue(0)))
24146 // Quit if not used as a boolean value.
24147 if (CC != X86::COND_E && CC != X86::COND_NE)
24150 // Check CMP operands. One of them should be 0 or 1 and the other should be
24151 // an SetCC or extended from it.
24152 SDValue Op1 = Cmp.getOperand(0);
24153 SDValue Op2 = Cmp.getOperand(1);
24156 const ConstantSDNode* C = nullptr;
24157 bool needOppositeCond = (CC == X86::COND_E);
24158 bool checkAgainstTrue = false; // Is it a comparison against 1?
24160 if ((C = dyn_cast<ConstantSDNode>(Op1)))
24162 else if ((C = dyn_cast<ConstantSDNode>(Op2)))
24164 else // Quit if all operands are not constants.
24167 if (C->getZExtValue() == 1) {
24168 needOppositeCond = !needOppositeCond;
24169 checkAgainstTrue = true;
24170 } else if (C->getZExtValue() != 0)
24171 // Quit if the constant is neither 0 or 1.
24174 bool truncatedToBoolWithAnd = false;
24175 // Skip (zext $x), (trunc $x), or (and $x, 1) node.
24176 while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
24177 SetCC.getOpcode() == ISD::TRUNCATE ||
24178 SetCC.getOpcode() == ISD::AND) {
24179 if (SetCC.getOpcode() == ISD::AND) {
24181 ConstantSDNode *CS;
24182 if ((CS = dyn_cast<ConstantSDNode>(SetCC.getOperand(0))) &&
24183 CS->getZExtValue() == 1)
24185 if ((CS = dyn_cast<ConstantSDNode>(SetCC.getOperand(1))) &&
24186 CS->getZExtValue() == 1)
24190 SetCC = SetCC.getOperand(OpIdx);
24191 truncatedToBoolWithAnd = true;
24193 SetCC = SetCC.getOperand(0);
24196 switch (SetCC.getOpcode()) {
24197 case X86ISD::SETCC_CARRY:
24198 // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
24199 // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
24200 // i.e. it's a comparison against true but the result of SETCC_CARRY is not
24201 // truncated to i1 using 'and'.
24202 if (checkAgainstTrue && !truncatedToBoolWithAnd)
24204 assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
24205 "Invalid use of SETCC_CARRY!");
24207 case X86ISD::SETCC:
24208 // Set the condition code or opposite one if necessary.
24209 CC = X86::CondCode(SetCC.getConstantOperandVal(0));
24210 if (needOppositeCond)
24211 CC = X86::GetOppositeBranchCondition(CC);
24212 return SetCC.getOperand(1);
24213 case X86ISD::CMOV: {
24214 // Check whether false/true value has canonical one, i.e. 0 or 1.
24215 ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
24216 ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
24217 // Quit if true value is not a constant.
24220 // Quit if false value is not a constant.
24222 SDValue Op = SetCC.getOperand(0);
24223 // Skip 'zext' or 'trunc' node.
24224 if (Op.getOpcode() == ISD::ZERO_EXTEND ||
24225 Op.getOpcode() == ISD::TRUNCATE)
24226 Op = Op.getOperand(0);
24227 // A special case for rdrand/rdseed, where 0 is set if false cond is
24229 if ((Op.getOpcode() != X86ISD::RDRAND &&
24230 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
24233 // Quit if false value is not the constant 0 or 1.
24234 bool FValIsFalse = true;
24235 if (FVal && FVal->getZExtValue() != 0) {
24236 if (FVal->getZExtValue() != 1)
24238 // If FVal is 1, opposite cond is needed.
24239 needOppositeCond = !needOppositeCond;
24240 FValIsFalse = false;
24242 // Quit if TVal is not the constant opposite of FVal.
24243 if (FValIsFalse && TVal->getZExtValue() != 1)
24245 if (!FValIsFalse && TVal->getZExtValue() != 0)
24247 CC = X86::CondCode(SetCC.getConstantOperandVal(2));
24248 if (needOppositeCond)
24249 CC = X86::GetOppositeBranchCondition(CC);
24250 return SetCC.getOperand(3);
24257 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
24258 static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
24259 TargetLowering::DAGCombinerInfo &DCI,
24260 const X86Subtarget *Subtarget) {
24263 // If the flag operand isn't dead, don't touch this CMOV.
24264 if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
24267 SDValue FalseOp = N->getOperand(0);
24268 SDValue TrueOp = N->getOperand(1);
24269 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
24270 SDValue Cond = N->getOperand(3);
24272 if (CC == X86::COND_E || CC == X86::COND_NE) {
24273 switch (Cond.getOpcode()) {
24277 // If operand of BSR / BSF are proven never zero, then ZF cannot be set.
24278 if (DAG.isKnownNeverZero(Cond.getOperand(0)))
24279 return (CC == X86::COND_E) ? FalseOp : TrueOp;
24285 Flags = checkBoolTestSetCCCombine(Cond, CC);
24286 if (Flags.getNode() &&
24287 // Extra check as FCMOV only supports a subset of X86 cond.
24288 (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) {
24289 SDValue Ops[] = { FalseOp, TrueOp,
24290 DAG.getConstant(CC, MVT::i8), Flags };
24291 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
24294 // If this is a select between two integer constants, try to do some
24295 // optimizations. Note that the operands are ordered the opposite of SELECT
24297 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
24298 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
24299 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
24300 // larger than FalseC (the false value).
24301 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
24302 CC = X86::GetOppositeBranchCondition(CC);
24303 std::swap(TrueC, FalseC);
24304 std::swap(TrueOp, FalseOp);
24307 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0.
24308 // This is efficient for any integer data type (including i8/i16) and
24310 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
24311 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
24312 DAG.getConstant(CC, MVT::i8), Cond);
24314 // Zero extend the condition if needed.
24315 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
24317 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
24318 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
24319 DAG.getConstant(ShAmt, MVT::i8));
24320 if (N->getNumValues() == 2) // Dead flag value?
24321 return DCI.CombineTo(N, Cond, SDValue());
24325 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
24326 // for any integer data type, including i8/i16.
24327 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
24328 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
24329 DAG.getConstant(CC, MVT::i8), Cond);
24331 // Zero extend the condition if needed.
24332 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
24333 FalseC->getValueType(0), Cond);
24334 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
24335 SDValue(FalseC, 0));
24337 if (N->getNumValues() == 2) // Dead flag value?
24338 return DCI.CombineTo(N, Cond, SDValue());
24342 // Optimize cases that will turn into an LEA instruction. This requires
24343 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
24344 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
24345 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
24346 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
24348 bool isFastMultiplier = false;
24350 switch ((unsigned char)Diff) {
24352 case 1: // result = add base, cond
24353 case 2: // result = lea base( , cond*2)
24354 case 3: // result = lea base(cond, cond*2)
24355 case 4: // result = lea base( , cond*4)
24356 case 5: // result = lea base(cond, cond*4)
24357 case 8: // result = lea base( , cond*8)
24358 case 9: // result = lea base(cond, cond*8)
24359 isFastMultiplier = true;
24364 if (isFastMultiplier) {
24365 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
24366 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
24367 DAG.getConstant(CC, MVT::i8), Cond);
24368 // Zero extend the condition if needed.
24369 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
24371 // Scale the condition by the difference.
24373 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
24374 DAG.getConstant(Diff, Cond.getValueType()));
24376 // Add the base if non-zero.
24377 if (FalseC->getAPIntValue() != 0)
24378 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
24379 SDValue(FalseC, 0));
24380 if (N->getNumValues() == 2) // Dead flag value?
24381 return DCI.CombineTo(N, Cond, SDValue());
24388 // Handle these cases:
24389 // (select (x != c), e, c) -> select (x != c), e, x),
24390 // (select (x == c), c, e) -> select (x == c), x, e)
24391 // where the c is an integer constant, and the "select" is the combination
24392 // of CMOV and CMP.
24394 // The rationale for this change is that the conditional-move from a constant
24395 // needs two instructions, however, conditional-move from a register needs
24396 // only one instruction.
24398 // CAVEAT: By replacing a constant with a symbolic value, it may obscure
24399 // some instruction-combining opportunities. This opt needs to be
24400 // postponed as late as possible.
24402 if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
24403 // the DCI.xxxx conditions are provided to postpone the optimization as
24404 // late as possible.
24406 ConstantSDNode *CmpAgainst = nullptr;
24407 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
24408 (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
24409 !isa<ConstantSDNode>(Cond.getOperand(0))) {
24411 if (CC == X86::COND_NE &&
24412 CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
24413 CC = X86::GetOppositeBranchCondition(CC);
24414 std::swap(TrueOp, FalseOp);
24417 if (CC == X86::COND_E &&
24418 CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
24419 SDValue Ops[] = { FalseOp, Cond.getOperand(0),
24420 DAG.getConstant(CC, MVT::i8), Cond };
24421 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops);
24429 static SDValue PerformINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
24430 const X86Subtarget *Subtarget) {
24431 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
24433 default: return SDValue();
24434 // SSE/AVX/AVX2 blend intrinsics.
24435 case Intrinsic::x86_avx2_pblendvb:
24436 case Intrinsic::x86_avx2_pblendw:
24437 case Intrinsic::x86_avx2_pblendd_128:
24438 case Intrinsic::x86_avx2_pblendd_256:
24439 // Don't try to simplify this intrinsic if we don't have AVX2.
24440 if (!Subtarget->hasAVX2())
24443 case Intrinsic::x86_avx_blend_pd_256:
24444 case Intrinsic::x86_avx_blend_ps_256:
24445 case Intrinsic::x86_avx_blendv_pd_256:
24446 case Intrinsic::x86_avx_blendv_ps_256:
24447 // Don't try to simplify this intrinsic if we don't have AVX.
24448 if (!Subtarget->hasAVX())
24451 case Intrinsic::x86_sse41_pblendw:
24452 case Intrinsic::x86_sse41_blendpd:
24453 case Intrinsic::x86_sse41_blendps:
24454 case Intrinsic::x86_sse41_blendvps:
24455 case Intrinsic::x86_sse41_blendvpd:
24456 case Intrinsic::x86_sse41_pblendvb: {
24457 SDValue Op0 = N->getOperand(1);
24458 SDValue Op1 = N->getOperand(2);
24459 SDValue Mask = N->getOperand(3);
24461 // Don't try to simplify this intrinsic if we don't have SSE4.1.
24462 if (!Subtarget->hasSSE41())
24465 // fold (blend A, A, Mask) -> A
24468 // fold (blend A, B, allZeros) -> A
24469 if (ISD::isBuildVectorAllZeros(Mask.getNode()))
24471 // fold (blend A, B, allOnes) -> B
24472 if (ISD::isBuildVectorAllOnes(Mask.getNode()))
24475 // Simplify the case where the mask is a constant i32 value.
24476 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Mask)) {
24477 if (C->isNullValue())
24479 if (C->isAllOnesValue())
24486 // Packed SSE2/AVX2 arithmetic shift immediate intrinsics.
24487 case Intrinsic::x86_sse2_psrai_w:
24488 case Intrinsic::x86_sse2_psrai_d:
24489 case Intrinsic::x86_avx2_psrai_w:
24490 case Intrinsic::x86_avx2_psrai_d:
24491 case Intrinsic::x86_sse2_psra_w:
24492 case Intrinsic::x86_sse2_psra_d:
24493 case Intrinsic::x86_avx2_psra_w:
24494 case Intrinsic::x86_avx2_psra_d: {
24495 SDValue Op0 = N->getOperand(1);
24496 SDValue Op1 = N->getOperand(2);
24497 EVT VT = Op0.getValueType();
24498 assert(VT.isVector() && "Expected a vector type!");
24500 if (isa<BuildVectorSDNode>(Op1))
24501 Op1 = Op1.getOperand(0);
24503 if (!isa<ConstantSDNode>(Op1))
24506 EVT SVT = VT.getVectorElementType();
24507 unsigned SVTBits = SVT.getSizeInBits();
24509 ConstantSDNode *CND = cast<ConstantSDNode>(Op1);
24510 const APInt &C = APInt(SVTBits, CND->getAPIntValue().getZExtValue());
24511 uint64_t ShAmt = C.getZExtValue();
24513 // Don't try to convert this shift into a ISD::SRA if the shift
24514 // count is bigger than or equal to the element size.
24515 if (ShAmt >= SVTBits)
24518 // Trivial case: if the shift count is zero, then fold this
24519 // into the first operand.
24523 // Replace this packed shift intrinsic with a target independent
24525 SDValue Splat = DAG.getConstant(C, VT);
24526 return DAG.getNode(ISD::SRA, SDLoc(N), VT, Op0, Splat);
24531 /// PerformMulCombine - Optimize a single multiply with constant into two
24532 /// in order to implement it with two cheaper instructions, e.g.
24533 /// LEA + SHL, LEA + LEA.
24534 static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG,
24535 TargetLowering::DAGCombinerInfo &DCI) {
24536 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
24539 EVT VT = N->getValueType(0);
24540 if (VT != MVT::i64 && VT != MVT::i32)
24543 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
24546 uint64_t MulAmt = C->getZExtValue();
24547 if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9)
24550 uint64_t MulAmt1 = 0;
24551 uint64_t MulAmt2 = 0;
24552 if ((MulAmt % 9) == 0) {
24554 MulAmt2 = MulAmt / 9;
24555 } else if ((MulAmt % 5) == 0) {
24557 MulAmt2 = MulAmt / 5;
24558 } else if ((MulAmt % 3) == 0) {
24560 MulAmt2 = MulAmt / 3;
24563 (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){
24566 if (isPowerOf2_64(MulAmt2) &&
24567 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD))
24568 // If second multiplifer is pow2, issue it first. We want the multiply by
24569 // 3, 5, or 9 to be folded into the addressing mode unless the lone use
24571 std::swap(MulAmt1, MulAmt2);
24574 if (isPowerOf2_64(MulAmt1))
24575 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
24576 DAG.getConstant(Log2_64(MulAmt1), MVT::i8));
24578 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
24579 DAG.getConstant(MulAmt1, VT));
24581 if (isPowerOf2_64(MulAmt2))
24582 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
24583 DAG.getConstant(Log2_64(MulAmt2), MVT::i8));
24585 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
24586 DAG.getConstant(MulAmt2, VT));
24588 // Do not add new nodes to DAG combiner worklist.
24589 DCI.CombineTo(N, NewMul, false);
24594 static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) {
24595 SDValue N0 = N->getOperand(0);
24596 SDValue N1 = N->getOperand(1);
24597 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
24598 EVT VT = N0.getValueType();
24600 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
24601 // since the result of setcc_c is all zero's or all ones.
24602 if (VT.isInteger() && !VT.isVector() &&
24603 N1C && N0.getOpcode() == ISD::AND &&
24604 N0.getOperand(1).getOpcode() == ISD::Constant) {
24605 SDValue N00 = N0.getOperand(0);
24606 if (N00.getOpcode() == X86ISD::SETCC_CARRY ||
24607 ((N00.getOpcode() == ISD::ANY_EXTEND ||
24608 N00.getOpcode() == ISD::ZERO_EXTEND) &&
24609 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) {
24610 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
24611 APInt ShAmt = N1C->getAPIntValue();
24612 Mask = Mask.shl(ShAmt);
24614 return DAG.getNode(ISD::AND, SDLoc(N), VT,
24615 N00, DAG.getConstant(Mask, VT));
24619 // Hardware support for vector shifts is sparse which makes us scalarize the
24620 // vector operations in many cases. Also, on sandybridge ADD is faster than
24622 // (shl V, 1) -> add V,V
24623 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
24624 if (auto *N1SplatC = N1BV->getConstantSplatNode()) {
24625 assert(N0.getValueType().isVector() && "Invalid vector shift type");
24626 // We shift all of the values by one. In many cases we do not have
24627 // hardware support for this operation. This is better expressed as an ADD
24629 if (N1SplatC->getZExtValue() == 1)
24630 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);
24636 /// \brief Returns a vector of 0s if the node in input is a vector logical
24637 /// shift by a constant amount which is known to be bigger than or equal
24638 /// to the vector element size in bits.
24639 static SDValue performShiftToAllZeros(SDNode *N, SelectionDAG &DAG,
24640 const X86Subtarget *Subtarget) {
24641 EVT VT = N->getValueType(0);
24643 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 &&
24644 (!Subtarget->hasInt256() ||
24645 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16)))
24648 SDValue Amt = N->getOperand(1);
24650 if (auto *AmtBV = dyn_cast<BuildVectorSDNode>(Amt))
24651 if (auto *AmtSplat = AmtBV->getConstantSplatNode()) {
24652 APInt ShiftAmt = AmtSplat->getAPIntValue();
24653 unsigned MaxAmount = VT.getVectorElementType().getSizeInBits();
24655 // SSE2/AVX2 logical shifts always return a vector of 0s
24656 // if the shift amount is bigger than or equal to
24657 // the element size. The constant shift amount will be
24658 // encoded as a 8-bit immediate.
24659 if (ShiftAmt.trunc(8).uge(MaxAmount))
24660 return getZeroVector(VT, Subtarget, DAG, DL);
24666 /// PerformShiftCombine - Combine shifts.
24667 static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG,
24668 TargetLowering::DAGCombinerInfo &DCI,
24669 const X86Subtarget *Subtarget) {
24670 if (N->getOpcode() == ISD::SHL) {
24671 SDValue V = PerformSHLCombine(N, DAG);
24672 if (V.getNode()) return V;
24675 if (N->getOpcode() != ISD::SRA) {
24676 // Try to fold this logical shift into a zero vector.
24677 SDValue V = performShiftToAllZeros(N, DAG, Subtarget);
24678 if (V.getNode()) return V;
24684 // CMPEQCombine - Recognize the distinctive (AND (setcc ...) (setcc ..))
24685 // where both setccs reference the same FP CMP, and rewrite for CMPEQSS
24686 // and friends. Likewise for OR -> CMPNEQSS.
24687 static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
24688 TargetLowering::DAGCombinerInfo &DCI,
24689 const X86Subtarget *Subtarget) {
24692 // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
24693 // we're requiring SSE2 for both.
24694 if (Subtarget->hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
24695 SDValue N0 = N->getOperand(0);
24696 SDValue N1 = N->getOperand(1);
24697 SDValue CMP0 = N0->getOperand(1);
24698 SDValue CMP1 = N1->getOperand(1);
24701 // The SETCCs should both refer to the same CMP.
24702 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
24705 SDValue CMP00 = CMP0->getOperand(0);
24706 SDValue CMP01 = CMP0->getOperand(1);
24707 EVT VT = CMP00.getValueType();
24709 if (VT == MVT::f32 || VT == MVT::f64) {
24710 bool ExpectingFlags = false;
24711 // Check for any users that want flags:
24712 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
24713 !ExpectingFlags && UI != UE; ++UI)
24714 switch (UI->getOpcode()) {
24719 ExpectingFlags = true;
24721 case ISD::CopyToReg:
24722 case ISD::SIGN_EXTEND:
24723 case ISD::ZERO_EXTEND:
24724 case ISD::ANY_EXTEND:
24728 if (!ExpectingFlags) {
24729 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
24730 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
24732 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
24733 X86::CondCode tmp = cc0;
24738 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
24739 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
24740 // FIXME: need symbolic constants for these magic numbers.
24741 // See X86ATTInstPrinter.cpp:printSSECC().
24742 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
24743 if (Subtarget->hasAVX512()) {
24744 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00,
24745 CMP01, DAG.getConstant(x86cc, MVT::i8));
24746 if (N->getValueType(0) != MVT::i1)
24747 return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0),
24751 SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
24752 CMP00.getValueType(), CMP00, CMP01,
24753 DAG.getConstant(x86cc, MVT::i8));
24755 bool is64BitFP = (CMP00.getValueType() == MVT::f64);
24756 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
24758 if (is64BitFP && !Subtarget->is64Bit()) {
24759 // On a 32-bit target, we cannot bitcast the 64-bit float to a
24760 // 64-bit integer, since that's not a legal type. Since
24761 // OnesOrZeroesF is all ones of all zeroes, we don't need all the
24762 // bits, but can do this little dance to extract the lowest 32 bits
24763 // and work with those going forward.
24764 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
24766 SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32,
24768 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
24769 Vector32, DAG.getIntPtrConstant(0));
24773 SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, IntVT, OnesOrZeroesF);
24774 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
24775 DAG.getConstant(1, IntVT));
24776 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed);
24777 return OneBitOfTruth;
24785 /// CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector
24786 /// so it can be folded inside ANDNP.
24787 static bool CanFoldXORWithAllOnes(const SDNode *N) {
24788 EVT VT = N->getValueType(0);
24790 // Match direct AllOnes for 128 and 256-bit vectors
24791 if (ISD::isBuildVectorAllOnes(N))
24794 // Look through a bit convert.
24795 if (N->getOpcode() == ISD::BITCAST)
24796 N = N->getOperand(0).getNode();
24798 // Sometimes the operand may come from a insert_subvector building a 256-bit
24800 if (VT.is256BitVector() &&
24801 N->getOpcode() == ISD::INSERT_SUBVECTOR) {
24802 SDValue V1 = N->getOperand(0);
24803 SDValue V2 = N->getOperand(1);
24805 if (V1.getOpcode() == ISD::INSERT_SUBVECTOR &&
24806 V1.getOperand(0).getOpcode() == ISD::UNDEF &&
24807 ISD::isBuildVectorAllOnes(V1.getOperand(1).getNode()) &&
24808 ISD::isBuildVectorAllOnes(V2.getNode()))
24815 // On AVX/AVX2 the type v8i1 is legalized to v8i16, which is an XMM sized
24816 // register. In most cases we actually compare or select YMM-sized registers
24817 // and mixing the two types creates horrible code. This method optimizes
24818 // some of the transition sequences.
24819 static SDValue WidenMaskArithmetic(SDNode *N, SelectionDAG &DAG,
24820 TargetLowering::DAGCombinerInfo &DCI,
24821 const X86Subtarget *Subtarget) {
24822 EVT VT = N->getValueType(0);
24823 if (!VT.is256BitVector())
24826 assert((N->getOpcode() == ISD::ANY_EXTEND ||
24827 N->getOpcode() == ISD::ZERO_EXTEND ||
24828 N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
24830 SDValue Narrow = N->getOperand(0);
24831 EVT NarrowVT = Narrow->getValueType(0);
24832 if (!NarrowVT.is128BitVector())
24835 if (Narrow->getOpcode() != ISD::XOR &&
24836 Narrow->getOpcode() != ISD::AND &&
24837 Narrow->getOpcode() != ISD::OR)
24840 SDValue N0 = Narrow->getOperand(0);
24841 SDValue N1 = Narrow->getOperand(1);
24844 // The Left side has to be a trunc.
24845 if (N0.getOpcode() != ISD::TRUNCATE)
24848 // The type of the truncated inputs.
24849 EVT WideVT = N0->getOperand(0)->getValueType(0);
24853 // The right side has to be a 'trunc' or a constant vector.
24854 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE;
24855 ConstantSDNode *RHSConstSplat = nullptr;
24856 if (auto *RHSBV = dyn_cast<BuildVectorSDNode>(N1))
24857 RHSConstSplat = RHSBV->getConstantSplatNode();
24858 if (!RHSTrunc && !RHSConstSplat)
24861 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24863 if (!TLI.isOperationLegalOrPromote(Narrow->getOpcode(), WideVT))
24866 // Set N0 and N1 to hold the inputs to the new wide operation.
24867 N0 = N0->getOperand(0);
24868 if (RHSConstSplat) {
24869 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT.getScalarType(),
24870 SDValue(RHSConstSplat, 0));
24871 SmallVector<SDValue, 8> C(WideVT.getVectorNumElements(), N1);
24872 N1 = DAG.getNode(ISD::BUILD_VECTOR, DL, WideVT, C);
24873 } else if (RHSTrunc) {
24874 N1 = N1->getOperand(0);
24877 // Generate the wide operation.
24878 SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, WideVT, N0, N1);
24879 unsigned Opcode = N->getOpcode();
24881 case ISD::ANY_EXTEND:
24883 case ISD::ZERO_EXTEND: {
24884 unsigned InBits = NarrowVT.getScalarType().getSizeInBits();
24885 APInt Mask = APInt::getAllOnesValue(InBits);
24886 Mask = Mask.zext(VT.getScalarType().getSizeInBits());
24887 return DAG.getNode(ISD::AND, DL, VT,
24888 Op, DAG.getConstant(Mask, VT));
24890 case ISD::SIGN_EXTEND:
24891 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
24892 Op, DAG.getValueType(NarrowVT));
24894 llvm_unreachable("Unexpected opcode");
24898 static SDValue VectorZextCombine(SDNode *N, SelectionDAG &DAG,
24899 TargetLowering::DAGCombinerInfo &DCI,
24900 const X86Subtarget *Subtarget) {
24901 SDValue N0 = N->getOperand(0);
24902 SDValue N1 = N->getOperand(1);
24905 // A vector zext_in_reg may be represented as a shuffle,
24906 // feeding into a bitcast (this represents anyext) feeding into
24907 // an and with a mask.
24908 // We'd like to try to combine that into a shuffle with zero
24909 // plus a bitcast, removing the and.
24910 if (N0.getOpcode() != ISD::BITCAST ||
24911 N0.getOperand(0).getOpcode() != ISD::VECTOR_SHUFFLE)
24914 // The other side of the AND should be a splat of 2^C, where C
24915 // is the number of bits in the source type.
24916 if (N1.getOpcode() == ISD::BITCAST)
24917 N1 = N1.getOperand(0);
24918 if (N1.getOpcode() != ISD::BUILD_VECTOR)
24920 BuildVectorSDNode *Vector = cast<BuildVectorSDNode>(N1);
24922 ShuffleVectorSDNode *Shuffle = cast<ShuffleVectorSDNode>(N0.getOperand(0));
24923 EVT SrcType = Shuffle->getValueType(0);
24925 // We expect a single-source shuffle
24926 if (Shuffle->getOperand(1)->getOpcode() != ISD::UNDEF)
24929 unsigned SrcSize = SrcType.getScalarSizeInBits();
24931 APInt SplatValue, SplatUndef;
24932 unsigned SplatBitSize;
24934 if (!Vector->isConstantSplat(SplatValue, SplatUndef,
24935 SplatBitSize, HasAnyUndefs))
24938 unsigned ResSize = N1.getValueType().getScalarSizeInBits();
24939 // Make sure the splat matches the mask we expect
24940 if (SplatBitSize > ResSize ||
24941 (SplatValue + 1).exactLogBase2() != (int)SrcSize)
24944 // Make sure the input and output size make sense
24945 if (SrcSize >= ResSize || ResSize % SrcSize)
24948 // We expect a shuffle of the form <0, u, u, u, 1, u, u, u...>
24949 // The number of u's between each two values depends on the ratio between
24950 // the source and dest type.
24951 unsigned ZextRatio = ResSize / SrcSize;
24952 bool IsZext = true;
24953 for (unsigned i = 0; i < SrcType.getVectorNumElements(); ++i) {
24954 if (i % ZextRatio) {
24955 if (Shuffle->getMaskElt(i) > 0) {
24961 if (Shuffle->getMaskElt(i) != (int)(i / ZextRatio)) {
24962 // Expected element number
24972 // Ok, perform the transformation - replace the shuffle with
24973 // a shuffle of the form <0, k, k, k, 1, k, k, k> with zero
24974 // (instead of undef) where the k elements come from the zero vector.
24975 SmallVector<int, 8> Mask;
24976 unsigned NumElems = SrcType.getVectorNumElements();
24977 for (unsigned i = 0; i < NumElems; ++i)
24979 Mask.push_back(NumElems);
24981 Mask.push_back(i / ZextRatio);
24983 SDValue NewShuffle = DAG.getVectorShuffle(Shuffle->getValueType(0), DL,
24984 Shuffle->getOperand(0), DAG.getConstant(0, SrcType), Mask);
24985 return DAG.getNode(ISD::BITCAST, DL, N0.getValueType(), NewShuffle);
24988 static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
24989 TargetLowering::DAGCombinerInfo &DCI,
24990 const X86Subtarget *Subtarget) {
24991 if (DCI.isBeforeLegalizeOps())
24994 SDValue Zext = VectorZextCombine(N, DAG, DCI, Subtarget);
24995 if (Zext.getNode())
24998 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
25002 EVT VT = N->getValueType(0);
25003 SDValue N0 = N->getOperand(0);
25004 SDValue N1 = N->getOperand(1);
25007 // Create BEXTR instructions
25008 // BEXTR is ((X >> imm) & (2**size-1))
25009 if (VT == MVT::i32 || VT == MVT::i64) {
25010 // Check for BEXTR.
25011 if ((Subtarget->hasBMI() || Subtarget->hasTBM()) &&
25012 (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL)) {
25013 ConstantSDNode *MaskNode = dyn_cast<ConstantSDNode>(N1);
25014 ConstantSDNode *ShiftNode = dyn_cast<ConstantSDNode>(N0.getOperand(1));
25015 if (MaskNode && ShiftNode) {
25016 uint64_t Mask = MaskNode->getZExtValue();
25017 uint64_t Shift = ShiftNode->getZExtValue();
25018 if (isMask_64(Mask)) {
25019 uint64_t MaskSize = countPopulation(Mask);
25020 if (Shift + MaskSize <= VT.getSizeInBits())
25021 return DAG.getNode(X86ISD::BEXTR, DL, VT, N0.getOperand(0),
25022 DAG.getConstant(Shift | (MaskSize << 8), VT));
25030 // Want to form ANDNP nodes:
25031 // 1) In the hopes of then easily combining them with OR and AND nodes
25032 // to form PBLEND/PSIGN.
25033 // 2) To match ANDN packed intrinsics
25034 if (VT != MVT::v2i64 && VT != MVT::v4i64)
25037 // Check LHS for vnot
25038 if (N0.getOpcode() == ISD::XOR &&
25039 //ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode()))
25040 CanFoldXORWithAllOnes(N0.getOperand(1).getNode()))
25041 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
25043 // Check RHS for vnot
25044 if (N1.getOpcode() == ISD::XOR &&
25045 //ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode()))
25046 CanFoldXORWithAllOnes(N1.getOperand(1).getNode()))
25047 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
25052 static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
25053 TargetLowering::DAGCombinerInfo &DCI,
25054 const X86Subtarget *Subtarget) {
25055 if (DCI.isBeforeLegalizeOps())
25058 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
25062 SDValue N0 = N->getOperand(0);
25063 SDValue N1 = N->getOperand(1);
25064 EVT VT = N->getValueType(0);
25066 // look for psign/blend
25067 if (VT == MVT::v2i64 || VT == MVT::v4i64) {
25068 if (!Subtarget->hasSSSE3() ||
25069 (VT == MVT::v4i64 && !Subtarget->hasInt256()))
25072 // Canonicalize pandn to RHS
25073 if (N0.getOpcode() == X86ISD::ANDNP)
25075 // or (and (m, y), (pandn m, x))
25076 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) {
25077 SDValue Mask = N1.getOperand(0);
25078 SDValue X = N1.getOperand(1);
25080 if (N0.getOperand(0) == Mask)
25081 Y = N0.getOperand(1);
25082 if (N0.getOperand(1) == Mask)
25083 Y = N0.getOperand(0);
25085 // Check to see if the mask appeared in both the AND and ANDNP and
25089 // Validate that X, Y, and Mask are BIT_CONVERTS, and see through them.
25090 // Look through mask bitcast.
25091 if (Mask.getOpcode() == ISD::BITCAST)
25092 Mask = Mask.getOperand(0);
25093 if (X.getOpcode() == ISD::BITCAST)
25094 X = X.getOperand(0);
25095 if (Y.getOpcode() == ISD::BITCAST)
25096 Y = Y.getOperand(0);
25098 EVT MaskVT = Mask.getValueType();
25100 // Validate that the Mask operand is a vector sra node.
25101 // FIXME: what to do for bytes, since there is a psignb/pblendvb, but
25102 // there is no psrai.b
25103 unsigned EltBits = MaskVT.getVectorElementType().getSizeInBits();
25104 unsigned SraAmt = ~0;
25105 if (Mask.getOpcode() == ISD::SRA) {
25106 if (auto *AmtBV = dyn_cast<BuildVectorSDNode>(Mask.getOperand(1)))
25107 if (auto *AmtConst = AmtBV->getConstantSplatNode())
25108 SraAmt = AmtConst->getZExtValue();
25109 } else if (Mask.getOpcode() == X86ISD::VSRAI) {
25110 SDValue SraC = Mask.getOperand(1);
25111 SraAmt = cast<ConstantSDNode>(SraC)->getZExtValue();
25113 if ((SraAmt + 1) != EltBits)
25118 // Now we know we at least have a plendvb with the mask val. See if
25119 // we can form a psignb/w/d.
25120 // psign = x.type == y.type == mask.type && y = sub(0, x);
25121 if (Y.getOpcode() == ISD::SUB && Y.getOperand(1) == X &&
25122 ISD::isBuildVectorAllZeros(Y.getOperand(0).getNode()) &&
25123 X.getValueType() == MaskVT && Y.getValueType() == MaskVT) {
25124 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
25125 "Unsupported VT for PSIGN");
25126 Mask = DAG.getNode(X86ISD::PSIGN, DL, MaskVT, X, Mask.getOperand(0));
25127 return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
25129 // PBLENDVB only available on SSE 4.1
25130 if (!Subtarget->hasSSE41())
25133 EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8;
25135 X = DAG.getNode(ISD::BITCAST, DL, BlendVT, X);
25136 Y = DAG.getNode(ISD::BITCAST, DL, BlendVT, Y);
25137 Mask = DAG.getNode(ISD::BITCAST, DL, BlendVT, Mask);
25138 Mask = DAG.getNode(ISD::VSELECT, DL, BlendVT, Mask, Y, X);
25139 return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
25143 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
25146 // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c)
25147 MachineFunction &MF = DAG.getMachineFunction();
25149 MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
25151 // SHLD/SHRD instructions have lower register pressure, but on some
25152 // platforms they have higher latency than the equivalent
25153 // series of shifts/or that would otherwise be generated.
25154 // Don't fold (or (x << c) | (y >> (64 - c))) if SHLD/SHRD instructions
25155 // have higher latencies and we are not optimizing for size.
25156 if (!OptForSize && Subtarget->isSHLDSlow())
25159 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
25161 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
25163 if (!N0.hasOneUse() || !N1.hasOneUse())
25166 SDValue ShAmt0 = N0.getOperand(1);
25167 if (ShAmt0.getValueType() != MVT::i8)
25169 SDValue ShAmt1 = N1.getOperand(1);
25170 if (ShAmt1.getValueType() != MVT::i8)
25172 if (ShAmt0.getOpcode() == ISD::TRUNCATE)
25173 ShAmt0 = ShAmt0.getOperand(0);
25174 if (ShAmt1.getOpcode() == ISD::TRUNCATE)
25175 ShAmt1 = ShAmt1.getOperand(0);
25178 unsigned Opc = X86ISD::SHLD;
25179 SDValue Op0 = N0.getOperand(0);
25180 SDValue Op1 = N1.getOperand(0);
25181 if (ShAmt0.getOpcode() == ISD::SUB) {
25182 Opc = X86ISD::SHRD;
25183 std::swap(Op0, Op1);
25184 std::swap(ShAmt0, ShAmt1);
25187 unsigned Bits = VT.getSizeInBits();
25188 if (ShAmt1.getOpcode() == ISD::SUB) {
25189 SDValue Sum = ShAmt1.getOperand(0);
25190 if (ConstantSDNode *SumC = dyn_cast<ConstantSDNode>(Sum)) {
25191 SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
25192 if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE)
25193 ShAmt1Op1 = ShAmt1Op1.getOperand(0);
25194 if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0)
25195 return DAG.getNode(Opc, DL, VT,
25197 DAG.getNode(ISD::TRUNCATE, DL,
25200 } else if (ConstantSDNode *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) {
25201 ConstantSDNode *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0);
25203 ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue() == Bits)
25204 return DAG.getNode(Opc, DL, VT,
25205 N0.getOperand(0), N1.getOperand(0),
25206 DAG.getNode(ISD::TRUNCATE, DL,
25213 // Generate NEG and CMOV for integer abs.
25214 static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
25215 EVT VT = N->getValueType(0);
25217 // Since X86 does not have CMOV for 8-bit integer, we don't convert
25218 // 8-bit integer abs to NEG and CMOV.
25219 if (VT.isInteger() && VT.getSizeInBits() == 8)
25222 SDValue N0 = N->getOperand(0);
25223 SDValue N1 = N->getOperand(1);
25226 // Check pattern of XOR(ADD(X,Y), Y) where Y is SRA(X, size(X)-1)
25227 // and change it to SUB and CMOV.
25228 if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
25229 N0.getOpcode() == ISD::ADD &&
25230 N0.getOperand(1) == N1 &&
25231 N1.getOpcode() == ISD::SRA &&
25232 N1.getOperand(0) == N0.getOperand(0))
25233 if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1)))
25234 if (Y1C->getAPIntValue() == VT.getSizeInBits()-1) {
25235 // Generate SUB & CMOV.
25236 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
25237 DAG.getConstant(0, VT), N0.getOperand(0));
25239 SDValue Ops[] = { N0.getOperand(0), Neg,
25240 DAG.getConstant(X86::COND_GE, MVT::i8),
25241 SDValue(Neg.getNode(), 1) };
25242 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops);
25247 // PerformXorCombine - Attempts to turn XOR nodes into BLSMSK nodes
25248 static SDValue PerformXorCombine(SDNode *N, SelectionDAG &DAG,
25249 TargetLowering::DAGCombinerInfo &DCI,
25250 const X86Subtarget *Subtarget) {
25251 if (DCI.isBeforeLegalizeOps())
25254 if (Subtarget->hasCMov()) {
25255 SDValue RV = performIntegerAbsCombine(N, DAG);
25263 /// PerformLOADCombine - Do target-specific dag combines on LOAD nodes.
25264 static SDValue PerformLOADCombine(SDNode *N, SelectionDAG &DAG,
25265 TargetLowering::DAGCombinerInfo &DCI,
25266 const X86Subtarget *Subtarget) {
25267 LoadSDNode *Ld = cast<LoadSDNode>(N);
25268 EVT RegVT = Ld->getValueType(0);
25269 EVT MemVT = Ld->getMemoryVT();
25271 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25273 // For chips with slow 32-byte unaligned loads, break the 32-byte operation
25274 // into two 16-byte operations.
25275 ISD::LoadExtType Ext = Ld->getExtensionType();
25276 unsigned Alignment = Ld->getAlignment();
25277 bool IsAligned = Alignment == 0 || Alignment >= MemVT.getSizeInBits()/8;
25278 if (RegVT.is256BitVector() && Subtarget->isUnalignedMem32Slow() &&
25279 !DCI.isBeforeLegalizeOps() && !IsAligned && Ext == ISD::NON_EXTLOAD) {
25280 unsigned NumElems = RegVT.getVectorNumElements();
25284 SDValue Ptr = Ld->getBasePtr();
25285 SDValue Increment = DAG.getConstant(16, TLI.getPointerTy());
25287 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
25289 SDValue Load1 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr,
25290 Ld->getPointerInfo(), Ld->isVolatile(),
25291 Ld->isNonTemporal(), Ld->isInvariant(),
25293 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
25294 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr,
25295 Ld->getPointerInfo(), Ld->isVolatile(),
25296 Ld->isNonTemporal(), Ld->isInvariant(),
25297 std::min(16U, Alignment));
25298 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
25300 Load2.getValue(1));
25302 SDValue NewVec = DAG.getUNDEF(RegVT);
25303 NewVec = Insert128BitVector(NewVec, Load1, 0, DAG, dl);
25304 NewVec = Insert128BitVector(NewVec, Load2, NumElems/2, DAG, dl);
25305 return DCI.CombineTo(N, NewVec, TF, true);
25311 /// PerformMLOADCombine - Resolve extending loads
25312 static SDValue PerformMLOADCombine(SDNode *N, SelectionDAG &DAG,
25313 TargetLowering::DAGCombinerInfo &DCI,
25314 const X86Subtarget *Subtarget) {
25315 MaskedLoadSDNode *Mld = cast<MaskedLoadSDNode>(N);
25316 if (Mld->getExtensionType() != ISD::SEXTLOAD)
25319 EVT VT = Mld->getValueType(0);
25320 unsigned NumElems = VT.getVectorNumElements();
25321 EVT LdVT = Mld->getMemoryVT();
25324 assert(LdVT != VT && "Cannot extend to the same type");
25325 unsigned ToSz = VT.getVectorElementType().getSizeInBits();
25326 unsigned FromSz = LdVT.getVectorElementType().getSizeInBits();
25327 // From, To sizes and ElemCount must be pow of two
25328 assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
25329 "Unexpected size for extending masked load");
25331 unsigned SizeRatio = ToSz / FromSz;
25332 assert(SizeRatio * NumElems * FromSz == VT.getSizeInBits());
25334 // Create a type on which we perform the shuffle
25335 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
25336 LdVT.getScalarType(), NumElems*SizeRatio);
25337 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
25339 // Convert Src0 value
25340 SDValue WideSrc0 = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mld->getSrc0());
25341 if (Mld->getSrc0().getOpcode() != ISD::UNDEF) {
25342 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
25343 for (unsigned i = 0; i != NumElems; ++i)
25344 ShuffleVec[i] = i * SizeRatio;
25346 // Can't shuffle using an illegal type.
25347 assert (DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT)
25348 && "WideVecVT should be legal");
25349 WideSrc0 = DAG.getVectorShuffle(WideVecVT, dl, WideSrc0,
25350 DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
25352 // Prepare the new mask
25354 SDValue Mask = Mld->getMask();
25355 if (Mask.getValueType() == VT) {
25356 // Mask and original value have the same type
25357 NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask);
25358 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
25359 for (unsigned i = 0; i != NumElems; ++i)
25360 ShuffleVec[i] = i * SizeRatio;
25361 for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
25362 ShuffleVec[i] = NumElems*SizeRatio;
25363 NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
25364 DAG.getConstant(0, WideVecVT),
25368 assert(Mask.getValueType().getVectorElementType() == MVT::i1);
25369 unsigned WidenNumElts = NumElems*SizeRatio;
25370 unsigned MaskNumElts = VT.getVectorNumElements();
25371 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
25374 unsigned NumConcat = WidenNumElts / MaskNumElts;
25375 SmallVector<SDValue, 16> Ops(NumConcat);
25376 SDValue ZeroVal = DAG.getConstant(0, Mask.getValueType());
25378 for (unsigned i = 1; i != NumConcat; ++i)
25381 NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops);
25384 SDValue WideLd = DAG.getMaskedLoad(WideVecVT, dl, Mld->getChain(),
25385 Mld->getBasePtr(), NewMask, WideSrc0,
25386 Mld->getMemoryVT(), Mld->getMemOperand(),
25388 SDValue NewVec = DAG.getNode(X86ISD::VSEXT, dl, VT, WideLd);
25389 return DCI.CombineTo(N, NewVec, WideLd.getValue(1), true);
25392 /// PerformMSTORECombine - Resolve truncating stores
25393 static SDValue PerformMSTORECombine(SDNode *N, SelectionDAG &DAG,
25394 const X86Subtarget *Subtarget) {
25395 MaskedStoreSDNode *Mst = cast<MaskedStoreSDNode>(N);
25396 if (!Mst->isTruncatingStore())
25399 EVT VT = Mst->getValue().getValueType();
25400 unsigned NumElems = VT.getVectorNumElements();
25401 EVT StVT = Mst->getMemoryVT();
25404 assert(StVT != VT && "Cannot truncate to the same type");
25405 unsigned FromSz = VT.getVectorElementType().getSizeInBits();
25406 unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
25408 // From, To sizes and ElemCount must be pow of two
25409 assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
25410 "Unexpected size for truncating masked store");
25411 // We are going to use the original vector elt for storing.
25412 // Accumulated smaller vector elements must be a multiple of the store size.
25413 assert (((NumElems * FromSz) % ToSz) == 0 &&
25414 "Unexpected ratio for truncating masked store");
25416 unsigned SizeRatio = FromSz / ToSz;
25417 assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
25419 // Create a type on which we perform the shuffle
25420 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
25421 StVT.getScalarType(), NumElems*SizeRatio);
25423 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
25425 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mst->getValue());
25426 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
25427 for (unsigned i = 0; i != NumElems; ++i)
25428 ShuffleVec[i] = i * SizeRatio;
25430 // Can't shuffle using an illegal type.
25431 assert (DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT)
25432 && "WideVecVT should be legal");
25434 SDValue TruncatedVal = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
25435 DAG.getUNDEF(WideVecVT),
25439 SDValue Mask = Mst->getMask();
25440 if (Mask.getValueType() == VT) {
25441 // Mask and original value have the same type
25442 NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask);
25443 for (unsigned i = 0; i != NumElems; ++i)
25444 ShuffleVec[i] = i * SizeRatio;
25445 for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
25446 ShuffleVec[i] = NumElems*SizeRatio;
25447 NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
25448 DAG.getConstant(0, WideVecVT),
25452 assert(Mask.getValueType().getVectorElementType() == MVT::i1);
25453 unsigned WidenNumElts = NumElems*SizeRatio;
25454 unsigned MaskNumElts = VT.getVectorNumElements();
25455 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
25458 unsigned NumConcat = WidenNumElts / MaskNumElts;
25459 SmallVector<SDValue, 16> Ops(NumConcat);
25460 SDValue ZeroVal = DAG.getConstant(0, Mask.getValueType());
25462 for (unsigned i = 1; i != NumConcat; ++i)
25465 NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops);
25468 return DAG.getMaskedStore(Mst->getChain(), dl, TruncatedVal, Mst->getBasePtr(),
25469 NewMask, StVT, Mst->getMemOperand(), false);
25471 /// PerformSTORECombine - Do target-specific dag combines on STORE nodes.
25472 static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
25473 const X86Subtarget *Subtarget) {
25474 StoreSDNode *St = cast<StoreSDNode>(N);
25475 EVT VT = St->getValue().getValueType();
25476 EVT StVT = St->getMemoryVT();
25478 SDValue StoredVal = St->getOperand(1);
25479 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25481 // If we are saving a concatenation of two XMM registers and 32-byte stores
25482 // are slow, such as on Sandy Bridge, perform two 16-byte stores.
25483 unsigned Alignment = St->getAlignment();
25484 bool IsAligned = Alignment == 0 || Alignment >= VT.getSizeInBits()/8;
25485 if (VT.is256BitVector() && Subtarget->isUnalignedMem32Slow() &&
25486 StVT == VT && !IsAligned) {
25487 unsigned NumElems = VT.getVectorNumElements();
25491 SDValue Value0 = Extract128BitVector(StoredVal, 0, DAG, dl);
25492 SDValue Value1 = Extract128BitVector(StoredVal, NumElems/2, DAG, dl);
25494 SDValue Stride = DAG.getConstant(16, TLI.getPointerTy());
25495 SDValue Ptr0 = St->getBasePtr();
25496 SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride);
25498 SDValue Ch0 = DAG.getStore(St->getChain(), dl, Value0, Ptr0,
25499 St->getPointerInfo(), St->isVolatile(),
25500 St->isNonTemporal(), Alignment);
25501 SDValue Ch1 = DAG.getStore(St->getChain(), dl, Value1, Ptr1,
25502 St->getPointerInfo(), St->isVolatile(),
25503 St->isNonTemporal(),
25504 std::min(16U, Alignment));
25505 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
25508 // Optimize trunc store (of multiple scalars) to shuffle and store.
25509 // First, pack all of the elements in one place. Next, store to memory
25510 // in fewer chunks.
25511 if (St->isTruncatingStore() && VT.isVector()) {
25512 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25513 unsigned NumElems = VT.getVectorNumElements();
25514 assert(StVT != VT && "Cannot truncate to the same type");
25515 unsigned FromSz = VT.getVectorElementType().getSizeInBits();
25516 unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
25518 // From, To sizes and ElemCount must be pow of two
25519 if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue();
25520 // We are going to use the original vector elt for storing.
25521 // Accumulated smaller vector elements must be a multiple of the store size.
25522 if (0 != (NumElems * FromSz) % ToSz) return SDValue();
25524 unsigned SizeRatio = FromSz / ToSz;
25526 assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
25528 // Create a type on which we perform the shuffle
25529 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
25530 StVT.getScalarType(), NumElems*SizeRatio);
25532 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
25534 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue());
25535 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
25536 for (unsigned i = 0; i != NumElems; ++i)
25537 ShuffleVec[i] = i * SizeRatio;
25539 // Can't shuffle using an illegal type.
25540 if (!TLI.isTypeLegal(WideVecVT))
25543 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
25544 DAG.getUNDEF(WideVecVT),
25546 // At this point all of the data is stored at the bottom of the
25547 // register. We now need to save it to mem.
25549 // Find the largest store unit
25550 MVT StoreType = MVT::i8;
25551 for (MVT Tp : MVT::integer_valuetypes()) {
25552 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToSz)
25556 // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
25557 if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 &&
25558 (64 <= NumElems * ToSz))
25559 StoreType = MVT::f64;
25561 // Bitcast the original vector into a vector of store-size units
25562 EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
25563 StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());
25564 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
25565 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff);
25566 SmallVector<SDValue, 8> Chains;
25567 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8,
25568 TLI.getPointerTy());
25569 SDValue Ptr = St->getBasePtr();
25571 // Perform one or more big stores into memory.
25572 for (unsigned i=0, e=(ToSz*NumElems)/StoreType.getSizeInBits(); i!=e; ++i) {
25573 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
25574 StoreType, ShuffWide,
25575 DAG.getIntPtrConstant(i));
25576 SDValue Ch = DAG.getStore(St->getChain(), dl, SubVec, Ptr,
25577 St->getPointerInfo(), St->isVolatile(),
25578 St->isNonTemporal(), St->getAlignment());
25579 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
25580 Chains.push_back(Ch);
25583 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
25586 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering
25587 // the FP state in cases where an emms may be missing.
25588 // A preferable solution to the general problem is to figure out the right
25589 // places to insert EMMS. This qualifies as a quick hack.
25591 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
25592 if (VT.getSizeInBits() != 64)
25595 const Function *F = DAG.getMachineFunction().getFunction();
25596 bool NoImplicitFloatOps = F->hasFnAttribute(Attribute::NoImplicitFloat);
25597 bool F64IsLegal = !DAG.getTarget().Options.UseSoftFloat && !NoImplicitFloatOps
25598 && Subtarget->hasSSE2();
25599 if ((VT.isVector() ||
25600 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
25601 isa<LoadSDNode>(St->getValue()) &&
25602 !cast<LoadSDNode>(St->getValue())->isVolatile() &&
25603 St->getChain().hasOneUse() && !St->isVolatile()) {
25604 SDNode* LdVal = St->getValue().getNode();
25605 LoadSDNode *Ld = nullptr;
25606 int TokenFactorIndex = -1;
25607 SmallVector<SDValue, 8> Ops;
25608 SDNode* ChainVal = St->getChain().getNode();
25609 // Must be a store of a load. We currently handle two cases: the load
25610 // is a direct child, and it's under an intervening TokenFactor. It is
25611 // possible to dig deeper under nested TokenFactors.
25612 if (ChainVal == LdVal)
25613 Ld = cast<LoadSDNode>(St->getChain());
25614 else if (St->getValue().hasOneUse() &&
25615 ChainVal->getOpcode() == ISD::TokenFactor) {
25616 for (unsigned i = 0, e = ChainVal->getNumOperands(); i != e; ++i) {
25617 if (ChainVal->getOperand(i).getNode() == LdVal) {
25618 TokenFactorIndex = i;
25619 Ld = cast<LoadSDNode>(St->getValue());
25621 Ops.push_back(ChainVal->getOperand(i));
25625 if (!Ld || !ISD::isNormalLoad(Ld))
25628 // If this is not the MMX case, i.e. we are just turning i64 load/store
25629 // into f64 load/store, avoid the transformation if there are multiple
25630 // uses of the loaded value.
25631 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))
25636 // If we are a 64-bit capable x86, lower to a single movq load/store pair.
25637 // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store
25639 if (Subtarget->is64Bit() || F64IsLegal) {
25640 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64;
25641 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(),
25642 Ld->getPointerInfo(), Ld->isVolatile(),
25643 Ld->isNonTemporal(), Ld->isInvariant(),
25644 Ld->getAlignment());
25645 SDValue NewChain = NewLd.getValue(1);
25646 if (TokenFactorIndex != -1) {
25647 Ops.push_back(NewChain);
25648 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops);
25650 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(),
25651 St->getPointerInfo(),
25652 St->isVolatile(), St->isNonTemporal(),
25653 St->getAlignment());
25656 // Otherwise, lower to two pairs of 32-bit loads / stores.
25657 SDValue LoAddr = Ld->getBasePtr();
25658 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr,
25659 DAG.getConstant(4, MVT::i32));
25661 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
25662 Ld->getPointerInfo(),
25663 Ld->isVolatile(), Ld->isNonTemporal(),
25664 Ld->isInvariant(), Ld->getAlignment());
25665 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr,
25666 Ld->getPointerInfo().getWithOffset(4),
25667 Ld->isVolatile(), Ld->isNonTemporal(),
25669 MinAlign(Ld->getAlignment(), 4));
25671 SDValue NewChain = LoLd.getValue(1);
25672 if (TokenFactorIndex != -1) {
25673 Ops.push_back(LoLd);
25674 Ops.push_back(HiLd);
25675 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops);
25678 LoAddr = St->getBasePtr();
25679 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr,
25680 DAG.getConstant(4, MVT::i32));
25682 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr,
25683 St->getPointerInfo(),
25684 St->isVolatile(), St->isNonTemporal(),
25685 St->getAlignment());
25686 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr,
25687 St->getPointerInfo().getWithOffset(4),
25689 St->isNonTemporal(),
25690 MinAlign(St->getAlignment(), 4));
25691 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt);
25696 /// Return 'true' if this vector operation is "horizontal"
25697 /// and return the operands for the horizontal operation in LHS and RHS. A
25698 /// horizontal operation performs the binary operation on successive elements
25699 /// of its first operand, then on successive elements of its second operand,
25700 /// returning the resulting values in a vector. For example, if
25701 /// A = < float a0, float a1, float a2, float a3 >
25703 /// B = < float b0, float b1, float b2, float b3 >
25704 /// then the result of doing a horizontal operation on A and B is
25705 /// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
25706 /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
25707 /// A horizontal-op B, for some already available A and B, and if so then LHS is
25708 /// set to A, RHS to B, and the routine returns 'true'.
25709 /// Note that the binary operation should have the property that if one of the
25710 /// operands is UNDEF then the result is UNDEF.
25711 static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, bool IsCommutative) {
25712 // Look for the following pattern: if
25713 // A = < float a0, float a1, float a2, float a3 >
25714 // B = < float b0, float b1, float b2, float b3 >
25716 // LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
25717 // RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
25718 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
25719 // which is A horizontal-op B.
25721 // At least one of the operands should be a vector shuffle.
25722 if (LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
25723 RHS.getOpcode() != ISD::VECTOR_SHUFFLE)
25726 MVT VT = LHS.getSimpleValueType();
25728 assert((VT.is128BitVector() || VT.is256BitVector()) &&
25729 "Unsupported vector type for horizontal add/sub");
25731 // Handle 128 and 256-bit vector lengths. AVX defines horizontal add/sub to
25732 // operate independently on 128-bit lanes.
25733 unsigned NumElts = VT.getVectorNumElements();
25734 unsigned NumLanes = VT.getSizeInBits()/128;
25735 unsigned NumLaneElts = NumElts / NumLanes;
25736 assert((NumLaneElts % 2 == 0) &&
25737 "Vector type should have an even number of elements in each lane");
25738 unsigned HalfLaneElts = NumLaneElts/2;
25740 // View LHS in the form
25741 // LHS = VECTOR_SHUFFLE A, B, LMask
25742 // If LHS is not a shuffle then pretend it is the shuffle
25743 // LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
25744 // NOTE: in what follows a default initialized SDValue represents an UNDEF of
25747 SmallVector<int, 16> LMask(NumElts);
25748 if (LHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
25749 if (LHS.getOperand(0).getOpcode() != ISD::UNDEF)
25750 A = LHS.getOperand(0);
25751 if (LHS.getOperand(1).getOpcode() != ISD::UNDEF)
25752 B = LHS.getOperand(1);
25753 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(LHS.getNode())->getMask();
25754 std::copy(Mask.begin(), Mask.end(), LMask.begin());
25756 if (LHS.getOpcode() != ISD::UNDEF)
25758 for (unsigned i = 0; i != NumElts; ++i)
25762 // Likewise, view RHS in the form
25763 // RHS = VECTOR_SHUFFLE C, D, RMask
25765 SmallVector<int, 16> RMask(NumElts);
25766 if (RHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
25767 if (RHS.getOperand(0).getOpcode() != ISD::UNDEF)
25768 C = RHS.getOperand(0);
25769 if (RHS.getOperand(1).getOpcode() != ISD::UNDEF)
25770 D = RHS.getOperand(1);
25771 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(RHS.getNode())->getMask();
25772 std::copy(Mask.begin(), Mask.end(), RMask.begin());
25774 if (RHS.getOpcode() != ISD::UNDEF)
25776 for (unsigned i = 0; i != NumElts; ++i)
25780 // Check that the shuffles are both shuffling the same vectors.
25781 if (!(A == C && B == D) && !(A == D && B == C))
25784 // If everything is UNDEF then bail out: it would be better to fold to UNDEF.
25785 if (!A.getNode() && !B.getNode())
25788 // If A and B occur in reverse order in RHS, then "swap" them (which means
25789 // rewriting the mask).
25791 CommuteVectorShuffleMask(RMask, NumElts);
25793 // At this point LHS and RHS are equivalent to
25794 // LHS = VECTOR_SHUFFLE A, B, LMask
25795 // RHS = VECTOR_SHUFFLE A, B, RMask
25796 // Check that the masks correspond to performing a horizontal operation.
25797 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
25798 for (unsigned i = 0; i != NumLaneElts; ++i) {
25799 int LIdx = LMask[i+l], RIdx = RMask[i+l];
25801 // Ignore any UNDEF components.
25802 if (LIdx < 0 || RIdx < 0 ||
25803 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
25804 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
25807 // Check that successive elements are being operated on. If not, this is
25808 // not a horizontal operation.
25809 unsigned Src = (i/HalfLaneElts); // each lane is split between srcs
25810 int Index = 2*(i%HalfLaneElts) + NumElts*Src + l;
25811 if (!(LIdx == Index && RIdx == Index + 1) &&
25812 !(IsCommutative && LIdx == Index + 1 && RIdx == Index))
25817 LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
25818 RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
25822 /// Do target-specific dag combines on floating point adds.
25823 static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
25824 const X86Subtarget *Subtarget) {
25825 EVT VT = N->getValueType(0);
25826 SDValue LHS = N->getOperand(0);
25827 SDValue RHS = N->getOperand(1);
25829 // Try to synthesize horizontal adds from adds of shuffles.
25830 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
25831 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
25832 isHorizontalBinOp(LHS, RHS, true))
25833 return DAG.getNode(X86ISD::FHADD, SDLoc(N), VT, LHS, RHS);
25837 /// Do target-specific dag combines on floating point subs.
25838 static SDValue PerformFSUBCombine(SDNode *N, SelectionDAG &DAG,
25839 const X86Subtarget *Subtarget) {
25840 EVT VT = N->getValueType(0);
25841 SDValue LHS = N->getOperand(0);
25842 SDValue RHS = N->getOperand(1);
25844 // Try to synthesize horizontal subs from subs of shuffles.
25845 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
25846 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
25847 isHorizontalBinOp(LHS, RHS, false))
25848 return DAG.getNode(X86ISD::FHSUB, SDLoc(N), VT, LHS, RHS);
25852 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
25853 static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) {
25854 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
25856 // F[X]OR(0.0, x) -> x
25857 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
25858 if (C->getValueAPF().isPosZero())
25859 return N->getOperand(1);
25861 // F[X]OR(x, 0.0) -> x
25862 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
25863 if (C->getValueAPF().isPosZero())
25864 return N->getOperand(0);
25868 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
25869 static SDValue PerformFMinFMaxCombine(SDNode *N, SelectionDAG &DAG) {
25870 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
25872 // Only perform optimizations if UnsafeMath is used.
25873 if (!DAG.getTarget().Options.UnsafeFPMath)
25876 // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
25877 // into FMINC and FMAXC, which are Commutative operations.
25878 unsigned NewOp = 0;
25879 switch (N->getOpcode()) {
25880 default: llvm_unreachable("unknown opcode");
25881 case X86ISD::FMIN: NewOp = X86ISD::FMINC; break;
25882 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
25885 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
25886 N->getOperand(0), N->getOperand(1));
25889 /// Do target-specific dag combines on X86ISD::FAND nodes.
25890 static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) {
25891 // FAND(0.0, x) -> 0.0
25892 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
25893 if (C->getValueAPF().isPosZero())
25894 return N->getOperand(0);
25896 // FAND(x, 0.0) -> 0.0
25897 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
25898 if (C->getValueAPF().isPosZero())
25899 return N->getOperand(1);
25904 /// Do target-specific dag combines on X86ISD::FANDN nodes
25905 static SDValue PerformFANDNCombine(SDNode *N, SelectionDAG &DAG) {
25906 // FANDN(0.0, x) -> x
25907 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
25908 if (C->getValueAPF().isPosZero())
25909 return N->getOperand(1);
25911 // FANDN(x, 0.0) -> 0.0
25912 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
25913 if (C->getValueAPF().isPosZero())
25914 return N->getOperand(1);
25919 static SDValue PerformBTCombine(SDNode *N,
25921 TargetLowering::DAGCombinerInfo &DCI) {
25922 // BT ignores high bits in the bit index operand.
25923 SDValue Op1 = N->getOperand(1);
25924 if (Op1.hasOneUse()) {
25925 unsigned BitWidth = Op1.getValueSizeInBits();
25926 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
25927 APInt KnownZero, KnownOne;
25928 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
25929 !DCI.isBeforeLegalizeOps());
25930 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25931 if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) ||
25932 TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO))
25933 DCI.CommitTargetLoweringOpt(TLO);
25938 static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) {
25939 SDValue Op = N->getOperand(0);
25940 if (Op.getOpcode() == ISD::BITCAST)
25941 Op = Op.getOperand(0);
25942 EVT VT = N->getValueType(0), OpVT = Op.getValueType();
25943 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
25944 VT.getVectorElementType().getSizeInBits() ==
25945 OpVT.getVectorElementType().getSizeInBits()) {
25946 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
25951 static SDValue PerformSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG,
25952 const X86Subtarget *Subtarget) {
25953 EVT VT = N->getValueType(0);
25954 if (!VT.isVector())
25957 SDValue N0 = N->getOperand(0);
25958 SDValue N1 = N->getOperand(1);
25959 EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
25962 // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
25963 // both SSE and AVX2 since there is no sign-extended shift right
25964 // operation on a vector with 64-bit elements.
25965 //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
25966 // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
25967 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
25968 N0.getOpcode() == ISD::SIGN_EXTEND)) {
25969 SDValue N00 = N0.getOperand(0);
25971 // EXTLOAD has a better solution on AVX2,
25972 // it may be replaced with X86ISD::VSEXT node.
25973 if (N00.getOpcode() == ISD::LOAD && Subtarget->hasInt256())
25974 if (!ISD::isNormalLoad(N00.getNode()))
25977 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
25978 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32,
25980 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
25986 static SDValue PerformSExtCombine(SDNode *N, SelectionDAG &DAG,
25987 TargetLowering::DAGCombinerInfo &DCI,
25988 const X86Subtarget *Subtarget) {
25989 SDValue N0 = N->getOperand(0);
25990 EVT VT = N->getValueType(0);
25992 // (i8,i32 sext (sdivrem (i8 x, i8 y)) ->
25993 // (i8,i32 (sdivrem_sext_hreg (i8 x, i8 y)
25994 // This exposes the sext to the sdivrem lowering, so that it directly extends
25995 // from AH (which we otherwise need to do contortions to access).
25996 if (N0.getOpcode() == ISD::SDIVREM && N0.getResNo() == 1 &&
25997 N0.getValueType() == MVT::i8 && VT == MVT::i32) {
25999 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT);
26000 SDValue R = DAG.getNode(X86ISD::SDIVREM8_SEXT_HREG, dl, NodeTys,
26001 N0.getOperand(0), N0.getOperand(1));
26002 DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0));
26003 return R.getValue(1);
26006 if (!DCI.isBeforeLegalizeOps())
26009 if (!Subtarget->hasFp256())
26012 if (VT.isVector() && VT.getSizeInBits() == 256) {
26013 SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
26021 static SDValue PerformFMACombine(SDNode *N, SelectionDAG &DAG,
26022 const X86Subtarget* Subtarget) {
26024 EVT VT = N->getValueType(0);
26026 // Let legalize expand this if it isn't a legal type yet.
26027 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
26030 EVT ScalarVT = VT.getScalarType();
26031 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
26032 (!Subtarget->hasFMA() && !Subtarget->hasFMA4()))
26035 SDValue A = N->getOperand(0);
26036 SDValue B = N->getOperand(1);
26037 SDValue C = N->getOperand(2);
26039 bool NegA = (A.getOpcode() == ISD::FNEG);
26040 bool NegB = (B.getOpcode() == ISD::FNEG);
26041 bool NegC = (C.getOpcode() == ISD::FNEG);
26043 // Negative multiplication when NegA xor NegB
26044 bool NegMul = (NegA != NegB);
26046 A = A.getOperand(0);
26048 B = B.getOperand(0);
26050 C = C.getOperand(0);
26054 Opcode = (!NegC) ? X86ISD::FMADD : X86ISD::FMSUB;
26056 Opcode = (!NegC) ? X86ISD::FNMADD : X86ISD::FNMSUB;
26058 return DAG.getNode(Opcode, dl, VT, A, B, C);
26061 static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG,
26062 TargetLowering::DAGCombinerInfo &DCI,
26063 const X86Subtarget *Subtarget) {
26064 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
26065 // (and (i32 x86isd::setcc_carry), 1)
26066 // This eliminates the zext. This transformation is necessary because
26067 // ISD::SETCC is always legalized to i8.
26069 SDValue N0 = N->getOperand(0);
26070 EVT VT = N->getValueType(0);
26072 if (N0.getOpcode() == ISD::AND &&
26074 N0.getOperand(0).hasOneUse()) {
26075 SDValue N00 = N0.getOperand(0);
26076 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
26077 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
26078 if (!C || C->getZExtValue() != 1)
26080 return DAG.getNode(ISD::AND, dl, VT,
26081 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
26082 N00.getOperand(0), N00.getOperand(1)),
26083 DAG.getConstant(1, VT));
26087 if (N0.getOpcode() == ISD::TRUNCATE &&
26089 N0.getOperand(0).hasOneUse()) {
26090 SDValue N00 = N0.getOperand(0);
26091 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
26092 return DAG.getNode(ISD::AND, dl, VT,
26093 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
26094 N00.getOperand(0), N00.getOperand(1)),
26095 DAG.getConstant(1, VT));
26098 if (VT.is256BitVector()) {
26099 SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
26104 // (i8,i32 zext (udivrem (i8 x, i8 y)) ->
26105 // (i8,i32 (udivrem_zext_hreg (i8 x, i8 y)
26106 // This exposes the zext to the udivrem lowering, so that it directly extends
26107 // from AH (which we otherwise need to do contortions to access).
26108 if (N0.getOpcode() == ISD::UDIVREM &&
26109 N0.getResNo() == 1 && N0.getValueType() == MVT::i8 &&
26110 (VT == MVT::i32 || VT == MVT::i64)) {
26111 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT);
26112 SDValue R = DAG.getNode(X86ISD::UDIVREM8_ZEXT_HREG, dl, NodeTys,
26113 N0.getOperand(0), N0.getOperand(1));
26114 DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0));
26115 return R.getValue(1);
26121 // Optimize x == -y --> x+y == 0
26122 // x != -y --> x+y != 0
26123 static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG,
26124 const X86Subtarget* Subtarget) {
26125 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
26126 SDValue LHS = N->getOperand(0);
26127 SDValue RHS = N->getOperand(1);
26128 EVT VT = N->getValueType(0);
26131 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB)
26132 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS.getOperand(0)))
26133 if (C->getAPIntValue() == 0 && LHS.hasOneUse()) {
26134 SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N),
26135 LHS.getValueType(), RHS, LHS.getOperand(1));
26136 return DAG.getSetCC(SDLoc(N), N->getValueType(0),
26137 addV, DAG.getConstant(0, addV.getValueType()), CC);
26139 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && RHS.getOpcode() == ISD::SUB)
26140 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getOperand(0)))
26141 if (C->getAPIntValue() == 0 && RHS.hasOneUse()) {
26142 SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N),
26143 RHS.getValueType(), LHS, RHS.getOperand(1));
26144 return DAG.getSetCC(SDLoc(N), N->getValueType(0),
26145 addV, DAG.getConstant(0, addV.getValueType()), CC);
26148 if (VT.getScalarType() == MVT::i1) {
26149 bool IsSEXT0 = (LHS.getOpcode() == ISD::SIGN_EXTEND) &&
26150 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1);
26151 bool IsVZero0 = ISD::isBuildVectorAllZeros(LHS.getNode());
26152 if (!IsSEXT0 && !IsVZero0)
26154 bool IsSEXT1 = (RHS.getOpcode() == ISD::SIGN_EXTEND) &&
26155 (RHS.getOperand(0).getValueType().getScalarType() == MVT::i1);
26156 bool IsVZero1 = ISD::isBuildVectorAllZeros(RHS.getNode());
26158 if (!IsSEXT1 && !IsVZero1)
26161 if (IsSEXT0 && IsVZero1) {
26162 assert(VT == LHS.getOperand(0).getValueType() && "Uexpected operand type");
26163 if (CC == ISD::SETEQ)
26164 return DAG.getNOT(DL, LHS.getOperand(0), VT);
26165 return LHS.getOperand(0);
26167 if (IsSEXT1 && IsVZero0) {
26168 assert(VT == RHS.getOperand(0).getValueType() && "Uexpected operand type");
26169 if (CC == ISD::SETEQ)
26170 return DAG.getNOT(DL, RHS.getOperand(0), VT);
26171 return RHS.getOperand(0);
26178 static SDValue PerformINSERTPSCombine(SDNode *N, SelectionDAG &DAG,
26179 const X86Subtarget *Subtarget) {
26181 MVT VT = N->getOperand(1)->getSimpleValueType(0);
26182 assert((VT == MVT::v4f32 || VT == MVT::v4i32) &&
26183 "X86insertps is only defined for v4x32");
26185 SDValue Ld = N->getOperand(1);
26186 if (MayFoldLoad(Ld)) {
26187 // Extract the countS bits from the immediate so we can get the proper
26188 // address when narrowing the vector load to a specific element.
26189 // When the second source op is a memory address, interps doesn't use
26190 // countS and just gets an f32 from that address.
26191 unsigned DestIndex =
26192 cast<ConstantSDNode>(N->getOperand(2))->getZExtValue() >> 6;
26193 Ld = NarrowVectorLoadToElement(cast<LoadSDNode>(Ld), DestIndex, DAG);
26197 // Create this as a scalar to vector to match the instruction pattern.
26198 SDValue LoadScalarToVector = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Ld);
26199 // countS bits are ignored when loading from memory on insertps, which
26200 // means we don't need to explicitly set them to 0.
26201 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N->getOperand(0),
26202 LoadScalarToVector, N->getOperand(2));
26205 // Helper function of PerformSETCCCombine. It is to materialize "setb reg"
26206 // as "sbb reg,reg", since it can be extended without zext and produces
26207 // an all-ones bit which is more useful than 0/1 in some cases.
26208 static SDValue MaterializeSETB(SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG,
26211 return DAG.getNode(ISD::AND, DL, VT,
26212 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
26213 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS),
26214 DAG.getConstant(1, VT));
26215 assert (VT == MVT::i1 && "Unexpected type for SECCC node");
26216 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1,
26217 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
26218 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS));
26221 // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
26222 static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG,
26223 TargetLowering::DAGCombinerInfo &DCI,
26224 const X86Subtarget *Subtarget) {
26226 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
26227 SDValue EFLAGS = N->getOperand(1);
26229 if (CC == X86::COND_A) {
26230 // Try to convert COND_A into COND_B in an attempt to facilitate
26231 // materializing "setb reg".
26233 // Do not flip "e > c", where "c" is a constant, because Cmp instruction
26234 // cannot take an immediate as its first operand.
26236 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
26237 EFLAGS.getValueType().isInteger() &&
26238 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
26239 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
26240 EFLAGS.getNode()->getVTList(),
26241 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
26242 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
26243 return MaterializeSETB(DL, NewEFLAGS, DAG, N->getSimpleValueType(0));
26247 // Materialize "setb reg" as "sbb reg,reg", since it can be extended without
26248 // a zext and produces an all-ones bit which is more useful than 0/1 in some
26250 if (CC == X86::COND_B)
26251 return MaterializeSETB(DL, EFLAGS, DAG, N->getSimpleValueType(0));
26255 Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
26256 if (Flags.getNode()) {
26257 SDValue Cond = DAG.getConstant(CC, MVT::i8);
26258 return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
26264 // Optimize branch condition evaluation.
26266 static SDValue PerformBrCondCombine(SDNode *N, SelectionDAG &DAG,
26267 TargetLowering::DAGCombinerInfo &DCI,
26268 const X86Subtarget *Subtarget) {
26270 SDValue Chain = N->getOperand(0);
26271 SDValue Dest = N->getOperand(1);
26272 SDValue EFLAGS = N->getOperand(3);
26273 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
26277 Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
26278 if (Flags.getNode()) {
26279 SDValue Cond = DAG.getConstant(CC, MVT::i8);
26280 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), Chain, Dest, Cond,
26287 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
26288 SelectionDAG &DAG) {
26289 // Take advantage of vector comparisons producing 0 or -1 in each lane to
26290 // optimize away operation when it's from a constant.
26292 // The general transformation is:
26293 // UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
26294 // AND(VECTOR_CMP(x,y), constant2)
26295 // constant2 = UNARYOP(constant)
26297 // Early exit if this isn't a vector operation, the operand of the
26298 // unary operation isn't a bitwise AND, or if the sizes of the operations
26299 // aren't the same.
26300 EVT VT = N->getValueType(0);
26301 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
26302 N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC ||
26303 VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
26306 // Now check that the other operand of the AND is a constant. We could
26307 // make the transformation for non-constant splats as well, but it's unclear
26308 // that would be a benefit as it would not eliminate any operations, just
26309 // perform one more step in scalar code before moving to the vector unit.
26310 if (BuildVectorSDNode *BV =
26311 dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
26312 // Bail out if the vector isn't a constant.
26313 if (!BV->isConstant())
26316 // Everything checks out. Build up the new and improved node.
26318 EVT IntVT = BV->getValueType(0);
26319 // Create a new constant of the appropriate type for the transformed
26321 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
26322 // The AND node needs bitcasts to/from an integer vector type around it.
26323 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
26324 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
26325 N->getOperand(0)->getOperand(0), MaskConst);
26326 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
26333 static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
26334 const X86Subtarget *Subtarget) {
26335 // First try to optimize away the conversion entirely when it's
26336 // conditionally from a constant. Vectors only.
26337 SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG);
26338 if (Res != SDValue())
26341 // Now move on to more general possibilities.
26342 SDValue Op0 = N->getOperand(0);
26343 EVT InVT = Op0->getValueType(0);
26345 // SINT_TO_FP(v4i8) -> SINT_TO_FP(SEXT(v4i8 to v4i32))
26346 if (InVT == MVT::v8i8 || InVT == MVT::v4i8) {
26348 MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32;
26349 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
26350 return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P);
26353 // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
26354 // a 32-bit target where SSE doesn't support i64->FP operations.
26355 if (Op0.getOpcode() == ISD::LOAD) {
26356 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
26357 EVT VT = Ld->getValueType(0);
26358 if (!Ld->isVolatile() && !N->getValueType(0).isVector() &&
26359 ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() &&
26360 !Subtarget->is64Bit() && VT == MVT::i64) {
26361 SDValue FILDChain = Subtarget->getTargetLowering()->BuildFILD(
26362 SDValue(N, 0), Ld->getValueType(0), Ld->getChain(), Op0, DAG);
26363 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1));
26370 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
26371 static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG,
26372 X86TargetLowering::DAGCombinerInfo &DCI) {
26373 // If the LHS and RHS of the ADC node are zero, then it can't overflow and
26374 // the result is either zero or one (depending on the input carry bit).
26375 // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
26376 if (X86::isZeroNode(N->getOperand(0)) &&
26377 X86::isZeroNode(N->getOperand(1)) &&
26378 // We don't have a good way to replace an EFLAGS use, so only do this when
26380 SDValue(N, 1).use_empty()) {
26382 EVT VT = N->getValueType(0);
26383 SDValue CarryOut = DAG.getConstant(0, N->getValueType(1));
26384 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,
26385 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
26386 DAG.getConstant(X86::COND_B,MVT::i8),
26388 DAG.getConstant(1, VT));
26389 return DCI.CombineTo(N, Res1, CarryOut);
26395 // fold (add Y, (sete X, 0)) -> adc 0, Y
26396 // (add Y, (setne X, 0)) -> sbb -1, Y
26397 // (sub (sete X, 0), Y) -> sbb 0, Y
26398 // (sub (setne X, 0), Y) -> adc -1, Y
26399 static SDValue OptimizeConditionalInDecrement(SDNode *N, SelectionDAG &DAG) {
26402 // Look through ZExts.
26403 SDValue Ext = N->getOperand(N->getOpcode() == ISD::SUB ? 1 : 0);
26404 if (Ext.getOpcode() != ISD::ZERO_EXTEND || !Ext.hasOneUse())
26407 SDValue SetCC = Ext.getOperand(0);
26408 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse())
26411 X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
26412 if (CC != X86::COND_E && CC != X86::COND_NE)
26415 SDValue Cmp = SetCC.getOperand(1);
26416 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
26417 !X86::isZeroNode(Cmp.getOperand(1)) ||
26418 !Cmp.getOperand(0).getValueType().isInteger())
26421 SDValue CmpOp0 = Cmp.getOperand(0);
26422 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
26423 DAG.getConstant(1, CmpOp0.getValueType()));
26425 SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1);
26426 if (CC == X86::COND_NE)
26427 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
26428 DL, OtherVal.getValueType(), OtherVal,
26429 DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp);
26430 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
26431 DL, OtherVal.getValueType(), OtherVal,
26432 DAG.getConstant(0, OtherVal.getValueType()), NewCmp);
26435 /// PerformADDCombine - Do target-specific dag combines on integer adds.
26436 static SDValue PerformAddCombine(SDNode *N, SelectionDAG &DAG,
26437 const X86Subtarget *Subtarget) {
26438 EVT VT = N->getValueType(0);
26439 SDValue Op0 = N->getOperand(0);
26440 SDValue Op1 = N->getOperand(1);
26442 // Try to synthesize horizontal adds from adds of shuffles.
26443 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
26444 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
26445 isHorizontalBinOp(Op0, Op1, true))
26446 return DAG.getNode(X86ISD::HADD, SDLoc(N), VT, Op0, Op1);
26448 return OptimizeConditionalInDecrement(N, DAG);
26451 static SDValue PerformSubCombine(SDNode *N, SelectionDAG &DAG,
26452 const X86Subtarget *Subtarget) {
26453 SDValue Op0 = N->getOperand(0);
26454 SDValue Op1 = N->getOperand(1);
26456 // X86 can't encode an immediate LHS of a sub. See if we can push the
26457 // negation into a preceding instruction.
26458 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op0)) {
26459 // If the RHS of the sub is a XOR with one use and a constant, invert the
26460 // immediate. Then add one to the LHS of the sub so we can turn
26461 // X-Y -> X+~Y+1, saving one register.
26462 if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR &&
26463 isa<ConstantSDNode>(Op1.getOperand(1))) {
26464 APInt XorC = cast<ConstantSDNode>(Op1.getOperand(1))->getAPIntValue();
26465 EVT VT = Op0.getValueType();
26466 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
26468 DAG.getConstant(~XorC, VT));
26469 return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor,
26470 DAG.getConstant(C->getAPIntValue()+1, VT));
26474 // Try to synthesize horizontal adds from adds of shuffles.
26475 EVT VT = N->getValueType(0);
26476 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
26477 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
26478 isHorizontalBinOp(Op0, Op1, true))
26479 return DAG.getNode(X86ISD::HSUB, SDLoc(N), VT, Op0, Op1);
26481 return OptimizeConditionalInDecrement(N, DAG);
26484 /// performVZEXTCombine - Performs build vector combines
26485 static SDValue performVZEXTCombine(SDNode *N, SelectionDAG &DAG,
26486 TargetLowering::DAGCombinerInfo &DCI,
26487 const X86Subtarget *Subtarget) {
26489 MVT VT = N->getSimpleValueType(0);
26490 SDValue Op = N->getOperand(0);
26491 MVT OpVT = Op.getSimpleValueType();
26492 MVT OpEltVT = OpVT.getVectorElementType();
26493 unsigned InputBits = OpEltVT.getSizeInBits() * VT.getVectorNumElements();
26495 // (vzext (bitcast (vzext (x)) -> (vzext x)
26497 while (V.getOpcode() == ISD::BITCAST)
26498 V = V.getOperand(0);
26500 if (V != Op && V.getOpcode() == X86ISD::VZEXT) {
26501 MVT InnerVT = V.getSimpleValueType();
26502 MVT InnerEltVT = InnerVT.getVectorElementType();
26504 // If the element sizes match exactly, we can just do one larger vzext. This
26505 // is always an exact type match as vzext operates on integer types.
26506 if (OpEltVT == InnerEltVT) {
26507 assert(OpVT == InnerVT && "Types must match for vzext!");
26508 return DAG.getNode(X86ISD::VZEXT, DL, VT, V.getOperand(0));
26511 // The only other way we can combine them is if only a single element of the
26512 // inner vzext is used in the input to the outer vzext.
26513 if (InnerEltVT.getSizeInBits() < InputBits)
26516 // In this case, the inner vzext is completely dead because we're going to
26517 // only look at bits inside of the low element. Just do the outer vzext on
26518 // a bitcast of the input to the inner.
26519 return DAG.getNode(X86ISD::VZEXT, DL, VT,
26520 DAG.getNode(ISD::BITCAST, DL, OpVT, V));
26523 // Check if we can bypass extracting and re-inserting an element of an input
26524 // vector. Essentialy:
26525 // (bitcast (sclr2vec (ext_vec_elt x))) -> (bitcast x)
26526 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
26527 V.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
26528 V.getOperand(0).getSimpleValueType().getSizeInBits() == InputBits) {
26529 SDValue ExtractedV = V.getOperand(0);
26530 SDValue OrigV = ExtractedV.getOperand(0);
26531 if (auto *ExtractIdx = dyn_cast<ConstantSDNode>(ExtractedV.getOperand(1)))
26532 if (ExtractIdx->getZExtValue() == 0) {
26533 MVT OrigVT = OrigV.getSimpleValueType();
26534 // Extract a subvector if necessary...
26535 if (OrigVT.getSizeInBits() > OpVT.getSizeInBits()) {
26536 int Ratio = OrigVT.getSizeInBits() / OpVT.getSizeInBits();
26537 OrigVT = MVT::getVectorVT(OrigVT.getVectorElementType(),
26538 OrigVT.getVectorNumElements() / Ratio);
26539 OrigV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigVT, OrigV,
26540 DAG.getIntPtrConstant(0));
26542 Op = DAG.getNode(ISD::BITCAST, DL, OpVT, OrigV);
26543 return DAG.getNode(X86ISD::VZEXT, DL, VT, Op);
26550 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
26551 DAGCombinerInfo &DCI) const {
26552 SelectionDAG &DAG = DCI.DAG;
26553 switch (N->getOpcode()) {
26555 case ISD::EXTRACT_VECTOR_ELT:
26556 return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, DCI);
26559 case X86ISD::SHRUNKBLEND:
26560 return PerformSELECTCombine(N, DAG, DCI, Subtarget);
26561 case ISD::BITCAST: return PerformBITCASTCombine(N, DAG);
26562 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI, Subtarget);
26563 case ISD::ADD: return PerformAddCombine(N, DAG, Subtarget);
26564 case ISD::SUB: return PerformSubCombine(N, DAG, Subtarget);
26565 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI);
26566 case ISD::MUL: return PerformMulCombine(N, DAG, DCI);
26569 case ISD::SRL: return PerformShiftCombine(N, DAG, DCI, Subtarget);
26570 case ISD::AND: return PerformAndCombine(N, DAG, DCI, Subtarget);
26571 case ISD::OR: return PerformOrCombine(N, DAG, DCI, Subtarget);
26572 case ISD::XOR: return PerformXorCombine(N, DAG, DCI, Subtarget);
26573 case ISD::LOAD: return PerformLOADCombine(N, DAG, DCI, Subtarget);
26574 case ISD::MLOAD: return PerformMLOADCombine(N, DAG, DCI, Subtarget);
26575 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget);
26576 case ISD::MSTORE: return PerformMSTORECombine(N, DAG, Subtarget);
26577 case ISD::SINT_TO_FP: return PerformSINT_TO_FPCombine(N, DAG, Subtarget);
26578 case ISD::FADD: return PerformFADDCombine(N, DAG, Subtarget);
26579 case ISD::FSUB: return PerformFSUBCombine(N, DAG, Subtarget);
26581 case X86ISD::FOR: return PerformFORCombine(N, DAG);
26583 case X86ISD::FMAX: return PerformFMinFMaxCombine(N, DAG);
26584 case X86ISD::FAND: return PerformFANDCombine(N, DAG);
26585 case X86ISD::FANDN: return PerformFANDNCombine(N, DAG);
26586 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI);
26587 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG);
26588 case ISD::ANY_EXTEND:
26589 case ISD::ZERO_EXTEND: return PerformZExtCombine(N, DAG, DCI, Subtarget);
26590 case ISD::SIGN_EXTEND: return PerformSExtCombine(N, DAG, DCI, Subtarget);
26591 case ISD::SIGN_EXTEND_INREG:
26592 return PerformSIGN_EXTEND_INREGCombine(N, DAG, Subtarget);
26593 case ISD::TRUNCATE: return PerformTruncateCombine(N, DAG,DCI,Subtarget);
26594 case ISD::SETCC: return PerformISDSETCCCombine(N, DAG, Subtarget);
26595 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG, DCI, Subtarget);
26596 case X86ISD::BRCOND: return PerformBrCondCombine(N, DAG, DCI, Subtarget);
26597 case X86ISD::VZEXT: return performVZEXTCombine(N, DAG, DCI, Subtarget);
26598 case X86ISD::SHUFP: // Handle all target specific shuffles
26599 case X86ISD::PALIGNR:
26600 case X86ISD::UNPCKH:
26601 case X86ISD::UNPCKL:
26602 case X86ISD::MOVHLPS:
26603 case X86ISD::MOVLHPS:
26604 case X86ISD::PSHUFB:
26605 case X86ISD::PSHUFD:
26606 case X86ISD::PSHUFHW:
26607 case X86ISD::PSHUFLW:
26608 case X86ISD::MOVSS:
26609 case X86ISD::MOVSD:
26610 case X86ISD::VPERMILPI:
26611 case X86ISD::VPERM2X128:
26612 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI,Subtarget);
26613 case ISD::FMA: return PerformFMACombine(N, DAG, Subtarget);
26614 case ISD::INTRINSIC_WO_CHAIN:
26615 return PerformINTRINSIC_WO_CHAINCombine(N, DAG, Subtarget);
26616 case X86ISD::INSERTPS: {
26617 if (getTargetMachine().getOptLevel() > CodeGenOpt::None)
26618 return PerformINSERTPSCombine(N, DAG, Subtarget);
26621 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DAG, Subtarget);
26627 /// isTypeDesirableForOp - Return true if the target has native support for
26628 /// the specified value type and it is 'desirable' to use the type for the
26629 /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
26630 /// instruction encodings are longer and some i16 instructions are slow.
26631 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
26632 if (!isTypeLegal(VT))
26634 if (VT != MVT::i16)
26641 case ISD::SIGN_EXTEND:
26642 case ISD::ZERO_EXTEND:
26643 case ISD::ANY_EXTEND:
26656 /// IsDesirableToPromoteOp - This method query the target whether it is
26657 /// beneficial for dag combiner to promote the specified node. If true, it
26658 /// should return the desired promotion type by reference.
26659 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
26660 EVT VT = Op.getValueType();
26661 if (VT != MVT::i16)
26664 bool Promote = false;
26665 bool Commute = false;
26666 switch (Op.getOpcode()) {
26669 LoadSDNode *LD = cast<LoadSDNode>(Op);
26670 // If the non-extending load has a single use and it's not live out, then it
26671 // might be folded.
26672 if (LD->getExtensionType() == ISD::NON_EXTLOAD /*&&
26673 Op.hasOneUse()*/) {
26674 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
26675 UE = Op.getNode()->use_end(); UI != UE; ++UI) {
26676 // The only case where we'd want to promote LOAD (rather then it being
26677 // promoted as an operand is when it's only use is liveout.
26678 if (UI->getOpcode() != ISD::CopyToReg)
26685 case ISD::SIGN_EXTEND:
26686 case ISD::ZERO_EXTEND:
26687 case ISD::ANY_EXTEND:
26692 SDValue N0 = Op.getOperand(0);
26693 // Look out for (store (shl (load), x)).
26694 if (MayFoldLoad(N0) && MayFoldIntoStore(Op))
26707 SDValue N0 = Op.getOperand(0);
26708 SDValue N1 = Op.getOperand(1);
26709 if (!Commute && MayFoldLoad(N1))
26711 // Avoid disabling potential load folding opportunities.
26712 if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op)))
26714 if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op)))
26724 //===----------------------------------------------------------------------===//
26725 // X86 Inline Assembly Support
26726 //===----------------------------------------------------------------------===//
26729 // Helper to match a string separated by whitespace.
26730 bool matchAsmImpl(StringRef s, ArrayRef<const StringRef *> args) {
26731 s = s.substr(s.find_first_not_of(" \t")); // Skip leading whitespace.
26733 for (unsigned i = 0, e = args.size(); i != e; ++i) {
26734 StringRef piece(*args[i]);
26735 if (!s.startswith(piece)) // Check if the piece matches.
26738 s = s.substr(piece.size());
26739 StringRef::size_type pos = s.find_first_not_of(" \t");
26740 if (pos == 0) // We matched a prefix.
26748 const VariadicFunction1<bool, StringRef, StringRef, matchAsmImpl> matchAsm={};
26751 static bool clobbersFlagRegisters(const SmallVector<StringRef, 4> &AsmPieces) {
26753 if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
26754 if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{cc}") &&
26755 std::count(AsmPieces.begin(), AsmPieces.end(), "~{flags}") &&
26756 std::count(AsmPieces.begin(), AsmPieces.end(), "~{fpsr}")) {
26758 if (AsmPieces.size() == 3)
26760 else if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{dirflag}"))
26767 bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
26768 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
26770 std::string AsmStr = IA->getAsmString();
26772 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
26773 if (!Ty || Ty->getBitWidth() % 16 != 0)
26776 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
26777 SmallVector<StringRef, 4> AsmPieces;
26778 SplitString(AsmStr, AsmPieces, ";\n");
26780 switch (AsmPieces.size()) {
26781 default: return false;
26783 // FIXME: this should verify that we are targeting a 486 or better. If not,
26784 // we will turn this bswap into something that will be lowered to logical
26785 // ops instead of emitting the bswap asm. For now, we don't support 486 or
26786 // lower so don't worry about this.
26788 if (matchAsm(AsmPieces[0], "bswap", "$0") ||
26789 matchAsm(AsmPieces[0], "bswapl", "$0") ||
26790 matchAsm(AsmPieces[0], "bswapq", "$0") ||
26791 matchAsm(AsmPieces[0], "bswap", "${0:q}") ||
26792 matchAsm(AsmPieces[0], "bswapl", "${0:q}") ||
26793 matchAsm(AsmPieces[0], "bswapq", "${0:q}")) {
26794 // No need to check constraints, nothing other than the equivalent of
26795 // "=r,0" would be valid here.
26796 return IntrinsicLowering::LowerToByteSwap(CI);
26799 // rorw $$8, ${0:w} --> llvm.bswap.i16
26800 if (CI->getType()->isIntegerTy(16) &&
26801 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
26802 (matchAsm(AsmPieces[0], "rorw", "$$8,", "${0:w}") ||
26803 matchAsm(AsmPieces[0], "rolw", "$$8,", "${0:w}"))) {
26805 const std::string &ConstraintsStr = IA->getConstraintString();
26806 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
26807 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
26808 if (clobbersFlagRegisters(AsmPieces))
26809 return IntrinsicLowering::LowerToByteSwap(CI);
26813 if (CI->getType()->isIntegerTy(32) &&
26814 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
26815 matchAsm(AsmPieces[0], "rorw", "$$8,", "${0:w}") &&
26816 matchAsm(AsmPieces[1], "rorl", "$$16,", "$0") &&
26817 matchAsm(AsmPieces[2], "rorw", "$$8,", "${0:w}")) {
26819 const std::string &ConstraintsStr = IA->getConstraintString();
26820 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
26821 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
26822 if (clobbersFlagRegisters(AsmPieces))
26823 return IntrinsicLowering::LowerToByteSwap(CI);
26826 if (CI->getType()->isIntegerTy(64)) {
26827 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
26828 if (Constraints.size() >= 2 &&
26829 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
26830 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
26831 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64
26832 if (matchAsm(AsmPieces[0], "bswap", "%eax") &&
26833 matchAsm(AsmPieces[1], "bswap", "%edx") &&
26834 matchAsm(AsmPieces[2], "xchgl", "%eax,", "%edx"))
26835 return IntrinsicLowering::LowerToByteSwap(CI);
26843 /// getConstraintType - Given a constraint letter, return the type of
26844 /// constraint it is for this target.
26845 X86TargetLowering::ConstraintType
26846 X86TargetLowering::getConstraintType(const std::string &Constraint) const {
26847 if (Constraint.size() == 1) {
26848 switch (Constraint[0]) {
26859 return C_RegisterClass;
26883 return TargetLowering::getConstraintType(Constraint);
26886 /// Examine constraint type and operand type and determine a weight value.
26887 /// This object must already have been set up with the operand type
26888 /// and the current alternative constraint selected.
26889 TargetLowering::ConstraintWeight
26890 X86TargetLowering::getSingleConstraintMatchWeight(
26891 AsmOperandInfo &info, const char *constraint) const {
26892 ConstraintWeight weight = CW_Invalid;
26893 Value *CallOperandVal = info.CallOperandVal;
26894 // If we don't have a value, we can't do a match,
26895 // but allow it at the lowest weight.
26896 if (!CallOperandVal)
26898 Type *type = CallOperandVal->getType();
26899 // Look at the constraint type.
26900 switch (*constraint) {
26902 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
26913 if (CallOperandVal->getType()->isIntegerTy())
26914 weight = CW_SpecificReg;
26919 if (type->isFloatingPointTy())
26920 weight = CW_SpecificReg;
26923 if (type->isX86_MMXTy() && Subtarget->hasMMX())
26924 weight = CW_SpecificReg;
26928 if (((type->getPrimitiveSizeInBits() == 128) && Subtarget->hasSSE1()) ||
26929 ((type->getPrimitiveSizeInBits() == 256) && Subtarget->hasFp256()))
26930 weight = CW_Register;
26933 if (ConstantInt *C = dyn_cast<ConstantInt>(info.CallOperandVal)) {
26934 if (C->getZExtValue() <= 31)
26935 weight = CW_Constant;
26939 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26940 if (C->getZExtValue() <= 63)
26941 weight = CW_Constant;
26945 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26946 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f))
26947 weight = CW_Constant;
26951 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26952 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff))
26953 weight = CW_Constant;
26957 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26958 if (C->getZExtValue() <= 3)
26959 weight = CW_Constant;
26963 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26964 if (C->getZExtValue() <= 0xff)
26965 weight = CW_Constant;
26970 if (dyn_cast<ConstantFP>(CallOperandVal)) {
26971 weight = CW_Constant;
26975 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26976 if ((C->getSExtValue() >= -0x80000000LL) &&
26977 (C->getSExtValue() <= 0x7fffffffLL))
26978 weight = CW_Constant;
26982 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26983 if (C->getZExtValue() <= 0xffffffff)
26984 weight = CW_Constant;
26991 /// LowerXConstraint - try to replace an X constraint, which matches anything,
26992 /// with another that has more specific requirements based on the type of the
26993 /// corresponding operand.
26994 const char *X86TargetLowering::
26995 LowerXConstraint(EVT ConstraintVT) const {
26996 // FP X constraints get lowered to SSE1/2 registers if available, otherwise
26997 // 'f' like normal targets.
26998 if (ConstraintVT.isFloatingPoint()) {
26999 if (Subtarget->hasSSE2())
27001 if (Subtarget->hasSSE1())
27005 return TargetLowering::LowerXConstraint(ConstraintVT);
27008 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
27009 /// vector. If it is invalid, don't add anything to Ops.
27010 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
27011 std::string &Constraint,
27012 std::vector<SDValue>&Ops,
27013 SelectionDAG &DAG) const {
27016 // Only support length 1 constraints for now.
27017 if (Constraint.length() > 1) return;
27019 char ConstraintLetter = Constraint[0];
27020 switch (ConstraintLetter) {
27023 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27024 if (C->getZExtValue() <= 31) {
27025 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27031 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27032 if (C->getZExtValue() <= 63) {
27033 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27039 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27040 if (isInt<8>(C->getSExtValue())) {
27041 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27047 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27048 if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
27049 (Subtarget->is64Bit() && C->getZExtValue() == 0xffffffff)) {
27050 Result = DAG.getTargetConstant(C->getSExtValue(), Op.getValueType());
27056 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27057 if (C->getZExtValue() <= 3) {
27058 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27064 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27065 if (C->getZExtValue() <= 255) {
27066 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27072 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27073 if (C->getZExtValue() <= 127) {
27074 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27080 // 32-bit signed value
27081 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27082 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
27083 C->getSExtValue())) {
27084 // Widen to 64 bits here to get it sign extended.
27085 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64);
27088 // FIXME gcc accepts some relocatable values here too, but only in certain
27089 // memory models; it's complicated.
27094 // 32-bit unsigned value
27095 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27096 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
27097 C->getZExtValue())) {
27098 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27102 // FIXME gcc accepts some relocatable values here too, but only in certain
27103 // memory models; it's complicated.
27107 // Literal immediates are always ok.
27108 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
27109 // Widen to 64 bits here to get it sign extended.
27110 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64);
27114 // In any sort of PIC mode addresses need to be computed at runtime by
27115 // adding in a register or some sort of table lookup. These can't
27116 // be used as immediates.
27117 if (Subtarget->isPICStyleGOT() || Subtarget->isPICStyleStubPIC())
27120 // If we are in non-pic codegen mode, we allow the address of a global (with
27121 // an optional displacement) to be used with 'i'.
27122 GlobalAddressSDNode *GA = nullptr;
27123 int64_t Offset = 0;
27125 // Match either (GA), (GA+C), (GA+C1+C2), etc.
27127 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) {
27128 Offset += GA->getOffset();
27130 } else if (Op.getOpcode() == ISD::ADD) {
27131 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
27132 Offset += C->getZExtValue();
27133 Op = Op.getOperand(0);
27136 } else if (Op.getOpcode() == ISD::SUB) {
27137 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
27138 Offset += -C->getZExtValue();
27139 Op = Op.getOperand(0);
27144 // Otherwise, this isn't something we can handle, reject it.
27148 const GlobalValue *GV = GA->getGlobal();
27149 // If we require an extra load to get this address, as in PIC mode, we
27150 // can't accept it.
27151 if (isGlobalStubReference(
27152 Subtarget->ClassifyGlobalReference(GV, DAG.getTarget())))
27155 Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op),
27156 GA->getValueType(0), Offset);
27161 if (Result.getNode()) {
27162 Ops.push_back(Result);
27165 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
27168 std::pair<unsigned, const TargetRegisterClass*>
27169 X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
27171 // First, see if this is a constraint that directly corresponds to an LLVM
27173 if (Constraint.size() == 1) {
27174 // GCC Constraint Letters
27175 switch (Constraint[0]) {
27177 // TODO: Slight differences here in allocation order and leaving
27178 // RIP in the class. Do they matter any more here than they do
27179 // in the normal allocation?
27180 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
27181 if (Subtarget->is64Bit()) {
27182 if (VT == MVT::i32 || VT == MVT::f32)
27183 return std::make_pair(0U, &X86::GR32RegClass);
27184 if (VT == MVT::i16)
27185 return std::make_pair(0U, &X86::GR16RegClass);
27186 if (VT == MVT::i8 || VT == MVT::i1)
27187 return std::make_pair(0U, &X86::GR8RegClass);
27188 if (VT == MVT::i64 || VT == MVT::f64)
27189 return std::make_pair(0U, &X86::GR64RegClass);
27192 // 32-bit fallthrough
27193 case 'Q': // Q_REGS
27194 if (VT == MVT::i32 || VT == MVT::f32)
27195 return std::make_pair(0U, &X86::GR32_ABCDRegClass);
27196 if (VT == MVT::i16)
27197 return std::make_pair(0U, &X86::GR16_ABCDRegClass);
27198 if (VT == MVT::i8 || VT == MVT::i1)
27199 return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
27200 if (VT == MVT::i64)
27201 return std::make_pair(0U, &X86::GR64_ABCDRegClass);
27203 case 'r': // GENERAL_REGS
27204 case 'l': // INDEX_REGS
27205 if (VT == MVT::i8 || VT == MVT::i1)
27206 return std::make_pair(0U, &X86::GR8RegClass);
27207 if (VT == MVT::i16)
27208 return std::make_pair(0U, &X86::GR16RegClass);
27209 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit())
27210 return std::make_pair(0U, &X86::GR32RegClass);
27211 return std::make_pair(0U, &X86::GR64RegClass);
27212 case 'R': // LEGACY_REGS
27213 if (VT == MVT::i8 || VT == MVT::i1)
27214 return std::make_pair(0U, &X86::GR8_NOREXRegClass);
27215 if (VT == MVT::i16)
27216 return std::make_pair(0U, &X86::GR16_NOREXRegClass);
27217 if (VT == MVT::i32 || !Subtarget->is64Bit())
27218 return std::make_pair(0U, &X86::GR32_NOREXRegClass);
27219 return std::make_pair(0U, &X86::GR64_NOREXRegClass);
27220 case 'f': // FP Stack registers.
27221 // If SSE is enabled for this VT, use f80 to ensure the isel moves the
27222 // value to the correct fpstack register class.
27223 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
27224 return std::make_pair(0U, &X86::RFP32RegClass);
27225 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
27226 return std::make_pair(0U, &X86::RFP64RegClass);
27227 return std::make_pair(0U, &X86::RFP80RegClass);
27228 case 'y': // MMX_REGS if MMX allowed.
27229 if (!Subtarget->hasMMX()) break;
27230 return std::make_pair(0U, &X86::VR64RegClass);
27231 case 'Y': // SSE_REGS if SSE2 allowed
27232 if (!Subtarget->hasSSE2()) break;
27234 case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
27235 if (!Subtarget->hasSSE1()) break;
27237 switch (VT.SimpleTy) {
27239 // Scalar SSE types.
27242 return std::make_pair(0U, &X86::FR32RegClass);
27245 return std::make_pair(0U, &X86::FR64RegClass);
27253 return std::make_pair(0U, &X86::VR128RegClass);
27261 return std::make_pair(0U, &X86::VR256RegClass);
27266 return std::make_pair(0U, &X86::VR512RegClass);
27272 // Use the default implementation in TargetLowering to convert the register
27273 // constraint into a member of a register class.
27274 std::pair<unsigned, const TargetRegisterClass*> Res;
27275 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
27277 // Not found as a standard register?
27279 // Map st(0) -> st(7) -> ST0
27280 if (Constraint.size() == 7 && Constraint[0] == '{' &&
27281 tolower(Constraint[1]) == 's' &&
27282 tolower(Constraint[2]) == 't' &&
27283 Constraint[3] == '(' &&
27284 (Constraint[4] >= '0' && Constraint[4] <= '7') &&
27285 Constraint[5] == ')' &&
27286 Constraint[6] == '}') {
27288 Res.first = X86::FP0+Constraint[4]-'0';
27289 Res.second = &X86::RFP80RegClass;
27293 // GCC allows "st(0)" to be called just plain "st".
27294 if (StringRef("{st}").equals_lower(Constraint)) {
27295 Res.first = X86::FP0;
27296 Res.second = &X86::RFP80RegClass;
27301 if (StringRef("{flags}").equals_lower(Constraint)) {
27302 Res.first = X86::EFLAGS;
27303 Res.second = &X86::CCRRegClass;
27307 // 'A' means EAX + EDX.
27308 if (Constraint == "A") {
27309 Res.first = X86::EAX;
27310 Res.second = &X86::GR32_ADRegClass;
27316 // Otherwise, check to see if this is a register class of the wrong value
27317 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
27318 // turn into {ax},{dx}.
27319 if (Res.second->hasType(VT))
27320 return Res; // Correct type already, nothing to do.
27322 // All of the single-register GCC register classes map their values onto
27323 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we
27324 // really want an 8-bit or 32-bit register, map to the appropriate register
27325 // class and return the appropriate register.
27326 if (Res.second == &X86::GR16RegClass) {
27327 if (VT == MVT::i8 || VT == MVT::i1) {
27328 unsigned DestReg = 0;
27329 switch (Res.first) {
27331 case X86::AX: DestReg = X86::AL; break;
27332 case X86::DX: DestReg = X86::DL; break;
27333 case X86::CX: DestReg = X86::CL; break;
27334 case X86::BX: DestReg = X86::BL; break;
27337 Res.first = DestReg;
27338 Res.second = &X86::GR8RegClass;
27340 } else if (VT == MVT::i32 || VT == MVT::f32) {
27341 unsigned DestReg = 0;
27342 switch (Res.first) {
27344 case X86::AX: DestReg = X86::EAX; break;
27345 case X86::DX: DestReg = X86::EDX; break;
27346 case X86::CX: DestReg = X86::ECX; break;
27347 case X86::BX: DestReg = X86::EBX; break;
27348 case X86::SI: DestReg = X86::ESI; break;
27349 case X86::DI: DestReg = X86::EDI; break;
27350 case X86::BP: DestReg = X86::EBP; break;
27351 case X86::SP: DestReg = X86::ESP; break;
27354 Res.first = DestReg;
27355 Res.second = &X86::GR32RegClass;
27357 } else if (VT == MVT::i64 || VT == MVT::f64) {
27358 unsigned DestReg = 0;
27359 switch (Res.first) {
27361 case X86::AX: DestReg = X86::RAX; break;
27362 case X86::DX: DestReg = X86::RDX; break;
27363 case X86::CX: DestReg = X86::RCX; break;
27364 case X86::BX: DestReg = X86::RBX; break;
27365 case X86::SI: DestReg = X86::RSI; break;
27366 case X86::DI: DestReg = X86::RDI; break;
27367 case X86::BP: DestReg = X86::RBP; break;
27368 case X86::SP: DestReg = X86::RSP; break;
27371 Res.first = DestReg;
27372 Res.second = &X86::GR64RegClass;
27375 } else if (Res.second == &X86::FR32RegClass ||
27376 Res.second == &X86::FR64RegClass ||
27377 Res.second == &X86::VR128RegClass ||
27378 Res.second == &X86::VR256RegClass ||
27379 Res.second == &X86::FR32XRegClass ||
27380 Res.second == &X86::FR64XRegClass ||
27381 Res.second == &X86::VR128XRegClass ||
27382 Res.second == &X86::VR256XRegClass ||
27383 Res.second == &X86::VR512RegClass) {
27384 // Handle references to XMM physical registers that got mapped into the
27385 // wrong class. This can happen with constraints like {xmm0} where the
27386 // target independent register mapper will just pick the first match it can
27387 // find, ignoring the required type.
27389 if (VT == MVT::f32 || VT == MVT::i32)
27390 Res.second = &X86::FR32RegClass;
27391 else if (VT == MVT::f64 || VT == MVT::i64)
27392 Res.second = &X86::FR64RegClass;
27393 else if (X86::VR128RegClass.hasType(VT))
27394 Res.second = &X86::VR128RegClass;
27395 else if (X86::VR256RegClass.hasType(VT))
27396 Res.second = &X86::VR256RegClass;
27397 else if (X86::VR512RegClass.hasType(VT))
27398 Res.second = &X86::VR512RegClass;
27404 int X86TargetLowering::getScalingFactorCost(const AddrMode &AM,
27406 // Scaling factors are not free at all.
27407 // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
27408 // will take 2 allocations in the out of order engine instead of 1
27409 // for plain addressing mode, i.e. inst (reg1).
27411 // vaddps (%rsi,%drx), %ymm0, %ymm1
27412 // Requires two allocations (one for the load, one for the computation)
27414 // vaddps (%rsi), %ymm0, %ymm1
27415 // Requires just 1 allocation, i.e., freeing allocations for other operations
27416 // and having less micro operations to execute.
27418 // For some X86 architectures, this is even worse because for instance for
27419 // stores, the complex addressing mode forces the instruction to use the
27420 // "load" ports instead of the dedicated "store" port.
27421 // E.g., on Haswell:
27422 // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
27423 // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
27424 if (isLegalAddressingMode(AM, Ty))
27425 // Scale represents reg2 * scale, thus account for 1
27426 // as soon as we use a second register.
27427 return AM.Scale != 0;
27431 bool X86TargetLowering::isTargetFTOL() const {
27432 return Subtarget->isTargetKnownWindowsMSVC() && !Subtarget->is64Bit();