1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the interfaces that X86 uses to lower LLVM code into a
13 //===----------------------------------------------------------------------===//
15 #include "X86ISelLowering.h"
16 #include "Utils/X86ShuffleDecode.h"
17 #include "X86CallingConv.h"
18 #include "X86FrameLowering.h"
19 #include "X86InstrBuilder.h"
20 #include "X86MachineFunctionInfo.h"
21 #include "X86TargetMachine.h"
22 #include "X86TargetObjectFile.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/ADT/StringExtras.h"
27 #include "llvm/ADT/StringSwitch.h"
28 #include "llvm/ADT/VariadicFunction.h"
29 #include "llvm/CodeGen/IntrinsicLowering.h"
30 #include "llvm/CodeGen/MachineFrameInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineJumpTableInfo.h"
34 #include "llvm/CodeGen/MachineModuleInfo.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/IR/CallSite.h"
37 #include "llvm/IR/CallingConv.h"
38 #include "llvm/IR/Constants.h"
39 #include "llvm/IR/DerivedTypes.h"
40 #include "llvm/IR/Function.h"
41 #include "llvm/IR/GlobalAlias.h"
42 #include "llvm/IR/GlobalVariable.h"
43 #include "llvm/IR/Instructions.h"
44 #include "llvm/IR/Intrinsics.h"
45 #include "llvm/MC/MCAsmInfo.h"
46 #include "llvm/MC/MCContext.h"
47 #include "llvm/MC/MCExpr.h"
48 #include "llvm/MC/MCSymbol.h"
49 #include "llvm/Support/CommandLine.h"
50 #include "llvm/Support/Debug.h"
51 #include "llvm/Support/ErrorHandling.h"
52 #include "llvm/Support/MathExtras.h"
53 #include "llvm/Target/TargetOptions.h"
54 #include "X86IntrinsicsInfo.h"
60 #define DEBUG_TYPE "x86-isel"
62 STATISTIC(NumTailCalls, "Number of tail calls");
64 static cl::opt<bool> ExperimentalVectorWideningLegalization(
65 "x86-experimental-vector-widening-legalization", cl::init(false),
66 cl::desc("Enable an experimental vector type legalization through widening "
67 "rather than promotion."),
70 static cl::opt<bool> ExperimentalVectorShuffleLowering(
71 "x86-experimental-vector-shuffle-lowering", cl::init(true),
72 cl::desc("Enable an experimental vector shuffle lowering code path."),
75 static cl::opt<bool> ExperimentalVectorShuffleLegality(
76 "x86-experimental-vector-shuffle-legality", cl::init(false),
77 cl::desc("Enable experimental shuffle legality based on the experimental "
78 "shuffle lowering. Should only be used with the experimental "
82 static cl::opt<int> ReciprocalEstimateRefinementSteps(
83 "x86-recip-refinement-steps", cl::init(1),
84 cl::desc("Specify the number of Newton-Raphson iterations applied to the "
85 "result of the hardware reciprocal estimate instruction."),
88 // Forward declarations.
89 static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
92 static SDValue ExtractSubVector(SDValue Vec, unsigned IdxVal,
93 SelectionDAG &DAG, SDLoc dl,
94 unsigned vectorWidth) {
95 assert((vectorWidth == 128 || vectorWidth == 256) &&
96 "Unsupported vector width");
97 EVT VT = Vec.getValueType();
98 EVT ElVT = VT.getVectorElementType();
99 unsigned Factor = VT.getSizeInBits()/vectorWidth;
100 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
101 VT.getVectorNumElements()/Factor);
103 // Extract from UNDEF is UNDEF.
104 if (Vec.getOpcode() == ISD::UNDEF)
105 return DAG.getUNDEF(ResultVT);
107 // Extract the relevant vectorWidth bits. Generate an EXTRACT_SUBVECTOR
108 unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
110 // This is the index of the first element of the vectorWidth-bit chunk
112 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / vectorWidth)
115 // If the input is a buildvector just emit a smaller one.
116 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
117 return DAG.getNode(ISD::BUILD_VECTOR, dl, ResultVT,
118 makeArrayRef(Vec->op_begin() + NormalizedIdxVal,
121 SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
122 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
125 /// Generate a DAG to grab 128-bits from a vector > 128 bits. This
126 /// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
127 /// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
128 /// instructions or a simple subregister reference. Idx is an index in the
129 /// 128 bits we want. It need not be aligned to a 128-bit boundary. That makes
130 /// lowering EXTRACT_VECTOR_ELT operations easier.
131 static SDValue Extract128BitVector(SDValue Vec, unsigned IdxVal,
132 SelectionDAG &DAG, SDLoc dl) {
133 assert((Vec.getValueType().is256BitVector() ||
134 Vec.getValueType().is512BitVector()) && "Unexpected vector size!");
135 return ExtractSubVector(Vec, IdxVal, DAG, dl, 128);
138 /// Generate a DAG to grab 256-bits from a 512-bit vector.
139 static SDValue Extract256BitVector(SDValue Vec, unsigned IdxVal,
140 SelectionDAG &DAG, SDLoc dl) {
141 assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!");
142 return ExtractSubVector(Vec, IdxVal, DAG, dl, 256);
145 static SDValue InsertSubVector(SDValue Result, SDValue Vec,
146 unsigned IdxVal, SelectionDAG &DAG,
147 SDLoc dl, unsigned vectorWidth) {
148 assert((vectorWidth == 128 || vectorWidth == 256) &&
149 "Unsupported vector width");
150 // Inserting UNDEF is Result
151 if (Vec.getOpcode() == ISD::UNDEF)
153 EVT VT = Vec.getValueType();
154 EVT ElVT = VT.getVectorElementType();
155 EVT ResultVT = Result.getValueType();
157 // Insert the relevant vectorWidth bits.
158 unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
160 // This is the index of the first element of the vectorWidth-bit chunk
162 unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/vectorWidth)
165 SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
166 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
169 /// Generate a DAG to put 128-bits into a vector > 128 bits. This
170 /// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
171 /// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
172 /// simple superregister reference. Idx is an index in the 128 bits
173 /// we want. It need not be aligned to a 128-bit boundary. That makes
174 /// lowering INSERT_VECTOR_ELT operations easier.
175 static SDValue Insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
176 SelectionDAG &DAG,SDLoc dl) {
177 assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!");
178 return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
181 static SDValue Insert256BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
182 SelectionDAG &DAG, SDLoc dl) {
183 assert(Vec.getValueType().is256BitVector() && "Unexpected vector size!");
184 return InsertSubVector(Result, Vec, IdxVal, DAG, dl, 256);
187 /// Concat two 128-bit vectors into a 256 bit vector using VINSERTF128
188 /// instructions. This is used because creating CONCAT_VECTOR nodes of
189 /// BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower
190 /// large BUILD_VECTORS.
191 static SDValue Concat128BitVectors(SDValue V1, SDValue V2, EVT VT,
192 unsigned NumElems, SelectionDAG &DAG,
194 SDValue V = Insert128BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
195 return Insert128BitVector(V, V2, NumElems/2, DAG, dl);
198 static SDValue Concat256BitVectors(SDValue V1, SDValue V2, EVT VT,
199 unsigned NumElems, SelectionDAG &DAG,
201 SDValue V = Insert256BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
202 return Insert256BitVector(V, V2, NumElems/2, DAG, dl);
205 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
206 const X86Subtarget &STI)
207 : TargetLowering(TM), Subtarget(&STI) {
208 X86ScalarSSEf64 = Subtarget->hasSSE2();
209 X86ScalarSSEf32 = Subtarget->hasSSE1();
210 TD = getDataLayout();
212 // Set up the TargetLowering object.
213 static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 };
215 // X86 is weird. It always uses i8 for shift amounts and setcc results.
216 setBooleanContents(ZeroOrOneBooleanContent);
217 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
218 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
220 // For 64-bit, since we have so many registers, use the ILP scheduler.
221 // For 32-bit, use the register pressure specific scheduling.
222 // For Atom, always use ILP scheduling.
223 if (Subtarget->isAtom())
224 setSchedulingPreference(Sched::ILP);
225 else if (Subtarget->is64Bit())
226 setSchedulingPreference(Sched::ILP);
228 setSchedulingPreference(Sched::RegPressure);
229 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
230 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
232 // Bypass expensive divides on Atom when compiling with O2.
233 if (TM.getOptLevel() >= CodeGenOpt::Default) {
234 if (Subtarget->hasSlowDivide32())
235 addBypassSlowDiv(32, 8);
236 if (Subtarget->hasSlowDivide64() && Subtarget->is64Bit())
237 addBypassSlowDiv(64, 16);
240 if (Subtarget->isTargetKnownWindowsMSVC()) {
241 // Setup Windows compiler runtime calls.
242 setLibcallName(RTLIB::SDIV_I64, "_alldiv");
243 setLibcallName(RTLIB::UDIV_I64, "_aulldiv");
244 setLibcallName(RTLIB::SREM_I64, "_allrem");
245 setLibcallName(RTLIB::UREM_I64, "_aullrem");
246 setLibcallName(RTLIB::MUL_I64, "_allmul");
247 setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall);
248 setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall);
249 setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall);
250 setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall);
251 setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall);
253 // The _ftol2 runtime function has an unusual calling conv, which
254 // is modeled by a special pseudo-instruction.
255 setLibcallName(RTLIB::FPTOUINT_F64_I64, nullptr);
256 setLibcallName(RTLIB::FPTOUINT_F32_I64, nullptr);
257 setLibcallName(RTLIB::FPTOUINT_F64_I32, nullptr);
258 setLibcallName(RTLIB::FPTOUINT_F32_I32, nullptr);
261 if (Subtarget->isTargetDarwin()) {
262 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
263 setUseUnderscoreSetJmp(false);
264 setUseUnderscoreLongJmp(false);
265 } else if (Subtarget->isTargetWindowsGNU()) {
266 // MS runtime is weird: it exports _setjmp, but longjmp!
267 setUseUnderscoreSetJmp(true);
268 setUseUnderscoreLongJmp(false);
270 setUseUnderscoreSetJmp(true);
271 setUseUnderscoreLongJmp(true);
274 // Set up the register classes.
275 addRegisterClass(MVT::i8, &X86::GR8RegClass);
276 addRegisterClass(MVT::i16, &X86::GR16RegClass);
277 addRegisterClass(MVT::i32, &X86::GR32RegClass);
278 if (Subtarget->is64Bit())
279 addRegisterClass(MVT::i64, &X86::GR64RegClass);
281 for (MVT VT : MVT::integer_valuetypes())
282 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
284 // We don't accept any truncstore of integer registers.
285 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
286 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
287 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
288 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
289 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
290 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
292 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
294 // SETOEQ and SETUNE require checking two conditions.
295 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
296 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
297 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand);
298 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand);
299 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
300 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand);
302 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
304 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
305 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
306 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
308 if (Subtarget->is64Bit()) {
309 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
310 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
311 } else if (!TM.Options.UseSoftFloat) {
312 // We have an algorithm for SSE2->double, and we turn this into a
313 // 64-bit FILD followed by conditional FADD for other targets.
314 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
315 // We have an algorithm for SSE2, and we turn this into a 64-bit
316 // FILD for other targets.
317 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom);
320 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
322 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
323 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
325 if (!TM.Options.UseSoftFloat) {
326 // SSE has no i16 to fp conversion, only i32
327 if (X86ScalarSSEf32) {
328 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
329 // f32 and f64 cases are Legal, f80 case is not
330 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
332 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
333 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
336 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
337 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote);
340 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
341 // are Legal, f80 is custom lowered.
342 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
343 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
345 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
347 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
348 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
350 if (X86ScalarSSEf32) {
351 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
352 // f32 and f64 cases are Legal, f80 case is not
353 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
355 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
356 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
359 // Handle FP_TO_UINT by promoting the destination to a larger signed
361 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
362 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
363 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
365 if (Subtarget->is64Bit()) {
366 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand);
367 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
368 } else if (!TM.Options.UseSoftFloat) {
369 // Since AVX is a superset of SSE3, only check for SSE here.
370 if (Subtarget->hasSSE1() && !Subtarget->hasSSE3())
371 // Expand FP_TO_UINT into a select.
372 // FIXME: We would like to use a Custom expander here eventually to do
373 // the optimal thing for SSE vs. the default expansion in the legalizer.
374 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
376 // With SSE3 we can use fisttpll to convert to a signed i64; without
377 // SSE, we're stuck with a fistpll.
378 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom);
381 if (isTargetFTOL()) {
382 // Use the _ftol2 runtime function, which has a pseudo-instruction
383 // to handle its weird calling convention.
384 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom);
387 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
388 if (!X86ScalarSSEf64) {
389 setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
390 setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
391 if (Subtarget->is64Bit()) {
392 setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
393 // Without SSE, i64->f64 goes through memory.
394 setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
398 // Scalar integer divide and remainder are lowered to use operations that
399 // produce two results, to match the available instructions. This exposes
400 // the two-result form to trivial CSE, which is able to combine x/y and x%y
401 // into a single instruction.
403 // Scalar integer multiply-high is also lowered to use two-result
404 // operations, to match the available instructions. However, plain multiply
405 // (low) operations are left as Legal, as there are single-result
406 // instructions for this in x86. Using the two-result multiply instructions
407 // when both high and low results are needed must be arranged by dagcombine.
408 for (unsigned i = 0; i != array_lengthof(IntVTs); ++i) {
410 setOperationAction(ISD::MULHS, VT, Expand);
411 setOperationAction(ISD::MULHU, VT, Expand);
412 setOperationAction(ISD::SDIV, VT, Expand);
413 setOperationAction(ISD::UDIV, VT, Expand);
414 setOperationAction(ISD::SREM, VT, Expand);
415 setOperationAction(ISD::UREM, VT, Expand);
417 // Add/Sub overflow ops with MVT::Glues are lowered to EFLAGS dependences.
418 setOperationAction(ISD::ADDC, VT, Custom);
419 setOperationAction(ISD::ADDE, VT, Custom);
420 setOperationAction(ISD::SUBC, VT, Custom);
421 setOperationAction(ISD::SUBE, VT, Custom);
424 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
425 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
426 setOperationAction(ISD::BR_CC , MVT::f32, Expand);
427 setOperationAction(ISD::BR_CC , MVT::f64, Expand);
428 setOperationAction(ISD::BR_CC , MVT::f80, Expand);
429 setOperationAction(ISD::BR_CC , MVT::i8, Expand);
430 setOperationAction(ISD::BR_CC , MVT::i16, Expand);
431 setOperationAction(ISD::BR_CC , MVT::i32, Expand);
432 setOperationAction(ISD::BR_CC , MVT::i64, Expand);
433 setOperationAction(ISD::SELECT_CC , MVT::f32, Expand);
434 setOperationAction(ISD::SELECT_CC , MVT::f64, Expand);
435 setOperationAction(ISD::SELECT_CC , MVT::f80, Expand);
436 setOperationAction(ISD::SELECT_CC , MVT::i8, Expand);
437 setOperationAction(ISD::SELECT_CC , MVT::i16, Expand);
438 setOperationAction(ISD::SELECT_CC , MVT::i32, Expand);
439 setOperationAction(ISD::SELECT_CC , MVT::i64, Expand);
440 if (Subtarget->is64Bit())
441 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
442 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
443 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
444 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
445 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
446 setOperationAction(ISD::FREM , MVT::f32 , Expand);
447 setOperationAction(ISD::FREM , MVT::f64 , Expand);
448 setOperationAction(ISD::FREM , MVT::f80 , Expand);
449 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom);
451 // Promote the i8 variants and force them on up to i32 which has a shorter
453 setOperationAction(ISD::CTTZ , MVT::i8 , Promote);
454 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32);
455 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote);
456 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32);
457 if (Subtarget->hasBMI()) {
458 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand);
459 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand);
460 if (Subtarget->is64Bit())
461 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
463 setOperationAction(ISD::CTTZ , MVT::i16 , Custom);
464 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
465 if (Subtarget->is64Bit())
466 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
469 if (Subtarget->hasLZCNT()) {
470 // When promoting the i8 variants, force them to i32 for a shorter
472 setOperationAction(ISD::CTLZ , MVT::i8 , Promote);
473 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32);
474 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote);
475 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
476 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand);
477 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand);
478 if (Subtarget->is64Bit())
479 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand);
481 setOperationAction(ISD::CTLZ , MVT::i8 , Custom);
482 setOperationAction(ISD::CTLZ , MVT::i16 , Custom);
483 setOperationAction(ISD::CTLZ , MVT::i32 , Custom);
484 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom);
485 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom);
486 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom);
487 if (Subtarget->is64Bit()) {
488 setOperationAction(ISD::CTLZ , MVT::i64 , Custom);
489 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);
493 // Special handling for half-precision floating point conversions.
494 // If we don't have F16C support, then lower half float conversions
495 // into library calls.
496 if (TM.Options.UseSoftFloat || !Subtarget->hasF16C()) {
497 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
498 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
501 // There's never any support for operations beyond MVT::f32.
502 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
503 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand);
504 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
505 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand);
507 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
508 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
509 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand);
510 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
511 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
512 setTruncStoreAction(MVT::f80, MVT::f16, Expand);
514 if (Subtarget->hasPOPCNT()) {
515 setOperationAction(ISD::CTPOP , MVT::i8 , Promote);
517 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
518 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
519 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
520 if (Subtarget->is64Bit())
521 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
524 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
526 if (!Subtarget->hasMOVBE())
527 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
529 // These should be promoted to a larger select which is supported.
530 setOperationAction(ISD::SELECT , MVT::i1 , Promote);
531 // X86 wants to expand cmov itself.
532 setOperationAction(ISD::SELECT , MVT::i8 , Custom);
533 setOperationAction(ISD::SELECT , MVT::i16 , Custom);
534 setOperationAction(ISD::SELECT , MVT::i32 , Custom);
535 setOperationAction(ISD::SELECT , MVT::f32 , Custom);
536 setOperationAction(ISD::SELECT , MVT::f64 , Custom);
537 setOperationAction(ISD::SELECT , MVT::f80 , Custom);
538 setOperationAction(ISD::SETCC , MVT::i8 , Custom);
539 setOperationAction(ISD::SETCC , MVT::i16 , Custom);
540 setOperationAction(ISD::SETCC , MVT::i32 , Custom);
541 setOperationAction(ISD::SETCC , MVT::f32 , Custom);
542 setOperationAction(ISD::SETCC , MVT::f64 , Custom);
543 setOperationAction(ISD::SETCC , MVT::f80 , Custom);
544 if (Subtarget->is64Bit()) {
545 setOperationAction(ISD::SELECT , MVT::i64 , Custom);
546 setOperationAction(ISD::SETCC , MVT::i64 , Custom);
548 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
549 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
550 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
551 // support continuation, user-level threading, and etc.. As a result, no
552 // other SjLj exception interfaces are implemented and please don't build
553 // your own exception handling based on them.
554 // LLVM/Clang supports zero-cost DWARF exception handling.
555 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
556 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
559 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom);
560 setOperationAction(ISD::JumpTable , MVT::i32 , Custom);
561 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom);
562 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom);
563 if (Subtarget->is64Bit())
564 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
565 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom);
566 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom);
567 if (Subtarget->is64Bit()) {
568 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom);
569 setOperationAction(ISD::JumpTable , MVT::i64 , Custom);
570 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom);
571 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom);
572 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom);
574 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
575 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom);
576 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom);
577 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom);
578 if (Subtarget->is64Bit()) {
579 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom);
580 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom);
581 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom);
584 if (Subtarget->hasSSE1())
585 setOperationAction(ISD::PREFETCH , MVT::Other, Legal);
587 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
589 // Expand certain atomics
590 for (unsigned i = 0; i != array_lengthof(IntVTs); ++i) {
592 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
593 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
594 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
597 if (Subtarget->hasCmpxchg16b()) {
598 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
601 // FIXME - use subtarget debug flags
602 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetELF() &&
603 !Subtarget->isTargetCygMing() && !Subtarget->isTargetWin64()) {
604 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
607 if (Subtarget->is64Bit()) {
608 setExceptionPointerRegister(X86::RAX);
609 setExceptionSelectorRegister(X86::RDX);
611 setExceptionPointerRegister(X86::EAX);
612 setExceptionSelectorRegister(X86::EDX);
614 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
615 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
617 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
618 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
620 setOperationAction(ISD::TRAP, MVT::Other, Legal);
621 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
623 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
624 setOperationAction(ISD::VASTART , MVT::Other, Custom);
625 setOperationAction(ISD::VAEND , MVT::Other, Expand);
626 if (Subtarget->is64Bit() && !Subtarget->isTargetWin64()) {
627 // TargetInfo::X86_64ABIBuiltinVaList
628 setOperationAction(ISD::VAARG , MVT::Other, Custom);
629 setOperationAction(ISD::VACOPY , MVT::Other, Custom);
631 // TargetInfo::CharPtrBuiltinVaList
632 setOperationAction(ISD::VAARG , MVT::Other, Expand);
633 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
636 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
637 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
639 setOperationAction(ISD::DYNAMIC_STACKALLOC, getPointerTy(), Custom);
641 if (!TM.Options.UseSoftFloat && X86ScalarSSEf64) {
642 // f32 and f64 use SSE.
643 // Set up the FP register classes.
644 addRegisterClass(MVT::f32, &X86::FR32RegClass);
645 addRegisterClass(MVT::f64, &X86::FR64RegClass);
647 // Use ANDPD to simulate FABS.
648 setOperationAction(ISD::FABS , MVT::f64, Custom);
649 setOperationAction(ISD::FABS , MVT::f32, Custom);
651 // Use XORP to simulate FNEG.
652 setOperationAction(ISD::FNEG , MVT::f64, Custom);
653 setOperationAction(ISD::FNEG , MVT::f32, Custom);
655 // Use ANDPD and ORPD to simulate FCOPYSIGN.
656 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
657 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
659 // Lower this to FGETSIGNx86 plus an AND.
660 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
661 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
663 // We don't support sin/cos/fmod
664 setOperationAction(ISD::FSIN , MVT::f64, Expand);
665 setOperationAction(ISD::FCOS , MVT::f64, Expand);
666 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
667 setOperationAction(ISD::FSIN , MVT::f32, Expand);
668 setOperationAction(ISD::FCOS , MVT::f32, Expand);
669 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
671 // Expand FP immediates into loads from the stack, except for the special
673 addLegalFPImmediate(APFloat(+0.0)); // xorpd
674 addLegalFPImmediate(APFloat(+0.0f)); // xorps
675 } else if (!TM.Options.UseSoftFloat && X86ScalarSSEf32) {
676 // Use SSE for f32, x87 for f64.
677 // Set up the FP register classes.
678 addRegisterClass(MVT::f32, &X86::FR32RegClass);
679 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
681 // Use ANDPS to simulate FABS.
682 setOperationAction(ISD::FABS , MVT::f32, Custom);
684 // Use XORP to simulate FNEG.
685 setOperationAction(ISD::FNEG , MVT::f32, Custom);
687 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
689 // Use ANDPS and ORPS to simulate FCOPYSIGN.
690 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
691 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
693 // We don't support sin/cos/fmod
694 setOperationAction(ISD::FSIN , MVT::f32, Expand);
695 setOperationAction(ISD::FCOS , MVT::f32, Expand);
696 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
698 // Special cases we handle for FP constants.
699 addLegalFPImmediate(APFloat(+0.0f)); // xorps
700 addLegalFPImmediate(APFloat(+0.0)); // FLD0
701 addLegalFPImmediate(APFloat(+1.0)); // FLD1
702 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
703 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
705 if (!TM.Options.UnsafeFPMath) {
706 setOperationAction(ISD::FSIN , MVT::f64, Expand);
707 setOperationAction(ISD::FCOS , MVT::f64, Expand);
708 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
710 } else if (!TM.Options.UseSoftFloat) {
711 // f32 and f64 in x87.
712 // Set up the FP register classes.
713 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
714 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
716 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
717 setOperationAction(ISD::UNDEF, MVT::f32, Expand);
718 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
719 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
721 if (!TM.Options.UnsafeFPMath) {
722 setOperationAction(ISD::FSIN , MVT::f64, Expand);
723 setOperationAction(ISD::FSIN , MVT::f32, Expand);
724 setOperationAction(ISD::FCOS , MVT::f64, Expand);
725 setOperationAction(ISD::FCOS , MVT::f32, Expand);
726 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
727 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
729 addLegalFPImmediate(APFloat(+0.0)); // FLD0
730 addLegalFPImmediate(APFloat(+1.0)); // FLD1
731 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
732 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
733 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
734 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
735 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
736 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
739 // We don't support FMA.
740 setOperationAction(ISD::FMA, MVT::f64, Expand);
741 setOperationAction(ISD::FMA, MVT::f32, Expand);
743 // Long double always uses X87.
744 if (!TM.Options.UseSoftFloat) {
745 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
746 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
747 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
749 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended);
750 addLegalFPImmediate(TmpFlt); // FLD0
752 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
755 APFloat TmpFlt2(+1.0);
756 TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven,
758 addLegalFPImmediate(TmpFlt2); // FLD1
759 TmpFlt2.changeSign();
760 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
763 if (!TM.Options.UnsafeFPMath) {
764 setOperationAction(ISD::FSIN , MVT::f80, Expand);
765 setOperationAction(ISD::FCOS , MVT::f80, Expand);
766 setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
769 setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
770 setOperationAction(ISD::FCEIL, MVT::f80, Expand);
771 setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
772 setOperationAction(ISD::FRINT, MVT::f80, Expand);
773 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
774 setOperationAction(ISD::FMA, MVT::f80, Expand);
777 // Always use a library call for pow.
778 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
779 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
780 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
782 setOperationAction(ISD::FLOG, MVT::f80, Expand);
783 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
784 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
785 setOperationAction(ISD::FEXP, MVT::f80, Expand);
786 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
787 setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
788 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
790 // First set operation action for all vector types to either promote
791 // (for widening) or expand (for scalarization). Then we will selectively
792 // turn on ones that can be effectively codegen'd.
793 for (MVT VT : MVT::vector_valuetypes()) {
794 setOperationAction(ISD::ADD , VT, Expand);
795 setOperationAction(ISD::SUB , VT, Expand);
796 setOperationAction(ISD::FADD, VT, Expand);
797 setOperationAction(ISD::FNEG, VT, Expand);
798 setOperationAction(ISD::FSUB, VT, Expand);
799 setOperationAction(ISD::MUL , VT, Expand);
800 setOperationAction(ISD::FMUL, VT, Expand);
801 setOperationAction(ISD::SDIV, VT, Expand);
802 setOperationAction(ISD::UDIV, VT, Expand);
803 setOperationAction(ISD::FDIV, VT, Expand);
804 setOperationAction(ISD::SREM, VT, Expand);
805 setOperationAction(ISD::UREM, VT, Expand);
806 setOperationAction(ISD::LOAD, VT, Expand);
807 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
808 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
809 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
810 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
811 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
812 setOperationAction(ISD::FABS, VT, Expand);
813 setOperationAction(ISD::FSIN, VT, Expand);
814 setOperationAction(ISD::FSINCOS, VT, Expand);
815 setOperationAction(ISD::FCOS, VT, Expand);
816 setOperationAction(ISD::FSINCOS, VT, Expand);
817 setOperationAction(ISD::FREM, VT, Expand);
818 setOperationAction(ISD::FMA, VT, Expand);
819 setOperationAction(ISD::FPOWI, VT, Expand);
820 setOperationAction(ISD::FSQRT, VT, Expand);
821 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
822 setOperationAction(ISD::FFLOOR, VT, Expand);
823 setOperationAction(ISD::FCEIL, VT, Expand);
824 setOperationAction(ISD::FTRUNC, VT, Expand);
825 setOperationAction(ISD::FRINT, VT, Expand);
826 setOperationAction(ISD::FNEARBYINT, VT, Expand);
827 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
828 setOperationAction(ISD::MULHS, VT, Expand);
829 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
830 setOperationAction(ISD::MULHU, VT, Expand);
831 setOperationAction(ISD::SDIVREM, VT, Expand);
832 setOperationAction(ISD::UDIVREM, VT, Expand);
833 setOperationAction(ISD::FPOW, VT, Expand);
834 setOperationAction(ISD::CTPOP, VT, Expand);
835 setOperationAction(ISD::CTTZ, VT, Expand);
836 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
837 setOperationAction(ISD::CTLZ, VT, Expand);
838 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
839 setOperationAction(ISD::SHL, VT, Expand);
840 setOperationAction(ISD::SRA, VT, Expand);
841 setOperationAction(ISD::SRL, VT, Expand);
842 setOperationAction(ISD::ROTL, VT, Expand);
843 setOperationAction(ISD::ROTR, VT, Expand);
844 setOperationAction(ISD::BSWAP, VT, Expand);
845 setOperationAction(ISD::SETCC, VT, Expand);
846 setOperationAction(ISD::FLOG, VT, Expand);
847 setOperationAction(ISD::FLOG2, VT, Expand);
848 setOperationAction(ISD::FLOG10, VT, Expand);
849 setOperationAction(ISD::FEXP, VT, Expand);
850 setOperationAction(ISD::FEXP2, VT, Expand);
851 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
852 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
853 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
854 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
855 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
856 setOperationAction(ISD::TRUNCATE, VT, Expand);
857 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
858 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
859 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
860 setOperationAction(ISD::VSELECT, VT, Expand);
861 setOperationAction(ISD::SELECT_CC, VT, Expand);
862 for (MVT InnerVT : MVT::vector_valuetypes()) {
863 setTruncStoreAction(InnerVT, VT, Expand);
865 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
866 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
868 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
869 // types, we have to deal with them whether we ask for Expansion or not.
870 // Setting Expand causes its own optimisation problems though, so leave
872 if (VT.getVectorElementType() == MVT::i1)
873 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
877 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
878 // with -msoft-float, disable use of MMX as well.
879 if (!TM.Options.UseSoftFloat && Subtarget->hasMMX()) {
880 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
881 // No operations on x86mmx supported, everything uses intrinsics.
884 // MMX-sized vectors (other than x86mmx) are expected to be expanded
885 // into smaller operations.
886 setOperationAction(ISD::MULHS, MVT::v8i8, Expand);
887 setOperationAction(ISD::MULHS, MVT::v4i16, Expand);
888 setOperationAction(ISD::MULHS, MVT::v2i32, Expand);
889 setOperationAction(ISD::MULHS, MVT::v1i64, Expand);
890 setOperationAction(ISD::AND, MVT::v8i8, Expand);
891 setOperationAction(ISD::AND, MVT::v4i16, Expand);
892 setOperationAction(ISD::AND, MVT::v2i32, Expand);
893 setOperationAction(ISD::AND, MVT::v1i64, Expand);
894 setOperationAction(ISD::OR, MVT::v8i8, Expand);
895 setOperationAction(ISD::OR, MVT::v4i16, Expand);
896 setOperationAction(ISD::OR, MVT::v2i32, Expand);
897 setOperationAction(ISD::OR, MVT::v1i64, Expand);
898 setOperationAction(ISD::XOR, MVT::v8i8, Expand);
899 setOperationAction(ISD::XOR, MVT::v4i16, Expand);
900 setOperationAction(ISD::XOR, MVT::v2i32, Expand);
901 setOperationAction(ISD::XOR, MVT::v1i64, Expand);
902 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i8, Expand);
903 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i16, Expand);
904 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i32, Expand);
905 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v1i64, Expand);
906 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand);
907 setOperationAction(ISD::SELECT, MVT::v8i8, Expand);
908 setOperationAction(ISD::SELECT, MVT::v4i16, Expand);
909 setOperationAction(ISD::SELECT, MVT::v2i32, Expand);
910 setOperationAction(ISD::SELECT, MVT::v1i64, Expand);
911 setOperationAction(ISD::BITCAST, MVT::v8i8, Expand);
912 setOperationAction(ISD::BITCAST, MVT::v4i16, Expand);
913 setOperationAction(ISD::BITCAST, MVT::v2i32, Expand);
914 setOperationAction(ISD::BITCAST, MVT::v1i64, Expand);
916 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE1()) {
917 addRegisterClass(MVT::v4f32, &X86::VR128RegClass);
919 setOperationAction(ISD::FADD, MVT::v4f32, Legal);
920 setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
921 setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
922 setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
923 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
924 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
925 setOperationAction(ISD::FABS, MVT::v4f32, Custom);
926 setOperationAction(ISD::LOAD, MVT::v4f32, Legal);
927 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
928 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
929 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
930 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
931 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
934 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE2()) {
935 addRegisterClass(MVT::v2f64, &X86::VR128RegClass);
937 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
938 // registers cannot be used even for integer operations.
939 addRegisterClass(MVT::v16i8, &X86::VR128RegClass);
940 addRegisterClass(MVT::v8i16, &X86::VR128RegClass);
941 addRegisterClass(MVT::v4i32, &X86::VR128RegClass);
942 addRegisterClass(MVT::v2i64, &X86::VR128RegClass);
944 setOperationAction(ISD::ADD, MVT::v16i8, Legal);
945 setOperationAction(ISD::ADD, MVT::v8i16, Legal);
946 setOperationAction(ISD::ADD, MVT::v4i32, Legal);
947 setOperationAction(ISD::ADD, MVT::v2i64, Legal);
948 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
949 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
950 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom);
951 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom);
952 setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
953 setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
954 setOperationAction(ISD::SUB, MVT::v16i8, Legal);
955 setOperationAction(ISD::SUB, MVT::v8i16, Legal);
956 setOperationAction(ISD::SUB, MVT::v4i32, Legal);
957 setOperationAction(ISD::SUB, MVT::v2i64, Legal);
958 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
959 setOperationAction(ISD::FADD, MVT::v2f64, Legal);
960 setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
961 setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
962 setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
963 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
964 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
965 setOperationAction(ISD::FABS, MVT::v2f64, Custom);
967 setOperationAction(ISD::SETCC, MVT::v2i64, Custom);
968 setOperationAction(ISD::SETCC, MVT::v16i8, Custom);
969 setOperationAction(ISD::SETCC, MVT::v8i16, Custom);
970 setOperationAction(ISD::SETCC, MVT::v4i32, Custom);
972 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
973 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
974 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
975 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
976 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
978 // Only provide customized ctpop vector bit twiddling for vector types we
979 // know to perform better than using the popcnt instructions on each vector
980 // element. If popcnt isn't supported, always provide the custom version.
981 if (!Subtarget->hasPOPCNT()) {
982 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
983 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom);
986 // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
987 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) {
988 MVT VT = (MVT::SimpleValueType)i;
989 // Do not attempt to custom lower non-power-of-2 vectors
990 if (!isPowerOf2_32(VT.getVectorNumElements()))
992 // Do not attempt to custom lower non-128-bit vectors
993 if (!VT.is128BitVector())
995 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
996 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
997 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1000 // We support custom legalizing of sext and anyext loads for specific
1001 // memory vector types which we can load as a scalar (or sequence of
1002 // scalars) and extend in-register to a legal 128-bit vector type. For sext
1003 // loads these must work with a single scalar load.
1004 for (MVT VT : MVT::integer_vector_valuetypes()) {
1005 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom);
1006 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom);
1007 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom);
1008 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom);
1009 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom);
1010 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom);
1011 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom);
1012 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom);
1013 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom);
1016 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
1017 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
1018 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
1019 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
1020 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
1021 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
1023 if (Subtarget->is64Bit()) {
1024 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom);
1025 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
1028 // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64.
1029 for (int i = MVT::v16i8; i != MVT::v2i64; ++i) {
1030 MVT VT = (MVT::SimpleValueType)i;
1032 // Do not attempt to promote non-128-bit vectors
1033 if (!VT.is128BitVector())
1036 setOperationAction(ISD::AND, VT, Promote);
1037 AddPromotedToType (ISD::AND, VT, MVT::v2i64);
1038 setOperationAction(ISD::OR, VT, Promote);
1039 AddPromotedToType (ISD::OR, VT, MVT::v2i64);
1040 setOperationAction(ISD::XOR, VT, Promote);
1041 AddPromotedToType (ISD::XOR, VT, MVT::v2i64);
1042 setOperationAction(ISD::LOAD, VT, Promote);
1043 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64);
1044 setOperationAction(ISD::SELECT, VT, Promote);
1045 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64);
1048 // Custom lower v2i64 and v2f64 selects.
1049 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
1050 setOperationAction(ISD::LOAD, MVT::v2i64, Legal);
1051 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
1052 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
1054 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
1055 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
1057 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom);
1058 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1059 // As there is no 64-bit GPR available, we need build a special custom
1060 // sequence to convert from v2i32 to v2f32.
1061 if (!Subtarget->is64Bit())
1062 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
1064 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
1065 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
1067 for (MVT VT : MVT::fp_vector_valuetypes())
1068 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal);
1070 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
1071 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
1072 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
1075 if (!TM.Options.UseSoftFloat && Subtarget->hasSSE41()) {
1076 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
1077 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
1078 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
1079 setOperationAction(ISD::FRINT, MVT::f32, Legal);
1080 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
1081 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
1082 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
1083 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
1084 setOperationAction(ISD::FRINT, MVT::f64, Legal);
1085 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
1087 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
1088 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
1089 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
1090 setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
1091 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
1092 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
1093 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
1094 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
1095 setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
1096 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
1098 // FIXME: Do we need to handle scalar-to-vector here?
1099 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1101 setOperationAction(ISD::VSELECT, MVT::v2f64, Custom);
1102 setOperationAction(ISD::VSELECT, MVT::v2i64, Custom);
1103 setOperationAction(ISD::VSELECT, MVT::v4i32, Custom);
1104 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
1105 setOperationAction(ISD::VSELECT, MVT::v8i16, Custom);
1106 // There is no BLENDI for byte vectors. We don't need to custom lower
1107 // some vselects for now.
1108 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1110 // SSE41 brings specific instructions for doing vector sign extend even in
1111 // cases where we don't have SRA.
1112 for (MVT VT : MVT::integer_vector_valuetypes()) {
1113 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom);
1114 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom);
1115 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom);
1118 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1119 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal);
1120 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal);
1121 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal);
1122 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal);
1123 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal);
1124 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal);
1126 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal);
1127 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal);
1128 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal);
1129 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal);
1130 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal);
1131 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal);
1133 // i8 and i16 vectors are custom because the source register and source
1134 // source memory operand types are not the same width. f32 vectors are
1135 // custom since the immediate controlling the insert encodes additional
1137 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1138 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
1139 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
1140 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
1142 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom);
1143 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom);
1144 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom);
1145 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
1147 // FIXME: these should be Legal, but that's only for the case where
1148 // the index is constant. For now custom expand to deal with that.
1149 if (Subtarget->is64Bit()) {
1150 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom);
1151 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
1155 if (Subtarget->hasSSE2()) {
1156 setOperationAction(ISD::SRL, MVT::v8i16, Custom);
1157 setOperationAction(ISD::SRL, MVT::v16i8, Custom);
1159 setOperationAction(ISD::SHL, MVT::v8i16, Custom);
1160 setOperationAction(ISD::SHL, MVT::v16i8, Custom);
1162 setOperationAction(ISD::SRA, MVT::v8i16, Custom);
1163 setOperationAction(ISD::SRA, MVT::v16i8, Custom);
1165 // In the customized shift lowering, the legal cases in AVX2 will be
1167 setOperationAction(ISD::SRL, MVT::v2i64, Custom);
1168 setOperationAction(ISD::SRL, MVT::v4i32, Custom);
1170 setOperationAction(ISD::SHL, MVT::v2i64, Custom);
1171 setOperationAction(ISD::SHL, MVT::v4i32, Custom);
1173 setOperationAction(ISD::SRA, MVT::v4i32, Custom);
1176 if (!TM.Options.UseSoftFloat && Subtarget->hasFp256()) {
1177 addRegisterClass(MVT::v32i8, &X86::VR256RegClass);
1178 addRegisterClass(MVT::v16i16, &X86::VR256RegClass);
1179 addRegisterClass(MVT::v8i32, &X86::VR256RegClass);
1180 addRegisterClass(MVT::v8f32, &X86::VR256RegClass);
1181 addRegisterClass(MVT::v4i64, &X86::VR256RegClass);
1182 addRegisterClass(MVT::v4f64, &X86::VR256RegClass);
1184 setOperationAction(ISD::LOAD, MVT::v8f32, Legal);
1185 setOperationAction(ISD::LOAD, MVT::v4f64, Legal);
1186 setOperationAction(ISD::LOAD, MVT::v4i64, Legal);
1188 setOperationAction(ISD::FADD, MVT::v8f32, Legal);
1189 setOperationAction(ISD::FSUB, MVT::v8f32, Legal);
1190 setOperationAction(ISD::FMUL, MVT::v8f32, Legal);
1191 setOperationAction(ISD::FDIV, MVT::v8f32, Legal);
1192 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal);
1193 setOperationAction(ISD::FFLOOR, MVT::v8f32, Legal);
1194 setOperationAction(ISD::FCEIL, MVT::v8f32, Legal);
1195 setOperationAction(ISD::FTRUNC, MVT::v8f32, Legal);
1196 setOperationAction(ISD::FRINT, MVT::v8f32, Legal);
1197 setOperationAction(ISD::FNEARBYINT, MVT::v8f32, Legal);
1198 setOperationAction(ISD::FNEG, MVT::v8f32, Custom);
1199 setOperationAction(ISD::FABS, MVT::v8f32, Custom);
1201 setOperationAction(ISD::FADD, MVT::v4f64, Legal);
1202 setOperationAction(ISD::FSUB, MVT::v4f64, Legal);
1203 setOperationAction(ISD::FMUL, MVT::v4f64, Legal);
1204 setOperationAction(ISD::FDIV, MVT::v4f64, Legal);
1205 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal);
1206 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal);
1207 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal);
1208 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal);
1209 setOperationAction(ISD::FRINT, MVT::v4f64, Legal);
1210 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Legal);
1211 setOperationAction(ISD::FNEG, MVT::v4f64, Custom);
1212 setOperationAction(ISD::FABS, MVT::v4f64, Custom);
1214 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1215 // even though v8i16 is a legal type.
1216 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote);
1217 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote);
1218 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal);
1220 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote);
1221 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal);
1222 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal);
1224 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom);
1225 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
1227 for (MVT VT : MVT::fp_vector_valuetypes())
1228 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal);
1230 setOperationAction(ISD::SRL, MVT::v16i16, Custom);
1231 setOperationAction(ISD::SRL, MVT::v32i8, Custom);
1233 setOperationAction(ISD::SHL, MVT::v16i16, Custom);
1234 setOperationAction(ISD::SHL, MVT::v32i8, Custom);
1236 setOperationAction(ISD::SRA, MVT::v16i16, Custom);
1237 setOperationAction(ISD::SRA, MVT::v32i8, Custom);
1239 setOperationAction(ISD::SETCC, MVT::v32i8, Custom);
1240 setOperationAction(ISD::SETCC, MVT::v16i16, Custom);
1241 setOperationAction(ISD::SETCC, MVT::v8i32, Custom);
1242 setOperationAction(ISD::SETCC, MVT::v4i64, Custom);
1244 setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1245 setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1246 setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1248 setOperationAction(ISD::VSELECT, MVT::v4f64, Custom);
1249 setOperationAction(ISD::VSELECT, MVT::v4i64, Custom);
1250 setOperationAction(ISD::VSELECT, MVT::v8i32, Custom);
1251 setOperationAction(ISD::VSELECT, MVT::v8f32, Custom);
1253 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1254 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom);
1255 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
1256 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom);
1257 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom);
1258 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom);
1259 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom);
1260 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom);
1261 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom);
1262 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1263 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1264 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1266 if (Subtarget->hasFMA() || Subtarget->hasFMA4()) {
1267 setOperationAction(ISD::FMA, MVT::v8f32, Legal);
1268 setOperationAction(ISD::FMA, MVT::v4f64, Legal);
1269 setOperationAction(ISD::FMA, MVT::v4f32, Legal);
1270 setOperationAction(ISD::FMA, MVT::v2f64, Legal);
1271 setOperationAction(ISD::FMA, MVT::f32, Legal);
1272 setOperationAction(ISD::FMA, MVT::f64, Legal);
1275 if (Subtarget->hasInt256()) {
1276 setOperationAction(ISD::ADD, MVT::v4i64, Legal);
1277 setOperationAction(ISD::ADD, MVT::v8i32, Legal);
1278 setOperationAction(ISD::ADD, MVT::v16i16, Legal);
1279 setOperationAction(ISD::ADD, MVT::v32i8, Legal);
1281 setOperationAction(ISD::SUB, MVT::v4i64, Legal);
1282 setOperationAction(ISD::SUB, MVT::v8i32, Legal);
1283 setOperationAction(ISD::SUB, MVT::v16i16, Legal);
1284 setOperationAction(ISD::SUB, MVT::v32i8, Legal);
1286 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1287 setOperationAction(ISD::MUL, MVT::v8i32, Legal);
1288 setOperationAction(ISD::MUL, MVT::v16i16, Legal);
1289 // Don't lower v32i8 because there is no 128-bit byte mul
1291 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom);
1292 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom);
1293 setOperationAction(ISD::MULHU, MVT::v16i16, Legal);
1294 setOperationAction(ISD::MULHS, MVT::v16i16, Legal);
1296 setOperationAction(ISD::VSELECT, MVT::v16i16, Custom);
1297 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1299 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1300 // when we have a 256bit-wide blend with immediate.
1301 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1303 // Only provide customized ctpop vector bit twiddling for vector types we
1304 // know to perform better than using the popcnt instructions on each
1305 // vector element. If popcnt isn't supported, always provide the custom
1307 if (!Subtarget->hasPOPCNT())
1308 setOperationAction(ISD::CTPOP, MVT::v4i64, Custom);
1310 // Custom CTPOP always performs better on natively supported v8i32
1311 setOperationAction(ISD::CTPOP, MVT::v8i32, Custom);
1313 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1314 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
1315 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal);
1316 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal);
1317 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal);
1318 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal);
1319 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal);
1321 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal);
1322 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal);
1323 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal);
1324 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal);
1325 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal);
1326 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal);
1328 setOperationAction(ISD::ADD, MVT::v4i64, Custom);
1329 setOperationAction(ISD::ADD, MVT::v8i32, Custom);
1330 setOperationAction(ISD::ADD, MVT::v16i16, Custom);
1331 setOperationAction(ISD::ADD, MVT::v32i8, Custom);
1333 setOperationAction(ISD::SUB, MVT::v4i64, Custom);
1334 setOperationAction(ISD::SUB, MVT::v8i32, Custom);
1335 setOperationAction(ISD::SUB, MVT::v16i16, Custom);
1336 setOperationAction(ISD::SUB, MVT::v32i8, Custom);
1338 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1339 setOperationAction(ISD::MUL, MVT::v8i32, Custom);
1340 setOperationAction(ISD::MUL, MVT::v16i16, Custom);
1341 // Don't lower v32i8 because there is no 128-bit byte mul
1344 // In the customized shift lowering, the legal cases in AVX2 will be
1346 setOperationAction(ISD::SRL, MVT::v4i64, Custom);
1347 setOperationAction(ISD::SRL, MVT::v8i32, Custom);
1349 setOperationAction(ISD::SHL, MVT::v4i64, Custom);
1350 setOperationAction(ISD::SHL, MVT::v8i32, Custom);
1352 setOperationAction(ISD::SRA, MVT::v8i32, Custom);
1354 // Custom lower several nodes for 256-bit types.
1355 for (MVT VT : MVT::vector_valuetypes()) {
1356 if (VT.getScalarSizeInBits() >= 32) {
1357 setOperationAction(ISD::MLOAD, VT, Legal);
1358 setOperationAction(ISD::MSTORE, VT, Legal);
1360 // Extract subvector is special because the value type
1361 // (result) is 128-bit but the source is 256-bit wide.
1362 if (VT.is128BitVector()) {
1363 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1365 // Do not attempt to custom lower other non-256-bit vectors
1366 if (!VT.is256BitVector())
1369 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1370 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1371 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1372 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1373 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1374 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1375 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1378 // Promote v32i8, v16i16, v8i32 select, and, or, xor to v4i64.
1379 for (int i = MVT::v32i8; i != MVT::v4i64; ++i) {
1380 MVT VT = (MVT::SimpleValueType)i;
1382 // Do not attempt to promote non-256-bit vectors
1383 if (!VT.is256BitVector())
1386 setOperationAction(ISD::AND, VT, Promote);
1387 AddPromotedToType (ISD::AND, VT, MVT::v4i64);
1388 setOperationAction(ISD::OR, VT, Promote);
1389 AddPromotedToType (ISD::OR, VT, MVT::v4i64);
1390 setOperationAction(ISD::XOR, VT, Promote);
1391 AddPromotedToType (ISD::XOR, VT, MVT::v4i64);
1392 setOperationAction(ISD::LOAD, VT, Promote);
1393 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64);
1394 setOperationAction(ISD::SELECT, VT, Promote);
1395 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64);
1399 if (!TM.Options.UseSoftFloat && Subtarget->hasAVX512()) {
1400 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1401 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1402 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1403 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1405 addRegisterClass(MVT::i1, &X86::VK1RegClass);
1406 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1407 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1409 for (MVT VT : MVT::fp_vector_valuetypes())
1410 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal);
1412 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
1413 setOperationAction(ISD::SETCC, MVT::i1, Custom);
1414 setOperationAction(ISD::XOR, MVT::i1, Legal);
1415 setOperationAction(ISD::OR, MVT::i1, Legal);
1416 setOperationAction(ISD::AND, MVT::i1, Legal);
1417 setOperationAction(ISD::LOAD, MVT::v16f32, Legal);
1418 setOperationAction(ISD::LOAD, MVT::v8f64, Legal);
1419 setOperationAction(ISD::LOAD, MVT::v8i64, Legal);
1420 setOperationAction(ISD::LOAD, MVT::v16i32, Legal);
1421 setOperationAction(ISD::LOAD, MVT::v16i1, Legal);
1423 setOperationAction(ISD::FADD, MVT::v16f32, Legal);
1424 setOperationAction(ISD::FSUB, MVT::v16f32, Legal);
1425 setOperationAction(ISD::FMUL, MVT::v16f32, Legal);
1426 setOperationAction(ISD::FDIV, MVT::v16f32, Legal);
1427 setOperationAction(ISD::FSQRT, MVT::v16f32, Legal);
1428 setOperationAction(ISD::FNEG, MVT::v16f32, Custom);
1430 setOperationAction(ISD::FADD, MVT::v8f64, Legal);
1431 setOperationAction(ISD::FSUB, MVT::v8f64, Legal);
1432 setOperationAction(ISD::FMUL, MVT::v8f64, Legal);
1433 setOperationAction(ISD::FDIV, MVT::v8f64, Legal);
1434 setOperationAction(ISD::FSQRT, MVT::v8f64, Legal);
1435 setOperationAction(ISD::FNEG, MVT::v8f64, Custom);
1436 setOperationAction(ISD::FMA, MVT::v8f64, Legal);
1437 setOperationAction(ISD::FMA, MVT::v16f32, Legal);
1439 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
1440 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
1441 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
1442 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
1443 if (Subtarget->is64Bit()) {
1444 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Legal);
1445 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Legal);
1446 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Legal);
1447 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Legal);
1449 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal);
1450 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal);
1451 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal);
1452 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
1453 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal);
1454 setOperationAction(ISD::SINT_TO_FP, MVT::v8i1, Custom);
1455 setOperationAction(ISD::SINT_TO_FP, MVT::v16i1, Custom);
1456 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Promote);
1457 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Promote);
1458 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal);
1459 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal);
1460 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
1461 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal);
1462 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal);
1464 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom);
1465 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1466 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1467 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom);
1468 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom);
1469 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1470 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1471 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1472 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1473 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1474 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom);
1475 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom);
1476 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
1478 setOperationAction(ISD::FFLOOR, MVT::v16f32, Legal);
1479 setOperationAction(ISD::FFLOOR, MVT::v8f64, Legal);
1480 setOperationAction(ISD::FCEIL, MVT::v16f32, Legal);
1481 setOperationAction(ISD::FCEIL, MVT::v8f64, Legal);
1482 setOperationAction(ISD::FTRUNC, MVT::v16f32, Legal);
1483 setOperationAction(ISD::FTRUNC, MVT::v8f64, Legal);
1484 setOperationAction(ISD::FRINT, MVT::v16f32, Legal);
1485 setOperationAction(ISD::FRINT, MVT::v8f64, Legal);
1486 setOperationAction(ISD::FNEARBYINT, MVT::v16f32, Legal);
1487 setOperationAction(ISD::FNEARBYINT, MVT::v8f64, Legal);
1489 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom);
1490 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom);
1491 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom);
1492 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom);
1493 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom);
1494 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Legal);
1496 setOperationAction(ISD::SETCC, MVT::v16i1, Custom);
1497 setOperationAction(ISD::SETCC, MVT::v8i1, Custom);
1499 setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1501 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom);
1502 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom);
1503 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom);
1504 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom);
1505 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom);
1506 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom);
1507 setOperationAction(ISD::SELECT, MVT::v8f64, Custom);
1508 setOperationAction(ISD::SELECT, MVT::v8i64, Custom);
1509 setOperationAction(ISD::SELECT, MVT::v16f32, Custom);
1511 setOperationAction(ISD::ADD, MVT::v8i64, Legal);
1512 setOperationAction(ISD::ADD, MVT::v16i32, Legal);
1514 setOperationAction(ISD::SUB, MVT::v8i64, Legal);
1515 setOperationAction(ISD::SUB, MVT::v16i32, Legal);
1517 setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1519 setOperationAction(ISD::SRL, MVT::v8i64, Custom);
1520 setOperationAction(ISD::SRL, MVT::v16i32, Custom);
1522 setOperationAction(ISD::SHL, MVT::v8i64, Custom);
1523 setOperationAction(ISD::SHL, MVT::v16i32, Custom);
1525 setOperationAction(ISD::SRA, MVT::v8i64, Custom);
1526 setOperationAction(ISD::SRA, MVT::v16i32, Custom);
1528 setOperationAction(ISD::AND, MVT::v8i64, Legal);
1529 setOperationAction(ISD::OR, MVT::v8i64, Legal);
1530 setOperationAction(ISD::XOR, MVT::v8i64, Legal);
1531 setOperationAction(ISD::AND, MVT::v16i32, Legal);
1532 setOperationAction(ISD::OR, MVT::v16i32, Legal);
1533 setOperationAction(ISD::XOR, MVT::v16i32, Legal);
1535 if (Subtarget->hasCDI()) {
1536 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal);
1537 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal);
1540 // Custom lower several nodes.
1541 for (MVT VT : MVT::vector_valuetypes()) {
1542 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
1543 // Extract subvector is special because the value type
1544 // (result) is 256/128-bit but the source is 512-bit wide.
1545 if (VT.is128BitVector() || VT.is256BitVector()) {
1546 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1548 if (VT.getVectorElementType() == MVT::i1)
1549 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1551 // Do not attempt to custom lower other non-512-bit vectors
1552 if (!VT.is512BitVector())
1555 if ( EltSize >= 32) {
1556 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1557 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1558 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1559 setOperationAction(ISD::VSELECT, VT, Legal);
1560 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1561 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1562 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1563 setOperationAction(ISD::MLOAD, VT, Legal);
1564 setOperationAction(ISD::MSTORE, VT, Legal);
1567 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) {
1568 MVT VT = (MVT::SimpleValueType)i;
1570 // Do not attempt to promote non-512-bit vectors.
1571 if (!VT.is512BitVector())
1574 setOperationAction(ISD::SELECT, VT, Promote);
1575 AddPromotedToType (ISD::SELECT, VT, MVT::v8i64);
1579 if (!TM.Options.UseSoftFloat && Subtarget->hasBWI()) {
1580 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1581 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1583 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
1584 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
1586 setOperationAction(ISD::LOAD, MVT::v32i16, Legal);
1587 setOperationAction(ISD::LOAD, MVT::v64i8, Legal);
1588 setOperationAction(ISD::SETCC, MVT::v32i1, Custom);
1589 setOperationAction(ISD::SETCC, MVT::v64i1, Custom);
1590 setOperationAction(ISD::ADD, MVT::v32i16, Legal);
1591 setOperationAction(ISD::ADD, MVT::v64i8, Legal);
1592 setOperationAction(ISD::SUB, MVT::v32i16, Legal);
1593 setOperationAction(ISD::SUB, MVT::v64i8, Legal);
1594 setOperationAction(ISD::MUL, MVT::v32i16, Legal);
1596 for (int i = MVT::v32i8; i != MVT::v8i64; ++i) {
1597 const MVT VT = (MVT::SimpleValueType)i;
1599 const unsigned EltSize = VT.getVectorElementType().getSizeInBits();
1601 // Do not attempt to promote non-512-bit vectors.
1602 if (!VT.is512BitVector())
1606 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1607 setOperationAction(ISD::VSELECT, VT, Legal);
1612 if (!TM.Options.UseSoftFloat && Subtarget->hasVLX()) {
1613 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1614 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1616 setOperationAction(ISD::SETCC, MVT::v4i1, Custom);
1617 setOperationAction(ISD::SETCC, MVT::v2i1, Custom);
1618 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Legal);
1620 setOperationAction(ISD::AND, MVT::v8i32, Legal);
1621 setOperationAction(ISD::OR, MVT::v8i32, Legal);
1622 setOperationAction(ISD::XOR, MVT::v8i32, Legal);
1623 setOperationAction(ISD::AND, MVT::v4i32, Legal);
1624 setOperationAction(ISD::OR, MVT::v4i32, Legal);
1625 setOperationAction(ISD::XOR, MVT::v4i32, Legal);
1628 // SIGN_EXTEND_INREGs are evaluated by the extend type. Handle the expansion
1629 // of this type with custom code.
1630 for (MVT VT : MVT::vector_valuetypes())
1631 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
1633 // We want to custom lower some of our intrinsics.
1634 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1635 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
1636 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1637 if (!Subtarget->is64Bit())
1638 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
1640 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
1641 // handle type legalization for these operations here.
1643 // FIXME: We really should do custom legalization for addition and
1644 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
1645 // than generic legalization for 64-bit multiplication-with-overflow, though.
1646 for (unsigned i = 0, e = 3+Subtarget->is64Bit(); i != e; ++i) {
1647 // Add/Sub/Mul with overflow operations are custom lowered.
1649 setOperationAction(ISD::SADDO, VT, Custom);
1650 setOperationAction(ISD::UADDO, VT, Custom);
1651 setOperationAction(ISD::SSUBO, VT, Custom);
1652 setOperationAction(ISD::USUBO, VT, Custom);
1653 setOperationAction(ISD::SMULO, VT, Custom);
1654 setOperationAction(ISD::UMULO, VT, Custom);
1658 if (!Subtarget->is64Bit()) {
1659 // These libcalls are not available in 32-bit.
1660 setLibcallName(RTLIB::SHL_I128, nullptr);
1661 setLibcallName(RTLIB::SRL_I128, nullptr);
1662 setLibcallName(RTLIB::SRA_I128, nullptr);
1665 // Combine sin / cos into one node or libcall if possible.
1666 if (Subtarget->hasSinCos()) {
1667 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
1668 setLibcallName(RTLIB::SINCOS_F64, "sincos");
1669 if (Subtarget->isTargetDarwin()) {
1670 // For MacOSX, we don't want the normal expansion of a libcall to sincos.
1671 // We want to issue a libcall to __sincos_stret to avoid memory traffic.
1672 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1673 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1677 if (Subtarget->isTargetWin64()) {
1678 setOperationAction(ISD::SDIV, MVT::i128, Custom);
1679 setOperationAction(ISD::UDIV, MVT::i128, Custom);
1680 setOperationAction(ISD::SREM, MVT::i128, Custom);
1681 setOperationAction(ISD::UREM, MVT::i128, Custom);
1682 setOperationAction(ISD::SDIVREM, MVT::i128, Custom);
1683 setOperationAction(ISD::UDIVREM, MVT::i128, Custom);
1686 // We have target-specific dag combine patterns for the following nodes:
1687 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
1688 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
1689 setTargetDAGCombine(ISD::BITCAST);
1690 setTargetDAGCombine(ISD::VSELECT);
1691 setTargetDAGCombine(ISD::SELECT);
1692 setTargetDAGCombine(ISD::SHL);
1693 setTargetDAGCombine(ISD::SRA);
1694 setTargetDAGCombine(ISD::SRL);
1695 setTargetDAGCombine(ISD::OR);
1696 setTargetDAGCombine(ISD::AND);
1697 setTargetDAGCombine(ISD::ADD);
1698 setTargetDAGCombine(ISD::FADD);
1699 setTargetDAGCombine(ISD::FSUB);
1700 setTargetDAGCombine(ISD::FMA);
1701 setTargetDAGCombine(ISD::SUB);
1702 setTargetDAGCombine(ISD::LOAD);
1703 setTargetDAGCombine(ISD::MLOAD);
1704 setTargetDAGCombine(ISD::STORE);
1705 setTargetDAGCombine(ISD::MSTORE);
1706 setTargetDAGCombine(ISD::ZERO_EXTEND);
1707 setTargetDAGCombine(ISD::ANY_EXTEND);
1708 setTargetDAGCombine(ISD::SIGN_EXTEND);
1709 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1710 setTargetDAGCombine(ISD::TRUNCATE);
1711 setTargetDAGCombine(ISD::SINT_TO_FP);
1712 setTargetDAGCombine(ISD::SETCC);
1713 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
1714 setTargetDAGCombine(ISD::BUILD_VECTOR);
1715 setTargetDAGCombine(ISD::MUL);
1716 setTargetDAGCombine(ISD::XOR);
1718 computeRegisterProperties();
1720 // On Darwin, -Os means optimize for size without hurting performance,
1721 // do not reduce the limit.
1722 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
1723 MaxStoresPerMemsetOptSize = Subtarget->isTargetDarwin() ? 16 : 8;
1724 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
1725 MaxStoresPerMemcpyOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
1726 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
1727 MaxStoresPerMemmoveOptSize = Subtarget->isTargetDarwin() ? 8 : 4;
1728 setPrefLoopAlignment(4); // 2^4 bytes.
1730 // Predictable cmov don't hurt on atom because it's in-order.
1731 PredictableSelectIsExpensive = !Subtarget->isAtom();
1732 EnableExtLdPromotion = true;
1733 setPrefFunctionAlignment(4); // 2^4 bytes.
1735 verifyIntrinsicTables();
1738 // This has so far only been implemented for 64-bit MachO.
1739 bool X86TargetLowering::useLoadStackGuardNode() const {
1740 return Subtarget->isTargetMachO() && Subtarget->is64Bit();
1743 TargetLoweringBase::LegalizeTypeAction
1744 X86TargetLowering::getPreferredVectorAction(EVT VT) const {
1745 if (ExperimentalVectorWideningLegalization &&
1746 VT.getVectorNumElements() != 1 &&
1747 VT.getVectorElementType().getSimpleVT() != MVT::i1)
1748 return TypeWidenVector;
1750 return TargetLoweringBase::getPreferredVectorAction(VT);
1753 EVT X86TargetLowering::getSetCCResultType(LLVMContext &, EVT VT) const {
1755 return Subtarget->hasAVX512() ? MVT::i1: MVT::i8;
1757 const unsigned NumElts = VT.getVectorNumElements();
1758 const EVT EltVT = VT.getVectorElementType();
1759 if (VT.is512BitVector()) {
1760 if (Subtarget->hasAVX512())
1761 if (EltVT == MVT::i32 || EltVT == MVT::i64 ||
1762 EltVT == MVT::f32 || EltVT == MVT::f64)
1764 case 8: return MVT::v8i1;
1765 case 16: return MVT::v16i1;
1767 if (Subtarget->hasBWI())
1768 if (EltVT == MVT::i8 || EltVT == MVT::i16)
1770 case 32: return MVT::v32i1;
1771 case 64: return MVT::v64i1;
1775 if (VT.is256BitVector() || VT.is128BitVector()) {
1776 if (Subtarget->hasVLX())
1777 if (EltVT == MVT::i32 || EltVT == MVT::i64 ||
1778 EltVT == MVT::f32 || EltVT == MVT::f64)
1780 case 2: return MVT::v2i1;
1781 case 4: return MVT::v4i1;
1782 case 8: return MVT::v8i1;
1784 if (Subtarget->hasBWI() && Subtarget->hasVLX())
1785 if (EltVT == MVT::i8 || EltVT == MVT::i16)
1787 case 8: return MVT::v8i1;
1788 case 16: return MVT::v16i1;
1789 case 32: return MVT::v32i1;
1793 return VT.changeVectorElementTypeToInteger();
1796 /// Helper for getByValTypeAlignment to determine
1797 /// the desired ByVal argument alignment.
1798 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
1801 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1802 if (VTy->getBitWidth() == 128)
1804 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
1805 unsigned EltAlign = 0;
1806 getMaxByValAlign(ATy->getElementType(), EltAlign);
1807 if (EltAlign > MaxAlign)
1808 MaxAlign = EltAlign;
1809 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
1810 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1811 unsigned EltAlign = 0;
1812 getMaxByValAlign(STy->getElementType(i), EltAlign);
1813 if (EltAlign > MaxAlign)
1814 MaxAlign = EltAlign;
1821 /// Return the desired alignment for ByVal aggregate
1822 /// function arguments in the caller parameter area. For X86, aggregates
1823 /// that contain SSE vectors are placed at 16-byte boundaries while the rest
1824 /// are at 4-byte boundaries.
1825 unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty) const {
1826 if (Subtarget->is64Bit()) {
1827 // Max of 8 and alignment of type.
1828 unsigned TyAlign = TD->getABITypeAlignment(Ty);
1835 if (Subtarget->hasSSE1())
1836 getMaxByValAlign(Ty, Align);
1840 /// Returns the target specific optimal type for load
1841 /// and store operations as a result of memset, memcpy, and memmove
1842 /// lowering. If DstAlign is zero that means it's safe to destination
1843 /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
1844 /// means there isn't a need to check it against alignment requirement,
1845 /// probably because the source does not need to be loaded. If 'IsMemset' is
1846 /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that
1847 /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy
1848 /// source is constant so it does not need to be loaded.
1849 /// It returns EVT::Other if the type should be determined using generic
1850 /// target-independent logic.
1852 X86TargetLowering::getOptimalMemOpType(uint64_t Size,
1853 unsigned DstAlign, unsigned SrcAlign,
1854 bool IsMemset, bool ZeroMemset,
1856 MachineFunction &MF) const {
1857 const Function *F = MF.getFunction();
1858 if ((!IsMemset || ZeroMemset) &&
1859 !F->hasFnAttribute(Attribute::NoImplicitFloat)) {
1861 (Subtarget->isUnalignedMemAccessFast() ||
1862 ((DstAlign == 0 || DstAlign >= 16) &&
1863 (SrcAlign == 0 || SrcAlign >= 16)))) {
1865 if (Subtarget->hasInt256())
1867 if (Subtarget->hasFp256())
1870 if (Subtarget->hasSSE2())
1872 if (Subtarget->hasSSE1())
1874 } else if (!MemcpyStrSrc && Size >= 8 &&
1875 !Subtarget->is64Bit() &&
1876 Subtarget->hasSSE2()) {
1877 // Do not use f64 to lower memcpy if source is string constant. It's
1878 // better to use i32 to avoid the loads.
1882 if (Subtarget->is64Bit() && Size >= 8)
1887 bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
1889 return X86ScalarSSEf32;
1890 else if (VT == MVT::f64)
1891 return X86ScalarSSEf64;
1896 X86TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
1901 *Fast = Subtarget->isUnalignedMemAccessFast();
1905 /// Return the entry encoding for a jump table in the
1906 /// current function. The returned value is a member of the
1907 /// MachineJumpTableInfo::JTEntryKind enum.
1908 unsigned X86TargetLowering::getJumpTableEncoding() const {
1909 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
1911 if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1912 Subtarget->isPICStyleGOT())
1913 return MachineJumpTableInfo::EK_Custom32;
1915 // Otherwise, use the normal jump table encoding heuristics.
1916 return TargetLowering::getJumpTableEncoding();
1920 X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
1921 const MachineBasicBlock *MBB,
1922 unsigned uid,MCContext &Ctx) const{
1923 assert(MBB->getParent()->getTarget().getRelocationModel() == Reloc::PIC_ &&
1924 Subtarget->isPICStyleGOT());
1925 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
1927 return MCSymbolRefExpr::Create(MBB->getSymbol(),
1928 MCSymbolRefExpr::VK_GOTOFF, Ctx);
1931 /// Returns relocation base for the given PIC jumptable.
1932 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
1933 SelectionDAG &DAG) const {
1934 if (!Subtarget->is64Bit())
1935 // This doesn't have SDLoc associated with it, but is not really the
1936 // same as a Register.
1937 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy());
1941 /// This returns the relocation base for the given PIC jumptable,
1942 /// the same as getPICJumpTableRelocBase, but as an MCExpr.
1943 const MCExpr *X86TargetLowering::
1944 getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
1945 MCContext &Ctx) const {
1946 // X86-64 uses RIP relative addressing based on the jump table label.
1947 if (Subtarget->isPICStyleRIPRel())
1948 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
1950 // Otherwise, the reference is relative to the PIC base.
1951 return MCSymbolRefExpr::Create(MF->getPICBaseSymbol(), Ctx);
1954 // FIXME: Why this routine is here? Move to RegInfo!
1955 std::pair<const TargetRegisterClass*, uint8_t>
1956 X86TargetLowering::findRepresentativeClass(MVT VT) const{
1957 const TargetRegisterClass *RRC = nullptr;
1959 switch (VT.SimpleTy) {
1961 return TargetLowering::findRepresentativeClass(VT);
1962 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:
1963 RRC = Subtarget->is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
1966 RRC = &X86::VR64RegClass;
1968 case MVT::f32: case MVT::f64:
1969 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1970 case MVT::v4f32: case MVT::v2f64:
1971 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32:
1973 RRC = &X86::VR128RegClass;
1976 return std::make_pair(RRC, Cost);
1979 bool X86TargetLowering::getStackCookieLocation(unsigned &AddressSpace,
1980 unsigned &Offset) const {
1981 if (!Subtarget->isTargetLinux())
1984 if (Subtarget->is64Bit()) {
1985 // %fs:0x28, unless we're using a Kernel code model, in which case it's %gs:
1987 if (getTargetMachine().getCodeModel() == CodeModel::Kernel)
1999 bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
2000 unsigned DestAS) const {
2001 assert(SrcAS != DestAS && "Expected different address spaces!");
2003 return SrcAS < 256 && DestAS < 256;
2006 //===----------------------------------------------------------------------===//
2007 // Return Value Calling Convention Implementation
2008 //===----------------------------------------------------------------------===//
2010 #include "X86GenCallingConv.inc"
2013 X86TargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2014 MachineFunction &MF, bool isVarArg,
2015 const SmallVectorImpl<ISD::OutputArg> &Outs,
2016 LLVMContext &Context) const {
2017 SmallVector<CCValAssign, 16> RVLocs;
2018 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2019 return CCInfo.CheckReturn(Outs, RetCC_X86);
2022 const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {
2023 static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };
2028 X86TargetLowering::LowerReturn(SDValue Chain,
2029 CallingConv::ID CallConv, bool isVarArg,
2030 const SmallVectorImpl<ISD::OutputArg> &Outs,
2031 const SmallVectorImpl<SDValue> &OutVals,
2032 SDLoc dl, SelectionDAG &DAG) const {
2033 MachineFunction &MF = DAG.getMachineFunction();
2034 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2036 SmallVector<CCValAssign, 16> RVLocs;
2037 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
2038 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
2041 SmallVector<SDValue, 6> RetOps;
2042 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2043 // Operand #1 = Bytes To Pop
2044 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(),
2047 // Copy the result values into the output registers.
2048 for (unsigned i = 0; i != RVLocs.size(); ++i) {
2049 CCValAssign &VA = RVLocs[i];
2050 assert(VA.isRegLoc() && "Can only return in registers!");
2051 SDValue ValToCopy = OutVals[i];
2052 EVT ValVT = ValToCopy.getValueType();
2054 // Promote values to the appropriate types.
2055 if (VA.getLocInfo() == CCValAssign::SExt)
2056 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
2057 else if (VA.getLocInfo() == CCValAssign::ZExt)
2058 ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);
2059 else if (VA.getLocInfo() == CCValAssign::AExt)
2060 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
2061 else if (VA.getLocInfo() == CCValAssign::BCvt)
2062 ValToCopy = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), ValToCopy);
2064 assert(VA.getLocInfo() != CCValAssign::FPExt &&
2065 "Unexpected FP-extend for return value.");
2067 // If this is x86-64, and we disabled SSE, we can't return FP values,
2068 // or SSE or MMX vectors.
2069 if ((ValVT == MVT::f32 || ValVT == MVT::f64 ||
2070 VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) &&
2071 (Subtarget->is64Bit() && !Subtarget->hasSSE1())) {
2072 report_fatal_error("SSE register return with SSE disabled");
2074 // Likewise we can't return F64 values with SSE1 only. gcc does so, but
2075 // llvm-gcc has never done it right and no one has noticed, so this
2076 // should be OK for now.
2077 if (ValVT == MVT::f64 &&
2078 (Subtarget->is64Bit() && !Subtarget->hasSSE2()))
2079 report_fatal_error("SSE2 register return with SSE2 disabled");
2081 // Returns in ST0/ST1 are handled specially: these are pushed as operands to
2082 // the RET instruction and handled by the FP Stackifier.
2083 if (VA.getLocReg() == X86::FP0 ||
2084 VA.getLocReg() == X86::FP1) {
2085 // If this is a copy from an xmm register to ST(0), use an FPExtend to
2086 // change the value to the FP stack register class.
2087 if (isScalarFPTypeInSSEReg(VA.getValVT()))
2088 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
2089 RetOps.push_back(ValToCopy);
2090 // Don't emit a copytoreg.
2094 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
2095 // which is returned in RAX / RDX.
2096 if (Subtarget->is64Bit()) {
2097 if (ValVT == MVT::x86mmx) {
2098 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
2099 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy);
2100 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
2102 // If we don't have SSE2 available, convert to v4f32 so the generated
2103 // register is legal.
2104 if (!Subtarget->hasSSE2())
2105 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy);
2110 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag);
2111 Flag = Chain.getValue(1);
2112 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2115 // The x86-64 ABIs require that for returning structs by value we copy
2116 // the sret argument into %rax/%eax (depending on ABI) for the return.
2117 // Win32 requires us to put the sret argument to %eax as well.
2118 // We saved the argument into a virtual register in the entry block,
2119 // so now we copy the value out and into %rax/%eax.
2121 // Checking Function.hasStructRetAttr() here is insufficient because the IR
2122 // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
2123 // false, then an sret argument may be implicitly inserted in the SelDAG. In
2124 // either case FuncInfo->setSRetReturnReg() will have been called.
2125 if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
2126 assert((Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC()) &&
2127 "No need for an sret register");
2128 SDValue Val = DAG.getCopyFromReg(Chain, dl, SRetReg, getPointerTy());
2131 = (Subtarget->is64Bit() && !Subtarget->isTarget64BitILP32()) ?
2132 X86::RAX : X86::EAX;
2133 Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
2134 Flag = Chain.getValue(1);
2136 // RAX/EAX now acts like a return value.
2137 RetOps.push_back(DAG.getRegister(RetValReg, getPointerTy()));
2140 RetOps[0] = Chain; // Update chain.
2142 // Add the flag if we have it.
2144 RetOps.push_back(Flag);
2146 return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps);
2149 bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2150 if (N->getNumValues() != 1)
2152 if (!N->hasNUsesOfValue(1, 0))
2155 SDValue TCChain = Chain;
2156 SDNode *Copy = *N->use_begin();
2157 if (Copy->getOpcode() == ISD::CopyToReg) {
2158 // If the copy has a glue operand, we conservatively assume it isn't safe to
2159 // perform a tail call.
2160 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2162 TCChain = Copy->getOperand(0);
2163 } else if (Copy->getOpcode() != ISD::FP_EXTEND)
2166 bool HasRet = false;
2167 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
2169 if (UI->getOpcode() != X86ISD::RET_FLAG)
2171 // If we are returning more than one value, we can definitely
2172 // not make a tail call see PR19530
2173 if (UI->getNumOperands() > 4)
2175 if (UI->getNumOperands() == 4 &&
2176 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue)
2189 X86TargetLowering::getTypeForExtArgOrReturn(LLVMContext &Context, EVT VT,
2190 ISD::NodeType ExtendKind) const {
2192 // TODO: Is this also valid on 32-bit?
2193 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND)
2194 ReturnMVT = MVT::i8;
2196 ReturnMVT = MVT::i32;
2198 EVT MinVT = getRegisterType(Context, ReturnMVT);
2199 return VT.bitsLT(MinVT) ? MinVT : VT;
2202 /// Lower the result values of a call into the
2203 /// appropriate copies out of appropriate physical registers.
2206 X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
2207 CallingConv::ID CallConv, bool isVarArg,
2208 const SmallVectorImpl<ISD::InputArg> &Ins,
2209 SDLoc dl, SelectionDAG &DAG,
2210 SmallVectorImpl<SDValue> &InVals) const {
2212 // Assign locations to each value returned by this call.
2213 SmallVector<CCValAssign, 16> RVLocs;
2214 bool Is64Bit = Subtarget->is64Bit();
2215 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2217 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
2219 // Copy all of the result registers out of their specified physreg.
2220 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
2221 CCValAssign &VA = RVLocs[i];
2222 EVT CopyVT = VA.getValVT();
2224 // If this is x86-64, and we disabled SSE, we can't return FP values
2225 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
2226 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
2227 report_fatal_error("SSE register return with SSE disabled");
2230 // If we prefer to use the value in xmm registers, copy it out as f80 and
2231 // use a truncate to move it from fp stack reg to xmm reg.
2232 if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
2233 isScalarFPTypeInSSEReg(VA.getValVT()))
2236 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
2237 CopyVT, InFlag).getValue(1);
2238 SDValue Val = Chain.getValue(0);
2240 if (CopyVT != VA.getValVT())
2241 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
2242 // This truncation won't change the value.
2243 DAG.getIntPtrConstant(1));
2245 InFlag = Chain.getValue(2);
2246 InVals.push_back(Val);
2252 //===----------------------------------------------------------------------===//
2253 // C & StdCall & Fast Calling Convention implementation
2254 //===----------------------------------------------------------------------===//
2255 // StdCall calling convention seems to be standard for many Windows' API
2256 // routines and around. It differs from C calling convention just a little:
2257 // callee should clean up the stack, not caller. Symbols should be also
2258 // decorated in some fancy way :) It doesn't support any vector arguments.
2259 // For info on fast calling convention see Fast Calling Convention (tail call)
2260 // implementation LowerX86_32FastCCCallTo.
2262 /// CallIsStructReturn - Determines whether a call uses struct return
2264 enum StructReturnType {
2269 static StructReturnType
2270 callIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) {
2272 return NotStructReturn;
2274 const ISD::ArgFlagsTy &Flags = Outs[0].Flags;
2275 if (!Flags.isSRet())
2276 return NotStructReturn;
2277 if (Flags.isInReg())
2278 return RegStructReturn;
2279 return StackStructReturn;
2282 /// Determines whether a function uses struct return semantics.
2283 static StructReturnType
2284 argsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) {
2286 return NotStructReturn;
2288 const ISD::ArgFlagsTy &Flags = Ins[0].Flags;
2289 if (!Flags.isSRet())
2290 return NotStructReturn;
2291 if (Flags.isInReg())
2292 return RegStructReturn;
2293 return StackStructReturn;
2296 /// Make a copy of an aggregate at address specified by "Src" to address
2297 /// "Dst" with size and alignment information specified by the specific
2298 /// parameter attribute. The copy will be passed as a byval function parameter.
2300 CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
2301 ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
2303 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), MVT::i32);
2305 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
2306 /*isVolatile*/false, /*AlwaysInline=*/true,
2307 MachinePointerInfo(), MachinePointerInfo());
2310 /// Return true if the calling convention is one that
2311 /// supports tail call optimization.
2312 static bool IsTailCallConvention(CallingConv::ID CC) {
2313 return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
2314 CC == CallingConv::HiPE);
2317 /// \brief Return true if the calling convention is a C calling convention.
2318 static bool IsCCallConvention(CallingConv::ID CC) {
2319 return (CC == CallingConv::C || CC == CallingConv::X86_64_Win64 ||
2320 CC == CallingConv::X86_64_SysV);
2323 bool X86TargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
2324 if (!CI->isTailCall() || getTargetMachine().Options.DisableTailCalls)
2328 CallingConv::ID CalleeCC = CS.getCallingConv();
2329 if (!IsTailCallConvention(CalleeCC) && !IsCCallConvention(CalleeCC))
2335 /// Return true if the function is being made into
2336 /// a tailcall target by changing its ABI.
2337 static bool FuncIsMadeTailCallSafe(CallingConv::ID CC,
2338 bool GuaranteedTailCallOpt) {
2339 return GuaranteedTailCallOpt && IsTailCallConvention(CC);
2343 X86TargetLowering::LowerMemArgument(SDValue Chain,
2344 CallingConv::ID CallConv,
2345 const SmallVectorImpl<ISD::InputArg> &Ins,
2346 SDLoc dl, SelectionDAG &DAG,
2347 const CCValAssign &VA,
2348 MachineFrameInfo *MFI,
2350 // Create the nodes corresponding to a load from this parameter slot.
2351 ISD::ArgFlagsTy Flags = Ins[i].Flags;
2352 bool AlwaysUseMutable = FuncIsMadeTailCallSafe(
2353 CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt);
2354 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
2357 // If value is passed by pointer we have address passed instead of the value
2359 if (VA.getLocInfo() == CCValAssign::Indirect)
2360 ValVT = VA.getLocVT();
2362 ValVT = VA.getValVT();
2364 // FIXME: For now, all byval parameter objects are marked mutable. This can be
2365 // changed with more analysis.
2366 // In case of tail call optimization mark all arguments mutable. Since they
2367 // could be overwritten by lowering of arguments in case of a tail call.
2368 if (Flags.isByVal()) {
2369 unsigned Bytes = Flags.getByValSize();
2370 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
2371 int FI = MFI->CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable);
2372 return DAG.getFrameIndex(FI, getPointerTy());
2374 int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
2375 VA.getLocMemOffset(), isImmutable);
2376 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
2377 return DAG.getLoad(ValVT, dl, Chain, FIN,
2378 MachinePointerInfo::getFixedStack(FI),
2379 false, false, false, 0);
2383 // FIXME: Get this from tablegen.
2384 static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
2385 const X86Subtarget *Subtarget) {
2386 assert(Subtarget->is64Bit());
2388 if (Subtarget->isCallingConvWin64(CallConv)) {
2389 static const MCPhysReg GPR64ArgRegsWin64[] = {
2390 X86::RCX, X86::RDX, X86::R8, X86::R9
2392 return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
2395 static const MCPhysReg GPR64ArgRegs64Bit[] = {
2396 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
2398 return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
2401 // FIXME: Get this from tablegen.
2402 static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
2403 CallingConv::ID CallConv,
2404 const X86Subtarget *Subtarget) {
2405 assert(Subtarget->is64Bit());
2406 if (Subtarget->isCallingConvWin64(CallConv)) {
2407 // The XMM registers which might contain var arg parameters are shadowed
2408 // in their paired GPR. So we only need to save the GPR to their home
2410 // TODO: __vectorcall will change this.
2414 const Function *Fn = MF.getFunction();
2415 bool NoImplicitFloatOps = Fn->hasFnAttribute(Attribute::NoImplicitFloat);
2416 assert(!(MF.getTarget().Options.UseSoftFloat && NoImplicitFloatOps) &&
2417 "SSE register cannot be used when SSE is disabled!");
2418 if (MF.getTarget().Options.UseSoftFloat || NoImplicitFloatOps ||
2419 !Subtarget->hasSSE1())
2420 // Kernel mode asks for SSE to be disabled, so there are no XMM argument
2424 static const MCPhysReg XMMArgRegs64Bit[] = {
2425 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2426 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
2428 return makeArrayRef(std::begin(XMMArgRegs64Bit), std::end(XMMArgRegs64Bit));
2432 X86TargetLowering::LowerFormalArguments(SDValue Chain,
2433 CallingConv::ID CallConv,
2435 const SmallVectorImpl<ISD::InputArg> &Ins,
2438 SmallVectorImpl<SDValue> &InVals)
2440 MachineFunction &MF = DAG.getMachineFunction();
2441 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2443 const Function* Fn = MF.getFunction();
2444 if (Fn->hasExternalLinkage() &&
2445 Subtarget->isTargetCygMing() &&
2446 Fn->getName() == "main")
2447 FuncInfo->setForceFramePointer(true);
2449 MachineFrameInfo *MFI = MF.getFrameInfo();
2450 bool Is64Bit = Subtarget->is64Bit();
2451 bool IsWin64 = Subtarget->isCallingConvWin64(CallConv);
2453 assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
2454 "Var args not supported with calling convention fastcc, ghc or hipe");
2456 // Assign locations to all of the incoming arguments.
2457 SmallVector<CCValAssign, 16> ArgLocs;
2458 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
2460 // Allocate shadow area for Win64
2462 CCInfo.AllocateStack(32, 8);
2464 CCInfo.AnalyzeFormalArguments(Ins, CC_X86);
2466 unsigned LastVal = ~0U;
2468 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2469 CCValAssign &VA = ArgLocs[i];
2470 // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
2472 assert(VA.getValNo() != LastVal &&
2473 "Don't support value assigned to multiple locs yet");
2475 LastVal = VA.getValNo();
2477 if (VA.isRegLoc()) {
2478 EVT RegVT = VA.getLocVT();
2479 const TargetRegisterClass *RC;
2480 if (RegVT == MVT::i32)
2481 RC = &X86::GR32RegClass;
2482 else if (Is64Bit && RegVT == MVT::i64)
2483 RC = &X86::GR64RegClass;
2484 else if (RegVT == MVT::f32)
2485 RC = &X86::FR32RegClass;
2486 else if (RegVT == MVT::f64)
2487 RC = &X86::FR64RegClass;
2488 else if (RegVT.is512BitVector())
2489 RC = &X86::VR512RegClass;
2490 else if (RegVT.is256BitVector())
2491 RC = &X86::VR256RegClass;
2492 else if (RegVT.is128BitVector())
2493 RC = &X86::VR128RegClass;
2494 else if (RegVT == MVT::x86mmx)
2495 RC = &X86::VR64RegClass;
2496 else if (RegVT == MVT::i1)
2497 RC = &X86::VK1RegClass;
2498 else if (RegVT == MVT::v8i1)
2499 RC = &X86::VK8RegClass;
2500 else if (RegVT == MVT::v16i1)
2501 RC = &X86::VK16RegClass;
2502 else if (RegVT == MVT::v32i1)
2503 RC = &X86::VK32RegClass;
2504 else if (RegVT == MVT::v64i1)
2505 RC = &X86::VK64RegClass;
2507 llvm_unreachable("Unknown argument type!");
2509 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
2510 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
2512 // If this is an 8 or 16-bit value, it is really passed promoted to 32
2513 // bits. Insert an assert[sz]ext to capture this, then truncate to the
2515 if (VA.getLocInfo() == CCValAssign::SExt)
2516 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
2517 DAG.getValueType(VA.getValVT()));
2518 else if (VA.getLocInfo() == CCValAssign::ZExt)
2519 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
2520 DAG.getValueType(VA.getValVT()));
2521 else if (VA.getLocInfo() == CCValAssign::BCvt)
2522 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
2524 if (VA.isExtInLoc()) {
2525 // Handle MMX values passed in XMM regs.
2526 if (RegVT.isVector())
2527 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
2529 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
2532 assert(VA.isMemLoc());
2533 ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i);
2536 // If value is passed via pointer - do a load.
2537 if (VA.getLocInfo() == CCValAssign::Indirect)
2538 ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue,
2539 MachinePointerInfo(), false, false, false, 0);
2541 InVals.push_back(ArgValue);
2544 if (Subtarget->is64Bit() || Subtarget->isTargetKnownWindowsMSVC()) {
2545 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2546 // The x86-64 ABIs require that for returning structs by value we copy
2547 // the sret argument into %rax/%eax (depending on ABI) for the return.
2548 // Win32 requires us to put the sret argument to %eax as well.
2549 // Save the argument into a virtual register so that we can access it
2550 // from the return points.
2551 if (Ins[i].Flags.isSRet()) {
2552 unsigned Reg = FuncInfo->getSRetReturnReg();
2554 MVT PtrTy = getPointerTy();
2555 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
2556 FuncInfo->setSRetReturnReg(Reg);
2558 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[i]);
2559 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
2565 unsigned StackSize = CCInfo.getNextStackOffset();
2566 // Align stack specially for tail calls.
2567 if (FuncIsMadeTailCallSafe(CallConv,
2568 MF.getTarget().Options.GuaranteedTailCallOpt))
2569 StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
2571 // If the function takes variable number of arguments, make a frame index for
2572 // the start of the first vararg value... for expansion of llvm.va_start. We
2573 // can skip this if there are no va_start calls.
2574 if (MFI->hasVAStart() &&
2575 (Is64Bit || (CallConv != CallingConv::X86_FastCall &&
2576 CallConv != CallingConv::X86_ThisCall))) {
2577 FuncInfo->setVarArgsFrameIndex(
2578 MFI->CreateFixedObject(1, StackSize, true));
2581 // Figure out if XMM registers are in use.
2582 assert(!(MF.getTarget().Options.UseSoftFloat &&
2583 Fn->hasFnAttribute(Attribute::NoImplicitFloat)) &&
2584 "SSE register cannot be used when SSE is disabled!");
2586 // 64-bit calling conventions support varargs and register parameters, so we
2587 // have to do extra work to spill them in the prologue.
2588 if (Is64Bit && isVarArg && MFI->hasVAStart()) {
2589 // Find the first unallocated argument registers.
2590 ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);
2591 ArrayRef<MCPhysReg> ArgXMMs = get64BitArgumentXMMs(MF, CallConv, Subtarget);
2592 unsigned NumIntRegs =
2593 CCInfo.getFirstUnallocated(ArgGPRs.data(), ArgGPRs.size());
2594 unsigned NumXMMRegs =
2595 CCInfo.getFirstUnallocated(ArgXMMs.data(), ArgXMMs.size());
2596 assert(!(NumXMMRegs && !Subtarget->hasSSE1()) &&
2597 "SSE register cannot be used when SSE is disabled!");
2599 // Gather all the live in physical registers.
2600 SmallVector<SDValue, 6> LiveGPRs;
2601 SmallVector<SDValue, 8> LiveXMMRegs;
2603 for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {
2604 unsigned GPR = MF.addLiveIn(Reg, &X86::GR64RegClass);
2606 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64));
2608 if (!ArgXMMs.empty()) {
2609 unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
2610 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8);
2611 for (MCPhysReg Reg : ArgXMMs.slice(NumXMMRegs)) {
2612 unsigned XMMReg = MF.addLiveIn(Reg, &X86::VR128RegClass);
2613 LiveXMMRegs.push_back(
2614 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32));
2619 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
2620 // Get to the caller-allocated home save location. Add 8 to account
2621 // for the return address.
2622 int HomeOffset = TFI.getOffsetOfLocalArea() + 8;
2623 FuncInfo->setRegSaveFrameIndex(
2624 MFI->CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false));
2625 // Fixup to set vararg frame on shadow area (4 x i64).
2627 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());
2629 // For X86-64, if there are vararg parameters that are passed via
2630 // registers, then we must store them to their spots on the stack so
2631 // they may be loaded by deferencing the result of va_next.
2632 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
2633 FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
2634 FuncInfo->setRegSaveFrameIndex(MFI->CreateStackObject(
2635 ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false));
2638 // Store the integer parameter registers.
2639 SmallVector<SDValue, 8> MemOps;
2640 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
2642 unsigned Offset = FuncInfo->getVarArgsGPOffset();
2643 for (SDValue Val : LiveGPRs) {
2644 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN,
2645 DAG.getIntPtrConstant(Offset));
2647 DAG.getStore(Val.getValue(1), dl, Val, FIN,
2648 MachinePointerInfo::getFixedStack(
2649 FuncInfo->getRegSaveFrameIndex(), Offset),
2651 MemOps.push_back(Store);
2655 if (!ArgXMMs.empty() && NumXMMRegs != ArgXMMs.size()) {
2656 // Now store the XMM (fp + vector) parameter registers.
2657 SmallVector<SDValue, 12> SaveXMMOps;
2658 SaveXMMOps.push_back(Chain);
2659 SaveXMMOps.push_back(ALVal);
2660 SaveXMMOps.push_back(DAG.getIntPtrConstant(
2661 FuncInfo->getRegSaveFrameIndex()));
2662 SaveXMMOps.push_back(DAG.getIntPtrConstant(
2663 FuncInfo->getVarArgsFPOffset()));
2664 SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(),
2666 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
2667 MVT::Other, SaveXMMOps));
2670 if (!MemOps.empty())
2671 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
2674 if (isVarArg && MFI->hasMustTailInVarArgFunc()) {
2675 // Find the largest legal vector type.
2676 MVT VecVT = MVT::Other;
2677 // FIXME: Only some x86_32 calling conventions support AVX512.
2678 if (Subtarget->hasAVX512() &&
2679 (Is64Bit || (CallConv == CallingConv::X86_VectorCall ||
2680 CallConv == CallingConv::Intel_OCL_BI)))
2681 VecVT = MVT::v16f32;
2682 else if (Subtarget->hasAVX())
2684 else if (Subtarget->hasSSE2())
2687 // We forward some GPRs and some vector types.
2688 SmallVector<MVT, 2> RegParmTypes;
2689 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32;
2690 RegParmTypes.push_back(IntVT);
2691 if (VecVT != MVT::Other)
2692 RegParmTypes.push_back(VecVT);
2694 // Compute the set of forwarded registers. The rest are scratch.
2695 SmallVectorImpl<ForwardedRegister> &Forwards =
2696 FuncInfo->getForwardedMustTailRegParms();
2697 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86);
2699 // Conservatively forward AL on x86_64, since it might be used for varargs.
2700 if (Is64Bit && !CCInfo.isAllocated(X86::AL)) {
2701 unsigned ALVReg = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
2702 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));
2705 // Copy all forwards from physical to virtual registers.
2706 for (ForwardedRegister &F : Forwards) {
2707 // FIXME: Can we use a less constrained schedule?
2708 SDValue RegVal = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
2709 F.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(F.VT));
2710 Chain = DAG.getCopyToReg(Chain, dl, F.VReg, RegVal);
2714 // Some CCs need callee pop.
2715 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
2716 MF.getTarget().Options.GuaranteedTailCallOpt)) {
2717 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
2719 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
2720 // If this is an sret function, the return should pop the hidden pointer.
2721 if (!Is64Bit && !IsTailCallConvention(CallConv) &&
2722 !Subtarget->getTargetTriple().isOSMSVCRT() &&
2723 argsAreStructReturn(Ins) == StackStructReturn)
2724 FuncInfo->setBytesToPopOnReturn(4);
2728 // RegSaveFrameIndex is X86-64 only.
2729 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
2730 if (CallConv == CallingConv::X86_FastCall ||
2731 CallConv == CallingConv::X86_ThisCall)
2732 // fastcc functions can't have varargs.
2733 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
2736 FuncInfo->setArgumentStackSize(StackSize);
2742 X86TargetLowering::LowerMemOpCallTo(SDValue Chain,
2743 SDValue StackPtr, SDValue Arg,
2744 SDLoc dl, SelectionDAG &DAG,
2745 const CCValAssign &VA,
2746 ISD::ArgFlagsTy Flags) const {
2747 unsigned LocMemOffset = VA.getLocMemOffset();
2748 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
2749 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
2750 if (Flags.isByVal())
2751 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
2753 return DAG.getStore(Chain, dl, Arg, PtrOff,
2754 MachinePointerInfo::getStack(LocMemOffset),
2758 /// Emit a load of return address if tail call
2759 /// optimization is performed and it is required.
2761 X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG,
2762 SDValue &OutRetAddr, SDValue Chain,
2763 bool IsTailCall, bool Is64Bit,
2764 int FPDiff, SDLoc dl) const {
2765 // Adjust the Return address stack slot.
2766 EVT VT = getPointerTy();
2767 OutRetAddr = getReturnAddressFrameIndex(DAG);
2769 // Load the "old" Return address.
2770 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo(),
2771 false, false, false, 0);
2772 return SDValue(OutRetAddr.getNode(), 1);
2775 /// Emit a store of the return address if tail call
2776 /// optimization is performed and it is required (FPDiff!=0).
2777 static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
2778 SDValue Chain, SDValue RetAddrFrIdx,
2779 EVT PtrVT, unsigned SlotSize,
2780 int FPDiff, SDLoc dl) {
2781 // Store the return address to the appropriate stack slot.
2782 if (!FPDiff) return Chain;
2783 // Calculate the new stack slot for the return address.
2784 int NewReturnAddrFI =
2785 MF.getFrameInfo()->CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
2787 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
2788 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
2789 MachinePointerInfo::getFixedStack(NewReturnAddrFI),
2795 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2796 SmallVectorImpl<SDValue> &InVals) const {
2797 SelectionDAG &DAG = CLI.DAG;
2799 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2800 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2801 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2802 SDValue Chain = CLI.Chain;
2803 SDValue Callee = CLI.Callee;
2804 CallingConv::ID CallConv = CLI.CallConv;
2805 bool &isTailCall = CLI.IsTailCall;
2806 bool isVarArg = CLI.IsVarArg;
2808 MachineFunction &MF = DAG.getMachineFunction();
2809 bool Is64Bit = Subtarget->is64Bit();
2810 bool IsWin64 = Subtarget->isCallingConvWin64(CallConv);
2811 StructReturnType SR = callIsStructReturn(Outs);
2812 bool IsSibcall = false;
2813 X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
2815 if (MF.getTarget().Options.DisableTailCalls)
2818 bool IsMustTail = CLI.CS && CLI.CS->isMustTailCall();
2820 // Force this to be a tail call. The verifier rules are enough to ensure
2821 // that we can lower this successfully without moving the return address
2824 } else if (isTailCall) {
2825 // Check if it's really possible to do a tail call.
2826 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
2827 isVarArg, SR != NotStructReturn,
2828 MF.getFunction()->hasStructRetAttr(), CLI.RetTy,
2829 Outs, OutVals, Ins, DAG);
2831 // Sibcalls are automatically detected tailcalls which do not require
2833 if (!MF.getTarget().Options.GuaranteedTailCallOpt && isTailCall)
2840 assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
2841 "Var args not supported with calling convention fastcc, ghc or hipe");
2843 // Analyze operands of the call, assigning locations to each operand.
2844 SmallVector<CCValAssign, 16> ArgLocs;
2845 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
2847 // Allocate shadow area for Win64
2849 CCInfo.AllocateStack(32, 8);
2851 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
2853 // Get a count of how many bytes are to be pushed on the stack.
2854 unsigned NumBytes = CCInfo.getNextStackOffset();
2856 // This is a sibcall. The memory operands are available in caller's
2857 // own caller's stack.
2859 else if (MF.getTarget().Options.GuaranteedTailCallOpt &&
2860 IsTailCallConvention(CallConv))
2861 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
2864 if (isTailCall && !IsSibcall && !IsMustTail) {
2865 // Lower arguments at fp - stackoffset + fpdiff.
2866 unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
2868 FPDiff = NumBytesCallerPushed - NumBytes;
2870 // Set the delta of movement of the returnaddr stackslot.
2871 // But only set if delta is greater than previous delta.
2872 if (FPDiff < X86Info->getTCReturnAddrDelta())
2873 X86Info->setTCReturnAddrDelta(FPDiff);
2876 unsigned NumBytesToPush = NumBytes;
2877 unsigned NumBytesToPop = NumBytes;
2879 // If we have an inalloca argument, all stack space has already been allocated
2880 // for us and be right at the top of the stack. We don't support multiple
2881 // arguments passed in memory when using inalloca.
2882 if (!Outs.empty() && Outs.back().Flags.isInAlloca()) {
2884 if (!ArgLocs.back().isMemLoc())
2885 report_fatal_error("cannot use inalloca attribute on a register "
2887 if (ArgLocs.back().getLocMemOffset() != 0)
2888 report_fatal_error("any parameter with the inalloca attribute must be "
2889 "the only memory argument");
2893 Chain = DAG.getCALLSEQ_START(
2894 Chain, DAG.getIntPtrConstant(NumBytesToPush, true), dl);
2896 SDValue RetAddrFrIdx;
2897 // Load return address for tail calls.
2898 if (isTailCall && FPDiff)
2899 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
2900 Is64Bit, FPDiff, dl);
2902 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2903 SmallVector<SDValue, 8> MemOpChains;
2906 // Walk the register/memloc assignments, inserting copies/loads. In the case
2907 // of tail call optimization arguments are handle later.
2908 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2909 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2910 // Skip inalloca arguments, they have already been written.
2911 ISD::ArgFlagsTy Flags = Outs[i].Flags;
2912 if (Flags.isInAlloca())
2915 CCValAssign &VA = ArgLocs[i];
2916 EVT RegVT = VA.getLocVT();
2917 SDValue Arg = OutVals[i];
2918 bool isByVal = Flags.isByVal();
2920 // Promote the value if needed.
2921 switch (VA.getLocInfo()) {
2922 default: llvm_unreachable("Unknown loc info!");
2923 case CCValAssign::Full: break;
2924 case CCValAssign::SExt:
2925 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
2927 case CCValAssign::ZExt:
2928 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
2930 case CCValAssign::AExt:
2931 if (RegVT.is128BitVector()) {
2932 // Special case: passing MMX values in XMM registers.
2933 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg);
2934 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
2935 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
2937 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
2939 case CCValAssign::BCvt:
2940 Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg);
2942 case CCValAssign::Indirect: {
2943 // Store the argument.
2944 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
2945 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2946 Chain = DAG.getStore(Chain, dl, Arg, SpillSlot,
2947 MachinePointerInfo::getFixedStack(FI),
2954 if (VA.isRegLoc()) {
2955 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2956 if (isVarArg && IsWin64) {
2957 // Win64 ABI requires argument XMM reg to be copied to the corresponding
2958 // shadow reg if callee is a varargs function.
2959 unsigned ShadowReg = 0;
2960 switch (VA.getLocReg()) {
2961 case X86::XMM0: ShadowReg = X86::RCX; break;
2962 case X86::XMM1: ShadowReg = X86::RDX; break;
2963 case X86::XMM2: ShadowReg = X86::R8; break;
2964 case X86::XMM3: ShadowReg = X86::R9; break;
2967 RegsToPass.push_back(std::make_pair(ShadowReg, Arg));
2969 } else if (!IsSibcall && (!isTailCall || isByVal)) {
2970 assert(VA.isMemLoc());
2971 if (!StackPtr.getNode())
2972 StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
2974 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
2975 dl, DAG, VA, Flags));
2979 if (!MemOpChains.empty())
2980 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
2982 if (Subtarget->isPICStyleGOT()) {
2983 // ELF / PIC requires GOT in the EBX register before function calls via PLT
2986 RegsToPass.push_back(std::make_pair(unsigned(X86::EBX),
2987 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy())));
2989 // If we are tail calling and generating PIC/GOT style code load the
2990 // address of the callee into ECX. The value in ecx is used as target of
2991 // the tail jump. This is done to circumvent the ebx/callee-saved problem
2992 // for tail calls on PIC/GOT architectures. Normally we would just put the
2993 // address of GOT into ebx and then call target@PLT. But for tail calls
2994 // ebx would be restored (since ebx is callee saved) before jumping to the
2997 // Note: The actual moving to ECX is done further down.
2998 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
2999 if (G && !G->getGlobal()->hasHiddenVisibility() &&
3000 !G->getGlobal()->hasProtectedVisibility())
3001 Callee = LowerGlobalAddress(Callee, DAG);
3002 else if (isa<ExternalSymbolSDNode>(Callee))
3003 Callee = LowerExternalSymbol(Callee, DAG);
3007 if (Is64Bit && isVarArg && !IsWin64 && !IsMustTail) {
3008 // From AMD64 ABI document:
3009 // For calls that may call functions that use varargs or stdargs
3010 // (prototype-less calls or calls to functions containing ellipsis (...) in
3011 // the declaration) %al is used as hidden argument to specify the number
3012 // of SSE registers used. The contents of %al do not need to match exactly
3013 // the number of registers, but must be an ubound on the number of SSE
3014 // registers used and is in the range 0 - 8 inclusive.
3016 // Count the number of XMM registers allocated.
3017 static const MCPhysReg XMMArgRegs[] = {
3018 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3019 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3021 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);
3022 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3023 && "SSE registers cannot be used when SSE is disabled");
3025 RegsToPass.push_back(std::make_pair(unsigned(X86::AL),
3026 DAG.getConstant(NumXMMRegs, MVT::i8)));
3029 if (isVarArg && IsMustTail) {
3030 const auto &Forwards = X86Info->getForwardedMustTailRegParms();
3031 for (const auto &F : Forwards) {
3032 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
3033 RegsToPass.push_back(std::make_pair(unsigned(F.PReg), Val));
3037 // For tail calls lower the arguments to the 'real' stack slots. Sibcalls
3038 // don't need this because the eligibility check rejects calls that require
3039 // shuffling arguments passed in memory.
3040 if (!IsSibcall && isTailCall) {
3041 // Force all the incoming stack arguments to be loaded from the stack
3042 // before any new outgoing arguments are stored to the stack, because the
3043 // outgoing stack slots may alias the incoming argument stack slots, and
3044 // the alias isn't otherwise explicit. This is slightly more conservative
3045 // than necessary, because it means that each store effectively depends
3046 // on every argument instead of just those arguments it would clobber.
3047 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
3049 SmallVector<SDValue, 8> MemOpChains2;
3052 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3053 CCValAssign &VA = ArgLocs[i];
3056 assert(VA.isMemLoc());
3057 SDValue Arg = OutVals[i];
3058 ISD::ArgFlagsTy Flags = Outs[i].Flags;
3059 // Skip inalloca arguments. They don't require any work.
3060 if (Flags.isInAlloca())
3062 // Create frame index.
3063 int32_t Offset = VA.getLocMemOffset()+FPDiff;
3064 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
3065 FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true);
3066 FIN = DAG.getFrameIndex(FI, getPointerTy());
3068 if (Flags.isByVal()) {
3069 // Copy relative to framepointer.
3070 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
3071 if (!StackPtr.getNode())
3072 StackPtr = DAG.getCopyFromReg(Chain, dl,
3073 RegInfo->getStackRegister(),
3075 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source);
3077 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
3081 // Store relative to framepointer.
3082 MemOpChains2.push_back(
3083 DAG.getStore(ArgChain, dl, Arg, FIN,
3084 MachinePointerInfo::getFixedStack(FI),
3089 if (!MemOpChains2.empty())
3090 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
3092 // Store the return address to the appropriate stack slot.
3093 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
3094 getPointerTy(), RegInfo->getSlotSize(),
3098 // Build a sequence of copy-to-reg nodes chained together with token chain
3099 // and flag operands which copy the outgoing args into registers.
3101 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
3102 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
3103 RegsToPass[i].second, InFlag);
3104 InFlag = Chain.getValue(1);
3107 if (DAG.getTarget().getCodeModel() == CodeModel::Large) {
3108 assert(Is64Bit && "Large code model is only legal in 64-bit mode.");
3109 // In the 64-bit large code model, we have to make all calls
3110 // through a register, since the call instruction's 32-bit
3111 // pc-relative offset may not be large enough to hold the whole
3113 } else if (Callee->getOpcode() == ISD::GlobalAddress) {
3114 // If the callee is a GlobalAddress node (quite common, every direct call
3115 // is) turn it into a TargetGlobalAddress node so that legalize doesn't hack
3117 GlobalAddressSDNode* G = cast<GlobalAddressSDNode>(Callee);
3119 // We should use extra load for direct calls to dllimported functions in
3121 const GlobalValue *GV = G->getGlobal();
3122 if (!GV->hasDLLImportStorageClass()) {
3123 unsigned char OpFlags = 0;
3124 bool ExtraLoad = false;
3125 unsigned WrapperKind = ISD::DELETED_NODE;
3127 // On ELF targets, in both X86-64 and X86-32 mode, direct calls to
3128 // external symbols most go through the PLT in PIC mode. If the symbol
3129 // has hidden or protected visibility, or if it is static or local, then
3130 // we don't need to use the PLT - we can directly call it.
3131 if (Subtarget->isTargetELF() &&
3132 DAG.getTarget().getRelocationModel() == Reloc::PIC_ &&
3133 GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
3134 OpFlags = X86II::MO_PLT;
3135 } else if (Subtarget->isPICStyleStubAny() &&
3136 (GV->isDeclaration() || GV->isWeakForLinker()) &&
3137 (!Subtarget->getTargetTriple().isMacOSX() ||
3138 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
3139 // PC-relative references to external symbols should go through $stub,
3140 // unless we're building with the leopard linker or later, which
3141 // automatically synthesizes these stubs.
3142 OpFlags = X86II::MO_DARWIN_STUB;
3143 } else if (Subtarget->isPICStyleRIPRel() && isa<Function>(GV) &&
3144 cast<Function>(GV)->hasFnAttribute(Attribute::NonLazyBind)) {
3145 // If the function is marked as non-lazy, generate an indirect call
3146 // which loads from the GOT directly. This avoids runtime overhead
3147 // at the cost of eager binding (and one extra byte of encoding).
3148 OpFlags = X86II::MO_GOTPCREL;
3149 WrapperKind = X86ISD::WrapperRIP;
3153 Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(),
3154 G->getOffset(), OpFlags);
3156 // Add a wrapper if needed.
3157 if (WrapperKind != ISD::DELETED_NODE)
3158 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee);
3159 // Add extra indirection if needed.
3161 Callee = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Callee,
3162 MachinePointerInfo::getGOT(),
3163 false, false, false, 0);
3165 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
3166 unsigned char OpFlags = 0;
3168 // On ELF targets, in either X86-64 or X86-32 mode, direct calls to
3169 // external symbols should go through the PLT.
3170 if (Subtarget->isTargetELF() &&
3171 DAG.getTarget().getRelocationModel() == Reloc::PIC_) {
3172 OpFlags = X86II::MO_PLT;
3173 } else if (Subtarget->isPICStyleStubAny() &&
3174 (!Subtarget->getTargetTriple().isMacOSX() ||
3175 Subtarget->getTargetTriple().isMacOSXVersionLT(10, 5))) {
3176 // PC-relative references to external symbols should go through $stub,
3177 // unless we're building with the leopard linker or later, which
3178 // automatically synthesizes these stubs.
3179 OpFlags = X86II::MO_DARWIN_STUB;
3182 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(),
3184 } else if (Subtarget->isTarget64BitILP32() &&
3185 Callee->getValueType(0) == MVT::i32) {
3186 // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
3187 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee);
3190 // Returns a chain & a flag for retval copy to use.
3191 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3192 SmallVector<SDValue, 8> Ops;
3194 if (!IsSibcall && isTailCall) {
3195 Chain = DAG.getCALLSEQ_END(Chain,
3196 DAG.getIntPtrConstant(NumBytesToPop, true),
3197 DAG.getIntPtrConstant(0, true), InFlag, dl);
3198 InFlag = Chain.getValue(1);
3201 Ops.push_back(Chain);
3202 Ops.push_back(Callee);
3205 Ops.push_back(DAG.getConstant(FPDiff, MVT::i32));
3207 // Add argument registers to the end of the list so that they are known live
3209 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
3210 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
3211 RegsToPass[i].second.getValueType()));
3213 // Add a register mask operand representing the call-preserved registers.
3214 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
3215 const uint32_t *Mask = TRI->getCallPreservedMask(CallConv);
3216 assert(Mask && "Missing call preserved mask for calling convention");
3217 Ops.push_back(DAG.getRegisterMask(Mask));
3219 if (InFlag.getNode())
3220 Ops.push_back(InFlag);
3224 //// If this is the first return lowered for this function, add the regs
3225 //// to the liveout set for the function.
3226 // This isn't right, although it's probably harmless on x86; liveouts
3227 // should be computed from returns not tail calls. Consider a void
3228 // function making a tail call to a function returning int.
3229 return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
3232 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
3233 InFlag = Chain.getValue(1);
3235 // Create the CALLSEQ_END node.
3236 unsigned NumBytesForCalleeToPop;
3237 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
3238 DAG.getTarget().Options.GuaranteedTailCallOpt))
3239 NumBytesForCalleeToPop = NumBytes; // Callee pops everything
3240 else if (!Is64Bit && !IsTailCallConvention(CallConv) &&
3241 !Subtarget->getTargetTriple().isOSMSVCRT() &&
3242 SR == StackStructReturn)
3243 // If this is a call to a struct-return function, the callee
3244 // pops the hidden struct pointer, so we have to push it back.
3245 // This is common for Darwin/X86, Linux & Mingw32 targets.
3246 // For MSVC Win32 targets, the caller pops the hidden struct pointer.
3247 NumBytesForCalleeToPop = 4;
3249 NumBytesForCalleeToPop = 0; // Callee pops nothing.
3251 // Returns a flag for retval copy to use.
3253 Chain = DAG.getCALLSEQ_END(Chain,
3254 DAG.getIntPtrConstant(NumBytesToPop, true),
3255 DAG.getIntPtrConstant(NumBytesForCalleeToPop,
3258 InFlag = Chain.getValue(1);
3261 // Handle result values, copying them out of physregs into vregs that we
3263 return LowerCallResult(Chain, InFlag, CallConv, isVarArg,
3264 Ins, dl, DAG, InVals);
3267 //===----------------------------------------------------------------------===//
3268 // Fast Calling Convention (tail call) implementation
3269 //===----------------------------------------------------------------------===//
3271 // Like std call, callee cleans arguments, convention except that ECX is
3272 // reserved for storing the tail called function address. Only 2 registers are
3273 // free for argument passing (inreg). Tail call optimization is performed
3275 // * tailcallopt is enabled
3276 // * caller/callee are fastcc
3277 // On X86_64 architecture with GOT-style position independent code only local
3278 // (within module) calls are supported at the moment.
3279 // To keep the stack aligned according to platform abi the function
3280 // GetAlignedArgumentStackSize ensures that argument delta is always multiples
3281 // of stack alignment. (Dynamic linkers need this - darwin's dyld for example)
3282 // If a tail called function callee has more arguments than the caller the
3283 // caller needs to make sure that there is room to move the RETADDR to. This is
3284 // achieved by reserving an area the size of the argument delta right after the
3285 // original RETADDR, but before the saved framepointer or the spilled registers
3286 // e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
3298 /// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned
3299 /// for a 16 byte align requirement.
3301 X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
3302 SelectionDAG& DAG) const {
3303 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3304 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
3305 unsigned StackAlignment = TFI.getStackAlignment();
3306 uint64_t AlignMask = StackAlignment - 1;
3307 int64_t Offset = StackSize;
3308 unsigned SlotSize = RegInfo->getSlotSize();
3309 if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) {
3310 // Number smaller than 12 so just add the difference.
3311 Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask));
3313 // Mask out lower bits, add stackalignment once plus the 12 bytes.
3314 Offset = ((~AlignMask) & Offset) + StackAlignment +
3315 (StackAlignment-SlotSize);
3320 /// MatchingStackOffset - Return true if the given stack call argument is
3321 /// already available in the same position (relatively) of the caller's
3322 /// incoming argument stack.
3324 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
3325 MachineFrameInfo *MFI, const MachineRegisterInfo *MRI,
3326 const X86InstrInfo *TII) {
3327 unsigned Bytes = Arg.getValueType().getSizeInBits() / 8;
3329 if (Arg.getOpcode() == ISD::CopyFromReg) {
3330 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
3331 if (!TargetRegisterInfo::isVirtualRegister(VR))
3333 MachineInstr *Def = MRI->getVRegDef(VR);
3336 if (!Flags.isByVal()) {
3337 if (!TII->isLoadFromStackSlot(Def, FI))
3340 unsigned Opcode = Def->getOpcode();
3341 if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r ||
3342 Opcode == X86::LEA64_32r) &&
3343 Def->getOperand(1).isFI()) {
3344 FI = Def->getOperand(1).getIndex();
3345 Bytes = Flags.getByValSize();
3349 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
3350 if (Flags.isByVal())
3351 // ByVal argument is passed in as a pointer but it's now being
3352 // dereferenced. e.g.
3353 // define @foo(%struct.X* %A) {
3354 // tail call @bar(%struct.X* byval %A)
3357 SDValue Ptr = Ld->getBasePtr();
3358 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
3361 FI = FINode->getIndex();
3362 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {
3363 FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg);
3364 FI = FINode->getIndex();
3365 Bytes = Flags.getByValSize();
3369 assert(FI != INT_MAX);
3370 if (!MFI->isFixedObjectIndex(FI))
3372 return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI);
3375 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
3376 /// for tail call optimization. Targets which want to do tail call
3377 /// optimization should implement this function.
3379 X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
3380 CallingConv::ID CalleeCC,
3382 bool isCalleeStructRet,
3383 bool isCallerStructRet,
3385 const SmallVectorImpl<ISD::OutputArg> &Outs,
3386 const SmallVectorImpl<SDValue> &OutVals,
3387 const SmallVectorImpl<ISD::InputArg> &Ins,
3388 SelectionDAG &DAG) const {
3389 if (!IsTailCallConvention(CalleeCC) && !IsCCallConvention(CalleeCC))
3392 // If -tailcallopt is specified, make fastcc functions tail-callable.
3393 const MachineFunction &MF = DAG.getMachineFunction();
3394 const Function *CallerF = MF.getFunction();
3396 // If the function return type is x86_fp80 and the callee return type is not,
3397 // then the FP_EXTEND of the call result is not a nop. It's not safe to
3398 // perform a tailcall optimization here.
3399 if (CallerF->getReturnType()->isX86_FP80Ty() && !RetTy->isX86_FP80Ty())
3402 CallingConv::ID CallerCC = CallerF->getCallingConv();
3403 bool CCMatch = CallerCC == CalleeCC;
3404 bool IsCalleeWin64 = Subtarget->isCallingConvWin64(CalleeCC);
3405 bool IsCallerWin64 = Subtarget->isCallingConvWin64(CallerCC);
3407 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
3408 if (IsTailCallConvention(CalleeCC) && CCMatch)
3413 // Look for obvious safe cases to perform tail call optimization that do not
3414 // require ABI changes. This is what gcc calls sibcall.
3416 // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to
3417 // emit a special epilogue.
3418 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3419 if (RegInfo->needsStackRealignment(MF))
3422 // Also avoid sibcall optimization if either caller or callee uses struct
3423 // return semantics.
3424 if (isCalleeStructRet || isCallerStructRet)
3427 // An stdcall/thiscall caller is expected to clean up its arguments; the
3428 // callee isn't going to do that.
3429 // FIXME: this is more restrictive than needed. We could produce a tailcall
3430 // when the stack adjustment matches. For example, with a thiscall that takes
3431 // only one argument.
3432 if (!CCMatch && (CallerCC == CallingConv::X86_StdCall ||
3433 CallerCC == CallingConv::X86_ThisCall))
3436 // Do not sibcall optimize vararg calls unless all arguments are passed via
3438 if (isVarArg && !Outs.empty()) {
3440 // Optimizing for varargs on Win64 is unlikely to be safe without
3441 // additional testing.
3442 if (IsCalleeWin64 || IsCallerWin64)
3445 SmallVector<CCValAssign, 16> ArgLocs;
3446 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), ArgLocs,
3449 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
3450 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i)
3451 if (!ArgLocs[i].isRegLoc())
3455 // If the call result is in ST0 / ST1, it needs to be popped off the x87
3456 // stack. Therefore, if it's not used by the call it is not safe to optimize
3457 // this into a sibcall.
3458 bool Unused = false;
3459 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
3466 SmallVector<CCValAssign, 16> RVLocs;
3467 CCState CCInfo(CalleeCC, false, DAG.getMachineFunction(), RVLocs,
3469 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
3470 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
3471 CCValAssign &VA = RVLocs[i];
3472 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
3477 // If the calling conventions do not match, then we'd better make sure the
3478 // results are returned in the same way as what the caller expects.
3480 SmallVector<CCValAssign, 16> RVLocs1;
3481 CCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(), RVLocs1,
3483 CCInfo1.AnalyzeCallResult(Ins, RetCC_X86);
3485 SmallVector<CCValAssign, 16> RVLocs2;
3486 CCState CCInfo2(CallerCC, false, DAG.getMachineFunction(), RVLocs2,
3488 CCInfo2.AnalyzeCallResult(Ins, RetCC_X86);
3490 if (RVLocs1.size() != RVLocs2.size())
3492 for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) {
3493 if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc())
3495 if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo())
3497 if (RVLocs1[i].isRegLoc()) {
3498 if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg())
3501 if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset())
3507 // If the callee takes no arguments then go on to check the results of the
3509 if (!Outs.empty()) {
3510 // Check if stack adjustment is needed. For now, do not do this if any
3511 // argument is passed on the stack.
3512 SmallVector<CCValAssign, 16> ArgLocs;
3513 CCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), ArgLocs,
3516 // Allocate shadow area for Win64
3518 CCInfo.AllocateStack(32, 8);
3520 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
3521 if (CCInfo.getNextStackOffset()) {
3522 MachineFunction &MF = DAG.getMachineFunction();
3523 if (MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn())
3526 // Check if the arguments are already laid out in the right way as
3527 // the caller's fixed stack objects.
3528 MachineFrameInfo *MFI = MF.getFrameInfo();
3529 const MachineRegisterInfo *MRI = &MF.getRegInfo();
3530 const X86InstrInfo *TII = Subtarget->getInstrInfo();
3531 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3532 CCValAssign &VA = ArgLocs[i];
3533 SDValue Arg = OutVals[i];
3534 ISD::ArgFlagsTy Flags = Outs[i].Flags;
3535 if (VA.getLocInfo() == CCValAssign::Indirect)
3537 if (!VA.isRegLoc()) {
3538 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
3545 // If the tailcall address may be in a register, then make sure it's
3546 // possible to register allocate for it. In 32-bit, the call address can
3547 // only target EAX, EDX, or ECX since the tail call must be scheduled after
3548 // callee-saved registers are restored. These happen to be the same
3549 // registers used to pass 'inreg' arguments so watch out for those.
3550 if (!Subtarget->is64Bit() &&
3551 ((!isa<GlobalAddressSDNode>(Callee) &&
3552 !isa<ExternalSymbolSDNode>(Callee)) ||
3553 DAG.getTarget().getRelocationModel() == Reloc::PIC_)) {
3554 unsigned NumInRegs = 0;
3555 // In PIC we need an extra register to formulate the address computation
3557 unsigned MaxInRegs =
3558 (DAG.getTarget().getRelocationModel() == Reloc::PIC_) ? 2 : 3;
3560 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3561 CCValAssign &VA = ArgLocs[i];
3564 unsigned Reg = VA.getLocReg();
3567 case X86::EAX: case X86::EDX: case X86::ECX:
3568 if (++NumInRegs == MaxInRegs)
3580 X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
3581 const TargetLibraryInfo *libInfo) const {
3582 return X86::createFastISel(funcInfo, libInfo);
3585 //===----------------------------------------------------------------------===//
3586 // Other Lowering Hooks
3587 //===----------------------------------------------------------------------===//
3589 static bool MayFoldLoad(SDValue Op) {
3590 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
3593 static bool MayFoldIntoStore(SDValue Op) {
3594 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
3597 static bool isTargetShuffle(unsigned Opcode) {
3599 default: return false;
3600 case X86ISD::BLENDI:
3601 case X86ISD::PSHUFB:
3602 case X86ISD::PSHUFD:
3603 case X86ISD::PSHUFHW:
3604 case X86ISD::PSHUFLW:
3606 case X86ISD::PALIGNR:
3607 case X86ISD::MOVLHPS:
3608 case X86ISD::MOVLHPD:
3609 case X86ISD::MOVHLPS:
3610 case X86ISD::MOVLPS:
3611 case X86ISD::MOVLPD:
3612 case X86ISD::MOVSHDUP:
3613 case X86ISD::MOVSLDUP:
3614 case X86ISD::MOVDDUP:
3617 case X86ISD::UNPCKL:
3618 case X86ISD::UNPCKH:
3619 case X86ISD::VPERMILPI:
3620 case X86ISD::VPERM2X128:
3621 case X86ISD::VPERMI:
3626 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3627 SDValue V1, SelectionDAG &DAG) {
3629 default: llvm_unreachable("Unknown x86 shuffle node");
3630 case X86ISD::MOVSHDUP:
3631 case X86ISD::MOVSLDUP:
3632 case X86ISD::MOVDDUP:
3633 return DAG.getNode(Opc, dl, VT, V1);
3637 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3638 SDValue V1, unsigned TargetMask,
3639 SelectionDAG &DAG) {
3641 default: llvm_unreachable("Unknown x86 shuffle node");
3642 case X86ISD::PSHUFD:
3643 case X86ISD::PSHUFHW:
3644 case X86ISD::PSHUFLW:
3645 case X86ISD::VPERMILPI:
3646 case X86ISD::VPERMI:
3647 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8));
3651 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3652 SDValue V1, SDValue V2, unsigned TargetMask,
3653 SelectionDAG &DAG) {
3655 default: llvm_unreachable("Unknown x86 shuffle node");
3656 case X86ISD::PALIGNR:
3657 case X86ISD::VALIGN:
3659 case X86ISD::VPERM2X128:
3660 return DAG.getNode(Opc, dl, VT, V1, V2,
3661 DAG.getConstant(TargetMask, MVT::i8));
3665 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, EVT VT,
3666 SDValue V1, SDValue V2, SelectionDAG &DAG) {
3668 default: llvm_unreachable("Unknown x86 shuffle node");
3669 case X86ISD::MOVLHPS:
3670 case X86ISD::MOVLHPD:
3671 case X86ISD::MOVHLPS:
3672 case X86ISD::MOVLPS:
3673 case X86ISD::MOVLPD:
3676 case X86ISD::UNPCKL:
3677 case X86ISD::UNPCKH:
3678 return DAG.getNode(Opc, dl, VT, V1, V2);
3682 SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
3683 MachineFunction &MF = DAG.getMachineFunction();
3684 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3685 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
3686 int ReturnAddrIndex = FuncInfo->getRAIndex();
3688 if (ReturnAddrIndex == 0) {
3689 // Set up a frame object for the return address.
3690 unsigned SlotSize = RegInfo->getSlotSize();
3691 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize,
3694 FuncInfo->setRAIndex(ReturnAddrIndex);
3697 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
3700 bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
3701 bool hasSymbolicDisplacement) {
3702 // Offset should fit into 32 bit immediate field.
3703 if (!isInt<32>(Offset))
3706 // If we don't have a symbolic displacement - we don't have any extra
3708 if (!hasSymbolicDisplacement)
3711 // FIXME: Some tweaks might be needed for medium code model.
3712 if (M != CodeModel::Small && M != CodeModel::Kernel)
3715 // For small code model we assume that latest object is 16MB before end of 31
3716 // bits boundary. We may also accept pretty large negative constants knowing
3717 // that all objects are in the positive half of address space.
3718 if (M == CodeModel::Small && Offset < 16*1024*1024)
3721 // For kernel code model we know that all object resist in the negative half
3722 // of 32bits address space. We may not accept negative offsets, since they may
3723 // be just off and we may accept pretty large positive ones.
3724 if (M == CodeModel::Kernel && Offset >= 0)
3730 /// isCalleePop - Determines whether the callee is required to pop its
3731 /// own arguments. Callee pop is necessary to support tail calls.
3732 bool X86::isCalleePop(CallingConv::ID CallingConv,
3733 bool is64Bit, bool IsVarArg, bool TailCallOpt) {
3734 switch (CallingConv) {
3737 case CallingConv::X86_StdCall:
3738 case CallingConv::X86_FastCall:
3739 case CallingConv::X86_ThisCall:
3741 case CallingConv::Fast:
3742 case CallingConv::GHC:
3743 case CallingConv::HiPE:
3750 /// \brief Return true if the condition is an unsigned comparison operation.
3751 static bool isX86CCUnsigned(unsigned X86CC) {
3753 default: llvm_unreachable("Invalid integer condition!");
3754 case X86::COND_E: return true;
3755 case X86::COND_G: return false;
3756 case X86::COND_GE: return false;
3757 case X86::COND_L: return false;
3758 case X86::COND_LE: return false;
3759 case X86::COND_NE: return true;
3760 case X86::COND_B: return true;
3761 case X86::COND_A: return true;
3762 case X86::COND_BE: return true;
3763 case X86::COND_AE: return true;
3765 llvm_unreachable("covered switch fell through?!");
3768 /// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86
3769 /// specific condition code, returning the condition code and the LHS/RHS of the
3770 /// comparison to make.
3771 static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
3772 SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) {
3774 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
3775 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
3776 // X > -1 -> X == 0, jump !sign.
3777 RHS = DAG.getConstant(0, RHS.getValueType());
3778 return X86::COND_NS;
3780 if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
3781 // X < 0 -> X == 0, jump on sign.
3784 if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) {
3786 RHS = DAG.getConstant(0, RHS.getValueType());
3787 return X86::COND_LE;
3791 switch (SetCCOpcode) {
3792 default: llvm_unreachable("Invalid integer condition!");
3793 case ISD::SETEQ: return X86::COND_E;
3794 case ISD::SETGT: return X86::COND_G;
3795 case ISD::SETGE: return X86::COND_GE;
3796 case ISD::SETLT: return X86::COND_L;
3797 case ISD::SETLE: return X86::COND_LE;
3798 case ISD::SETNE: return X86::COND_NE;
3799 case ISD::SETULT: return X86::COND_B;
3800 case ISD::SETUGT: return X86::COND_A;
3801 case ISD::SETULE: return X86::COND_BE;
3802 case ISD::SETUGE: return X86::COND_AE;
3806 // First determine if it is required or is profitable to flip the operands.
3808 // If LHS is a foldable load, but RHS is not, flip the condition.
3809 if (ISD::isNON_EXTLoad(LHS.getNode()) &&
3810 !ISD::isNON_EXTLoad(RHS.getNode())) {
3811 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
3812 std::swap(LHS, RHS);
3815 switch (SetCCOpcode) {
3821 std::swap(LHS, RHS);
3825 // On a floating point condition, the flags are set as follows:
3827 // 0 | 0 | 0 | X > Y
3828 // 0 | 0 | 1 | X < Y
3829 // 1 | 0 | 0 | X == Y
3830 // 1 | 1 | 1 | unordered
3831 switch (SetCCOpcode) {
3832 default: llvm_unreachable("Condcode should be pre-legalized away");
3834 case ISD::SETEQ: return X86::COND_E;
3835 case ISD::SETOLT: // flipped
3837 case ISD::SETGT: return X86::COND_A;
3838 case ISD::SETOLE: // flipped
3840 case ISD::SETGE: return X86::COND_AE;
3841 case ISD::SETUGT: // flipped
3843 case ISD::SETLT: return X86::COND_B;
3844 case ISD::SETUGE: // flipped
3846 case ISD::SETLE: return X86::COND_BE;
3848 case ISD::SETNE: return X86::COND_NE;
3849 case ISD::SETUO: return X86::COND_P;
3850 case ISD::SETO: return X86::COND_NP;
3852 case ISD::SETUNE: return X86::COND_INVALID;
3856 /// hasFPCMov - is there a floating point cmov for the specific X86 condition
3857 /// code. Current x86 isa includes the following FP cmov instructions:
3858 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
3859 static bool hasFPCMov(unsigned X86CC) {
3875 /// isFPImmLegal - Returns true if the target can instruction select the
3876 /// specified FP immediate natively. If false, the legalizer will
3877 /// materialize the FP immediate as a load from a constant pool.
3878 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
3879 for (unsigned i = 0, e = LegalFPImmediates.size(); i != e; ++i) {
3880 if (Imm.bitwiseIsEqual(LegalFPImmediates[i]))
3886 bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
3887 ISD::LoadExtType ExtTy,
3889 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3890 // relocation target a movq or addq instruction: don't let the load shrink.
3891 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3892 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3893 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3894 return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3898 /// \brief Returns true if it is beneficial to convert a load of a constant
3899 /// to just the constant itself.
3900 bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3902 assert(Ty->isIntegerTy());
3904 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3905 if (BitSize == 0 || BitSize > 64)
3910 bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT,
3911 unsigned Index) const {
3912 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
3915 return (Index == 0 || Index == ResVT.getVectorNumElements());
3918 bool X86TargetLowering::isCheapToSpeculateCttz() const {
3919 // Speculate cttz only if we can directly use TZCNT.
3920 return Subtarget->hasBMI();
3923 bool X86TargetLowering::isCheapToSpeculateCtlz() const {
3924 // Speculate ctlz only if we can directly use LZCNT.
3925 return Subtarget->hasLZCNT();
3928 /// isUndefOrInRange - Return true if Val is undef or if its value falls within
3929 /// the specified range (L, H].
3930 static bool isUndefOrInRange(int Val, int Low, int Hi) {
3931 return (Val < 0) || (Val >= Low && Val < Hi);
3934 /// isUndefOrEqual - Val is either less than zero (undef) or equal to the
3935 /// specified value.
3936 static bool isUndefOrEqual(int Val, int CmpVal) {
3937 return (Val < 0 || Val == CmpVal);
3940 /// isSequentialOrUndefInRange - Return true if every element in Mask, beginning
3941 /// from position Pos and ending in Pos+Size, falls within the specified
3942 /// sequential range (Low, Low+Size]. or is undef.
3943 static bool isSequentialOrUndefInRange(ArrayRef<int> Mask,
3944 unsigned Pos, unsigned Size, int Low) {
3945 for (unsigned i = Pos, e = Pos+Size; i != e; ++i, ++Low)
3946 if (!isUndefOrEqual(Mask[i], Low))
3951 /// isPSHUFDMask - Return true if the node specifies a shuffle of elements that
3952 /// is suitable for input to PSHUFD. That is, it doesn't reference the other
3953 /// operand - by default will match for first operand.
3954 static bool isPSHUFDMask(ArrayRef<int> Mask, MVT VT,
3955 bool TestSecondOperand = false) {
3956 if (VT != MVT::v4f32 && VT != MVT::v4i32 &&
3957 VT != MVT::v2f64 && VT != MVT::v2i64)
3960 unsigned NumElems = VT.getVectorNumElements();
3961 unsigned Lo = TestSecondOperand ? NumElems : 0;
3962 unsigned Hi = Lo + NumElems;
3964 for (unsigned i = 0; i < NumElems; ++i)
3965 if (!isUndefOrInRange(Mask[i], (int)Lo, (int)Hi))
3971 /// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that
3972 /// is suitable for input to PSHUFHW.
3973 static bool isPSHUFHWMask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
3974 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
3977 // Lower quadword copied in order or undef.
3978 if (!isSequentialOrUndefInRange(Mask, 0, 4, 0))
3981 // Upper quadword shuffled.
3982 for (unsigned i = 4; i != 8; ++i)
3983 if (!isUndefOrInRange(Mask[i], 4, 8))
3986 if (VT == MVT::v16i16) {
3987 // Lower quadword copied in order or undef.
3988 if (!isSequentialOrUndefInRange(Mask, 8, 4, 8))
3991 // Upper quadword shuffled.
3992 for (unsigned i = 12; i != 16; ++i)
3993 if (!isUndefOrInRange(Mask[i], 12, 16))
4000 /// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that
4001 /// is suitable for input to PSHUFLW.
4002 static bool isPSHUFLWMask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
4003 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
4006 // Upper quadword copied in order.
4007 if (!isSequentialOrUndefInRange(Mask, 4, 4, 4))
4010 // Lower quadword shuffled.
4011 for (unsigned i = 0; i != 4; ++i)
4012 if (!isUndefOrInRange(Mask[i], 0, 4))
4015 if (VT == MVT::v16i16) {
4016 // Upper quadword copied in order.
4017 if (!isSequentialOrUndefInRange(Mask, 12, 4, 12))
4020 // Lower quadword shuffled.
4021 for (unsigned i = 8; i != 12; ++i)
4022 if (!isUndefOrInRange(Mask[i], 8, 12))
4029 /// \brief Return true if the mask specifies a shuffle of elements that is
4030 /// suitable for input to intralane (palignr) or interlane (valign) vector
4032 static bool isAlignrMask(ArrayRef<int> Mask, MVT VT, bool InterLane) {
4033 unsigned NumElts = VT.getVectorNumElements();
4034 unsigned NumLanes = InterLane ? 1: VT.getSizeInBits()/128;
4035 unsigned NumLaneElts = NumElts/NumLanes;
4037 // Do not handle 64-bit element shuffles with palignr.
4038 if (NumLaneElts == 2)
4041 for (unsigned l = 0; l != NumElts; l+=NumLaneElts) {
4043 for (i = 0; i != NumLaneElts; ++i) {
4048 // Lane is all undef, go to next lane
4049 if (i == NumLaneElts)
4052 int Start = Mask[i+l];
4054 // Make sure its in this lane in one of the sources
4055 if (!isUndefOrInRange(Start, l, l+NumLaneElts) &&
4056 !isUndefOrInRange(Start, l+NumElts, l+NumElts+NumLaneElts))
4059 // If not lane 0, then we must match lane 0
4060 if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Start, Mask[i]+l))
4063 // Correct second source to be contiguous with first source
4064 if (Start >= (int)NumElts)
4065 Start -= NumElts - NumLaneElts;
4067 // Make sure we're shifting in the right direction.
4068 if (Start <= (int)(i+l))
4073 // Check the rest of the elements to see if they are consecutive.
4074 for (++i; i != NumLaneElts; ++i) {
4075 int Idx = Mask[i+l];
4077 // Make sure its in this lane
4078 if (!isUndefOrInRange(Idx, l, l+NumLaneElts) &&
4079 !isUndefOrInRange(Idx, l+NumElts, l+NumElts+NumLaneElts))
4082 // If not lane 0, then we must match lane 0
4083 if (l != 0 && Mask[i] >= 0 && !isUndefOrEqual(Idx, Mask[i]+l))
4086 if (Idx >= (int)NumElts)
4087 Idx -= NumElts - NumLaneElts;
4089 if (!isUndefOrEqual(Idx, Start+i))
4098 /// \brief Return true if the node specifies a shuffle of elements that is
4099 /// suitable for input to PALIGNR.
4100 static bool isPALIGNRMask(ArrayRef<int> Mask, MVT VT,
4101 const X86Subtarget *Subtarget) {
4102 if ((VT.is128BitVector() && !Subtarget->hasSSSE3()) ||
4103 (VT.is256BitVector() && !Subtarget->hasInt256()) ||
4104 VT.is512BitVector())
4105 // FIXME: Add AVX512BW.
4108 return isAlignrMask(Mask, VT, false);
4111 /// \brief Return true if the node specifies a shuffle of elements that is
4112 /// suitable for input to VALIGN.
4113 static bool isVALIGNMask(ArrayRef<int> Mask, MVT VT,
4114 const X86Subtarget *Subtarget) {
4115 // FIXME: Add AVX512VL.
4116 if (!VT.is512BitVector() || !Subtarget->hasAVX512())
4118 return isAlignrMask(Mask, VT, true);
4121 /// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming
4122 /// the two vector operands have swapped position.
4123 static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask,
4124 unsigned NumElems) {
4125 for (unsigned i = 0; i != NumElems; ++i) {
4129 else if (idx < (int)NumElems)
4130 Mask[i] = idx + NumElems;
4132 Mask[i] = idx - NumElems;
4136 /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
4137 /// specifies a shuffle of elements that is suitable for input to 128/256-bit
4138 /// SHUFPS and SHUFPD. If Commuted is true, then it checks for sources to be
4139 /// reverse of what x86 shuffles want.
4140 static bool isSHUFPMask(ArrayRef<int> Mask, MVT VT, bool Commuted = false) {
4142 unsigned NumElems = VT.getVectorNumElements();
4143 unsigned NumLanes = VT.getSizeInBits()/128;
4144 unsigned NumLaneElems = NumElems/NumLanes;
4146 if (NumLaneElems != 2 && NumLaneElems != 4)
4149 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4150 bool symmetricMaskRequired =
4151 (VT.getSizeInBits() >= 256) && (EltSize == 32);
4153 // VSHUFPSY divides the resulting vector into 4 chunks.
4154 // The sources are also splitted into 4 chunks, and each destination
4155 // chunk must come from a different source chunk.
4157 // SRC1 => X7 X6 X5 X4 X3 X2 X1 X0
4158 // SRC2 => Y7 Y6 Y5 Y4 Y3 Y2 Y1 Y9
4160 // DST => Y7..Y4, Y7..Y4, X7..X4, X7..X4,
4161 // Y3..Y0, Y3..Y0, X3..X0, X3..X0
4163 // VSHUFPDY divides the resulting vector into 4 chunks.
4164 // The sources are also splitted into 4 chunks, and each destination
4165 // chunk must come from a different source chunk.
4167 // SRC1 => X3 X2 X1 X0
4168 // SRC2 => Y3 Y2 Y1 Y0
4170 // DST => Y3..Y2, X3..X2, Y1..Y0, X1..X0
4172 SmallVector<int, 4> MaskVal(NumLaneElems, -1);
4173 unsigned HalfLaneElems = NumLaneElems/2;
4174 for (unsigned l = 0; l != NumElems; l += NumLaneElems) {
4175 for (unsigned i = 0; i != NumLaneElems; ++i) {
4176 int Idx = Mask[i+l];
4177 unsigned RngStart = l + ((Commuted == (i<HalfLaneElems)) ? NumElems : 0);
4178 if (!isUndefOrInRange(Idx, RngStart, RngStart+NumLaneElems))
4180 // For VSHUFPSY, the mask of the second half must be the same as the
4181 // first but with the appropriate offsets. This works in the same way as
4182 // VPERMILPS works with masks.
4183 if (!symmetricMaskRequired || Idx < 0)
4185 if (MaskVal[i] < 0) {
4186 MaskVal[i] = Idx - l;
4189 if ((signed)(Idx - l) != MaskVal[i])
4197 /// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
4198 /// specifies a shuffle of elements that is suitable for input to MOVHLPS.
4199 static bool isMOVHLPSMask(ArrayRef<int> Mask, MVT VT) {
4200 if (!VT.is128BitVector())
4203 unsigned NumElems = VT.getVectorNumElements();
4208 // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
4209 return isUndefOrEqual(Mask[0], 6) &&
4210 isUndefOrEqual(Mask[1], 7) &&
4211 isUndefOrEqual(Mask[2], 2) &&
4212 isUndefOrEqual(Mask[3], 3);
4215 /// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
4216 /// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
4218 static bool isMOVHLPS_v_undef_Mask(ArrayRef<int> Mask, MVT VT) {
4219 if (!VT.is128BitVector())
4222 unsigned NumElems = VT.getVectorNumElements();
4227 return isUndefOrEqual(Mask[0], 2) &&
4228 isUndefOrEqual(Mask[1], 3) &&
4229 isUndefOrEqual(Mask[2], 2) &&
4230 isUndefOrEqual(Mask[3], 3);
4233 /// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
4234 /// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
4235 static bool isMOVLPMask(ArrayRef<int> Mask, MVT VT) {
4236 if (!VT.is128BitVector())
4239 unsigned NumElems = VT.getVectorNumElements();
4241 if (NumElems != 2 && NumElems != 4)
4244 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
4245 if (!isUndefOrEqual(Mask[i], i + NumElems))
4248 for (unsigned i = NumElems/2, e = NumElems; i != e; ++i)
4249 if (!isUndefOrEqual(Mask[i], i))
4255 /// isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand
4256 /// specifies a shuffle of elements that is suitable for input to MOVLHPS.
4257 static bool isMOVLHPSMask(ArrayRef<int> Mask, MVT VT) {
4258 if (!VT.is128BitVector())
4261 unsigned NumElems = VT.getVectorNumElements();
4263 if (NumElems != 2 && NumElems != 4)
4266 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
4267 if (!isUndefOrEqual(Mask[i], i))
4270 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
4271 if (!isUndefOrEqual(Mask[i + e], i + NumElems))
4277 /// isINSERTPSMask - Return true if the specified VECTOR_SHUFFLE operand
4278 /// specifies a shuffle of elements that is suitable for input to INSERTPS.
4279 /// i. e: If all but one element come from the same vector.
4280 static bool isINSERTPSMask(ArrayRef<int> Mask, MVT VT) {
4281 // TODO: Deal with AVX's VINSERTPS
4282 if (!VT.is128BitVector() || (VT != MVT::v4f32 && VT != MVT::v4i32))
4285 unsigned CorrectPosV1 = 0;
4286 unsigned CorrectPosV2 = 0;
4287 for (int i = 0, e = (int)VT.getVectorNumElements(); i != e; ++i) {
4288 if (Mask[i] == -1) {
4296 else if (Mask[i] == i + 4)
4300 if (CorrectPosV1 == 3 || CorrectPosV2 == 3)
4301 // We have 3 elements (undefs count as elements from any vector) from one
4302 // vector, and one from another.
4309 // Some special combinations that can be optimized.
4312 SDValue Compact8x32ShuffleNode(ShuffleVectorSDNode *SVOp,
4313 SelectionDAG &DAG) {
4314 MVT VT = SVOp->getSimpleValueType(0);
4317 if (VT != MVT::v8i32 && VT != MVT::v8f32)
4320 ArrayRef<int> Mask = SVOp->getMask();
4322 // These are the special masks that may be optimized.
4323 static const int MaskToOptimizeEven[] = {0, 8, 2, 10, 4, 12, 6, 14};
4324 static const int MaskToOptimizeOdd[] = {1, 9, 3, 11, 5, 13, 7, 15};
4325 bool MatchEvenMask = true;
4326 bool MatchOddMask = true;
4327 for (int i=0; i<8; ++i) {
4328 if (!isUndefOrEqual(Mask[i], MaskToOptimizeEven[i]))
4329 MatchEvenMask = false;
4330 if (!isUndefOrEqual(Mask[i], MaskToOptimizeOdd[i]))
4331 MatchOddMask = false;
4334 if (!MatchEvenMask && !MatchOddMask)
4337 SDValue UndefNode = DAG.getNode(ISD::UNDEF, dl, VT);
4339 SDValue Op0 = SVOp->getOperand(0);
4340 SDValue Op1 = SVOp->getOperand(1);
4342 if (MatchEvenMask) {
4343 // Shift the second operand right to 32 bits.
4344 static const int ShiftRightMask[] = {-1, 0, -1, 2, -1, 4, -1, 6 };
4345 Op1 = DAG.getVectorShuffle(VT, dl, Op1, UndefNode, ShiftRightMask);
4347 // Shift the first operand left to 32 bits.
4348 static const int ShiftLeftMask[] = {1, -1, 3, -1, 5, -1, 7, -1 };
4349 Op0 = DAG.getVectorShuffle(VT, dl, Op0, UndefNode, ShiftLeftMask);
4351 static const int BlendMask[] = {0, 9, 2, 11, 4, 13, 6, 15};
4352 return DAG.getVectorShuffle(VT, dl, Op0, Op1, BlendMask);
4355 /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
4356 /// specifies a shuffle of elements that is suitable for input to UNPCKL.
4357 static bool isUNPCKLMask(ArrayRef<int> Mask, MVT VT,
4358 bool HasInt256, bool V2IsSplat = false) {
4360 assert(VT.getSizeInBits() >= 128 &&
4361 "Unsupported vector type for unpckl");
4363 unsigned NumElts = VT.getVectorNumElements();
4364 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
4365 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4368 assert((!VT.is512BitVector() || VT.getScalarType().getSizeInBits() >= 32) &&
4369 "Unsupported vector type for unpckh");
4371 // AVX defines UNPCK* to operate independently on 128-bit lanes.
4372 unsigned NumLanes = VT.getSizeInBits()/128;
4373 unsigned NumLaneElts = NumElts/NumLanes;
4375 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4376 for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) {
4377 int BitI = Mask[l+i];
4378 int BitI1 = Mask[l+i+1];
4379 if (!isUndefOrEqual(BitI, j))
4382 if (!isUndefOrEqual(BitI1, NumElts))
4385 if (!isUndefOrEqual(BitI1, j + NumElts))
4394 /// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
4395 /// specifies a shuffle of elements that is suitable for input to UNPCKH.
4396 static bool isUNPCKHMask(ArrayRef<int> Mask, MVT VT,
4397 bool HasInt256, bool V2IsSplat = false) {
4398 assert(VT.getSizeInBits() >= 128 &&
4399 "Unsupported vector type for unpckh");
4401 unsigned NumElts = VT.getVectorNumElements();
4402 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
4403 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4406 assert((!VT.is512BitVector() || VT.getScalarType().getSizeInBits() >= 32) &&
4407 "Unsupported vector type for unpckh");
4409 // AVX defines UNPCK* to operate independently on 128-bit lanes.
4410 unsigned NumLanes = VT.getSizeInBits()/128;
4411 unsigned NumLaneElts = NumElts/NumLanes;
4413 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4414 for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) {
4415 int BitI = Mask[l+i];
4416 int BitI1 = Mask[l+i+1];
4417 if (!isUndefOrEqual(BitI, j))
4420 if (isUndefOrEqual(BitI1, NumElts))
4423 if (!isUndefOrEqual(BitI1, j+NumElts))
4431 /// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
4432 /// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
4434 static bool isUNPCKL_v_undef_Mask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
4435 unsigned NumElts = VT.getVectorNumElements();
4436 bool Is256BitVec = VT.is256BitVector();
4438 if (VT.is512BitVector())
4440 assert((VT.is128BitVector() || VT.is256BitVector()) &&
4441 "Unsupported vector type for unpckh");
4443 if (Is256BitVec && NumElts != 4 && NumElts != 8 &&
4444 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4447 // For 256-bit i64/f64, use MOVDDUPY instead, so reject the matching pattern
4448 // FIXME: Need a better way to get rid of this, there's no latency difference
4449 // between UNPCKLPD and MOVDDUP, the later should always be checked first and
4450 // the former later. We should also remove the "_undef" special mask.
4451 if (NumElts == 4 && Is256BitVec)
4454 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
4455 // independently on 128-bit lanes.
4456 unsigned NumLanes = VT.getSizeInBits()/128;
4457 unsigned NumLaneElts = NumElts/NumLanes;
4459 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4460 for (unsigned i = 0, j = l; i != NumLaneElts; i += 2, ++j) {
4461 int BitI = Mask[l+i];
4462 int BitI1 = Mask[l+i+1];
4464 if (!isUndefOrEqual(BitI, j))
4466 if (!isUndefOrEqual(BitI1, j))
4474 /// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form
4475 /// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef,
4477 static bool isUNPCKH_v_undef_Mask(ArrayRef<int> Mask, MVT VT, bool HasInt256) {
4478 unsigned NumElts = VT.getVectorNumElements();
4480 if (VT.is512BitVector())
4483 assert((VT.is128BitVector() || VT.is256BitVector()) &&
4484 "Unsupported vector type for unpckh");
4486 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
4487 (!HasInt256 || (NumElts != 16 && NumElts != 32)))
4490 // Handle 128 and 256-bit vector lengths. AVX defines UNPCK* to operate
4491 // independently on 128-bit lanes.
4492 unsigned NumLanes = VT.getSizeInBits()/128;
4493 unsigned NumLaneElts = NumElts/NumLanes;
4495 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
4496 for (unsigned i = 0, j = l+NumLaneElts/2; i != NumLaneElts; i += 2, ++j) {
4497 int BitI = Mask[l+i];
4498 int BitI1 = Mask[l+i+1];
4499 if (!isUndefOrEqual(BitI, j))
4501 if (!isUndefOrEqual(BitI1, j))
4508 // Match for INSERTI64x4 INSERTF64x4 instructions (src0[0], src1[0]) or
4509 // (src1[0], src0[1]), manipulation with 256-bit sub-vectors
4510 static bool isINSERT64x4Mask(ArrayRef<int> Mask, MVT VT, unsigned int *Imm) {
4511 if (!VT.is512BitVector())
4514 unsigned NumElts = VT.getVectorNumElements();
4515 unsigned HalfSize = NumElts/2;
4516 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, 0)) {
4517 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, NumElts)) {
4522 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, NumElts)) {
4523 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, HalfSize)) {
4531 /// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
4532 /// specifies a shuffle of elements that is suitable for input to MOVSS,
4533 /// MOVSD, and MOVD, i.e. setting the lowest element.
4534 static bool isMOVLMask(ArrayRef<int> Mask, EVT VT) {
4535 if (VT.getVectorElementType().getSizeInBits() < 32)
4537 if (!VT.is128BitVector())
4540 unsigned NumElts = VT.getVectorNumElements();
4542 if (!isUndefOrEqual(Mask[0], NumElts))
4545 for (unsigned i = 1; i != NumElts; ++i)
4546 if (!isUndefOrEqual(Mask[i], i))
4552 /// isVPERM2X128Mask - Match 256-bit shuffles where the elements are considered
4553 /// as permutations between 128-bit chunks or halves. As an example: this
4555 /// vector_shuffle <4, 5, 6, 7, 12, 13, 14, 15>
4556 /// The first half comes from the second half of V1 and the second half from the
4557 /// the second half of V2.
4558 static bool isVPERM2X128Mask(ArrayRef<int> Mask, MVT VT, bool HasFp256) {
4559 if (!HasFp256 || !VT.is256BitVector())
4562 // The shuffle result is divided into half A and half B. In total the two
4563 // sources have 4 halves, namely: C, D, E, F. The final values of A and
4564 // B must come from C, D, E or F.
4565 unsigned HalfSize = VT.getVectorNumElements()/2;
4566 bool MatchA = false, MatchB = false;
4568 // Check if A comes from one of C, D, E, F.
4569 for (unsigned Half = 0; Half != 4; ++Half) {
4570 if (isSequentialOrUndefInRange(Mask, 0, HalfSize, Half*HalfSize)) {
4576 // Check if B comes from one of C, D, E, F.
4577 for (unsigned Half = 0; Half != 4; ++Half) {
4578 if (isSequentialOrUndefInRange(Mask, HalfSize, HalfSize, Half*HalfSize)) {
4584 return MatchA && MatchB;
4587 /// getShuffleVPERM2X128Immediate - Return the appropriate immediate to shuffle
4588 /// the specified VECTOR_MASK mask with VPERM2F128/VPERM2I128 instructions.
4589 static unsigned getShuffleVPERM2X128Immediate(ShuffleVectorSDNode *SVOp) {
4590 MVT VT = SVOp->getSimpleValueType(0);
4592 unsigned HalfSize = VT.getVectorNumElements()/2;
4594 unsigned FstHalf = 0, SndHalf = 0;
4595 for (unsigned i = 0; i < HalfSize; ++i) {
4596 if (SVOp->getMaskElt(i) > 0) {
4597 FstHalf = SVOp->getMaskElt(i)/HalfSize;
4601 for (unsigned i = HalfSize; i < HalfSize*2; ++i) {
4602 if (SVOp->getMaskElt(i) > 0) {
4603 SndHalf = SVOp->getMaskElt(i)/HalfSize;
4608 return (FstHalf | (SndHalf << 4));
4611 // Symmetric in-lane mask. Each lane has 4 elements (for imm8)
4612 static bool isPermImmMask(ArrayRef<int> Mask, MVT VT, unsigned& Imm8) {
4613 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4617 unsigned NumElts = VT.getVectorNumElements();
4619 if (VT.is128BitVector() || (VT.is256BitVector() && EltSize == 64)) {
4620 for (unsigned i = 0; i != NumElts; ++i) {
4623 Imm8 |= Mask[i] << (i*2);
4628 unsigned LaneSize = 4;
4629 SmallVector<int, 4> MaskVal(LaneSize, -1);
4631 for (unsigned l = 0; l != NumElts; l += LaneSize) {
4632 for (unsigned i = 0; i != LaneSize; ++i) {
4633 if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize))
4637 if (MaskVal[i] < 0) {
4638 MaskVal[i] = Mask[i+l] - l;
4639 Imm8 |= MaskVal[i] << (i*2);
4642 if (Mask[i+l] != (signed)(MaskVal[i]+l))
4649 /// isVPERMILPMask - Return true if the specified VECTOR_SHUFFLE operand
4650 /// specifies a shuffle of elements that is suitable for input to VPERMILPD*.
4651 /// Note that VPERMIL mask matching is different depending whether theunderlying
4652 /// type is 32 or 64. In the VPERMILPS the high half of the mask should point
4653 /// to the same elements of the low, but to the higher half of the source.
4654 /// In VPERMILPD the two lanes could be shuffled independently of each other
4655 /// with the same restriction that lanes can't be crossed. Also handles PSHUFDY.
4656 static bool isVPERMILPMask(ArrayRef<int> Mask, MVT VT) {
4657 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
4658 if (VT.getSizeInBits() < 256 || EltSize < 32)
4660 bool symmetricMaskRequired = (EltSize == 32);
4661 unsigned NumElts = VT.getVectorNumElements();
4663 unsigned NumLanes = VT.getSizeInBits()/128;
4664 unsigned LaneSize = NumElts/NumLanes;
4665 // 2 or 4 elements in one lane
4667 SmallVector<int, 4> ExpectedMaskVal(LaneSize, -1);
4668 for (unsigned l = 0; l != NumElts; l += LaneSize) {
4669 for (unsigned i = 0; i != LaneSize; ++i) {
4670 if (!isUndefOrInRange(Mask[i+l], l, l+LaneSize))
4672 if (symmetricMaskRequired) {
4673 if (ExpectedMaskVal[i] < 0 && Mask[i+l] >= 0) {
4674 ExpectedMaskVal[i] = Mask[i+l] - l;
4677 if (!isUndefOrEqual(Mask[i+l], ExpectedMaskVal[i]+l))
4685 /// isCommutedMOVLMask - Returns true if the shuffle mask is except the reverse
4686 /// of what x86 movss want. X86 movs requires the lowest element to be lowest
4687 /// element of vector 2 and the other elements to come from vector 1 in order.
4688 static bool isCommutedMOVLMask(ArrayRef<int> Mask, MVT VT,
4689 bool V2IsSplat = false, bool V2IsUndef = false) {
4690 if (!VT.is128BitVector())
4693 unsigned NumOps = VT.getVectorNumElements();
4694 if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
4697 if (!isUndefOrEqual(Mask[0], 0))
4700 for (unsigned i = 1; i != NumOps; ++i)
4701 if (!(isUndefOrEqual(Mask[i], i+NumOps) ||
4702 (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) ||
4703 (V2IsSplat && isUndefOrEqual(Mask[i], NumOps))))
4709 /// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
4710 /// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
4711 /// Masks to match: <1, 1, 3, 3> or <1, 1, 3, 3, 5, 5, 7, 7>
4712 static bool isMOVSHDUPMask(ArrayRef<int> Mask, MVT VT,
4713 const X86Subtarget *Subtarget) {
4714 if (!Subtarget->hasSSE3())
4717 unsigned NumElems = VT.getVectorNumElements();
4719 if ((VT.is128BitVector() && NumElems != 4) ||
4720 (VT.is256BitVector() && NumElems != 8) ||
4721 (VT.is512BitVector() && NumElems != 16))
4724 // "i+1" is the value the indexed mask element must have
4725 for (unsigned i = 0; i != NumElems; i += 2)
4726 if (!isUndefOrEqual(Mask[i], i+1) ||
4727 !isUndefOrEqual(Mask[i+1], i+1))
4733 /// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
4734 /// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
4735 /// Masks to match: <0, 0, 2, 2> or <0, 0, 2, 2, 4, 4, 6, 6>
4736 static bool isMOVSLDUPMask(ArrayRef<int> Mask, MVT VT,
4737 const X86Subtarget *Subtarget) {
4738 if (!Subtarget->hasSSE3())
4741 unsigned NumElems = VT.getVectorNumElements();
4743 if ((VT.is128BitVector() && NumElems != 4) ||
4744 (VT.is256BitVector() && NumElems != 8) ||
4745 (VT.is512BitVector() && NumElems != 16))
4748 // "i" is the value the indexed mask element must have
4749 for (unsigned i = 0; i != NumElems; i += 2)
4750 if (!isUndefOrEqual(Mask[i], i) ||
4751 !isUndefOrEqual(Mask[i+1], i))
4757 /// isMOVDDUPYMask - Return true if the specified VECTOR_SHUFFLE operand
4758 /// specifies a shuffle of elements that is suitable for input to 256-bit
4759 /// version of MOVDDUP.
4760 static bool isMOVDDUPYMask(ArrayRef<int> Mask, MVT VT, bool HasFp256) {
4761 if (!HasFp256 || !VT.is256BitVector())
4764 unsigned NumElts = VT.getVectorNumElements();
4768 for (unsigned i = 0; i != NumElts/2; ++i)
4769 if (!isUndefOrEqual(Mask[i], 0))
4771 for (unsigned i = NumElts/2; i != NumElts; ++i)
4772 if (!isUndefOrEqual(Mask[i], NumElts/2))
4777 /// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
4778 /// specifies a shuffle of elements that is suitable for input to 128-bit
4779 /// version of MOVDDUP.
4780 static bool isMOVDDUPMask(ArrayRef<int> Mask, MVT VT) {
4781 if (!VT.is128BitVector())
4784 unsigned e = VT.getVectorNumElements() / 2;
4785 for (unsigned i = 0; i != e; ++i)
4786 if (!isUndefOrEqual(Mask[i], i))
4788 for (unsigned i = 0; i != e; ++i)
4789 if (!isUndefOrEqual(Mask[e+i], i))
4794 /// isVEXTRACTIndex - Return true if the specified
4795 /// EXTRACT_SUBVECTOR operand specifies a vector extract that is
4796 /// suitable for instruction that extract 128 or 256 bit vectors
4797 static bool isVEXTRACTIndex(SDNode *N, unsigned vecWidth) {
4798 assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
4799 if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
4802 // The index should be aligned on a vecWidth-bit boundary.
4804 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
4806 MVT VT = N->getSimpleValueType(0);
4807 unsigned ElSize = VT.getVectorElementType().getSizeInBits();
4808 bool Result = (Index * ElSize) % vecWidth == 0;
4813 /// isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR
4814 /// operand specifies a subvector insert that is suitable for input to
4815 /// insertion of 128 or 256-bit subvectors
4816 static bool isVINSERTIndex(SDNode *N, unsigned vecWidth) {
4817 assert((vecWidth == 128 || vecWidth == 256) && "Unexpected vector width");
4818 if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
4820 // The index should be aligned on a vecWidth-bit boundary.
4822 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
4824 MVT VT = N->getSimpleValueType(0);
4825 unsigned ElSize = VT.getVectorElementType().getSizeInBits();
4826 bool Result = (Index * ElSize) % vecWidth == 0;
4831 bool X86::isVINSERT128Index(SDNode *N) {
4832 return isVINSERTIndex(N, 128);
4835 bool X86::isVINSERT256Index(SDNode *N) {
4836 return isVINSERTIndex(N, 256);
4839 bool X86::isVEXTRACT128Index(SDNode *N) {
4840 return isVEXTRACTIndex(N, 128);
4843 bool X86::isVEXTRACT256Index(SDNode *N) {
4844 return isVEXTRACTIndex(N, 256);
4847 /// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
4848 /// the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions.
4849 /// Handles 128-bit and 256-bit.
4850 static unsigned getShuffleSHUFImmediate(ShuffleVectorSDNode *N) {
4851 MVT VT = N->getSimpleValueType(0);
4853 assert((VT.getSizeInBits() >= 128) &&
4854 "Unsupported vector type for PSHUF/SHUFP");
4856 // Handle 128 and 256-bit vector lengths. AVX defines PSHUF/SHUFP to operate
4857 // independently on 128-bit lanes.
4858 unsigned NumElts = VT.getVectorNumElements();
4859 unsigned NumLanes = VT.getSizeInBits()/128;
4860 unsigned NumLaneElts = NumElts/NumLanes;
4862 assert((NumLaneElts == 2 || NumLaneElts == 4 || NumLaneElts == 8) &&
4863 "Only supports 2, 4 or 8 elements per lane");
4865 unsigned Shift = (NumLaneElts >= 4) ? 1 : 0;
4867 for (unsigned i = 0; i != NumElts; ++i) {
4868 int Elt = N->getMaskElt(i);
4869 if (Elt < 0) continue;
4870 Elt &= NumLaneElts - 1;
4871 unsigned ShAmt = (i << Shift) % 8;
4872 Mask |= Elt << ShAmt;
4878 /// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle
4879 /// the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction.
4880 static unsigned getShufflePSHUFHWImmediate(ShuffleVectorSDNode *N) {
4881 MVT VT = N->getSimpleValueType(0);
4883 assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
4884 "Unsupported vector type for PSHUFHW");
4886 unsigned NumElts = VT.getVectorNumElements();
4889 for (unsigned l = 0; l != NumElts; l += 8) {
4890 // 8 nodes per lane, but we only care about the last 4.
4891 for (unsigned i = 0; i < 4; ++i) {
4892 int Elt = N->getMaskElt(l+i+4);
4893 if (Elt < 0) continue;
4894 Elt &= 0x3; // only 2-bits.
4895 Mask |= Elt << (i * 2);
4902 /// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle
4903 /// the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction.
4904 static unsigned getShufflePSHUFLWImmediate(ShuffleVectorSDNode *N) {
4905 MVT VT = N->getSimpleValueType(0);
4907 assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
4908 "Unsupported vector type for PSHUFHW");
4910 unsigned NumElts = VT.getVectorNumElements();
4913 for (unsigned l = 0; l != NumElts; l += 8) {
4914 // 8 nodes per lane, but we only care about the first 4.
4915 for (unsigned i = 0; i < 4; ++i) {
4916 int Elt = N->getMaskElt(l+i);
4917 if (Elt < 0) continue;
4918 Elt &= 0x3; // only 2-bits
4919 Mask |= Elt << (i * 2);
4926 /// \brief Return the appropriate immediate to shuffle the specified
4927 /// VECTOR_SHUFFLE mask with the PALIGNR (if InterLane is false) or with
4928 /// VALIGN (if Interlane is true) instructions.
4929 static unsigned getShuffleAlignrImmediate(ShuffleVectorSDNode *SVOp,
4931 MVT VT = SVOp->getSimpleValueType(0);
4932 unsigned EltSize = InterLane ? 1 :
4933 VT.getVectorElementType().getSizeInBits() >> 3;
4935 unsigned NumElts = VT.getVectorNumElements();
4936 unsigned NumLanes = VT.is512BitVector() ? 1 : VT.getSizeInBits()/128;
4937 unsigned NumLaneElts = NumElts/NumLanes;
4941 for (i = 0; i != NumElts; ++i) {
4942 Val = SVOp->getMaskElt(i);
4946 if (Val >= (int)NumElts)
4947 Val -= NumElts - NumLaneElts;
4949 assert(Val - i > 0 && "PALIGNR imm should be positive");
4950 return (Val - i) * EltSize;
4953 /// \brief Return the appropriate immediate to shuffle the specified
4954 /// VECTOR_SHUFFLE mask with the PALIGNR instruction.
4955 static unsigned getShufflePALIGNRImmediate(ShuffleVectorSDNode *SVOp) {
4956 return getShuffleAlignrImmediate(SVOp, false);
4959 /// \brief Return the appropriate immediate to shuffle the specified
4960 /// VECTOR_SHUFFLE mask with the VALIGN instruction.
4961 static unsigned getShuffleVALIGNImmediate(ShuffleVectorSDNode *SVOp) {
4962 return getShuffleAlignrImmediate(SVOp, true);
4966 static unsigned getExtractVEXTRACTImmediate(SDNode *N, unsigned vecWidth) {
4967 assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width");
4968 if (!isa<ConstantSDNode>(N->getOperand(1).getNode()))
4969 llvm_unreachable("Illegal extract subvector for VEXTRACT");
4972 cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
4974 MVT VecVT = N->getOperand(0).getSimpleValueType();
4975 MVT ElVT = VecVT.getVectorElementType();
4977 unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
4978 return Index / NumElemsPerChunk;
4981 static unsigned getInsertVINSERTImmediate(SDNode *N, unsigned vecWidth) {
4982 assert((vecWidth == 128 || vecWidth == 256) && "Unsupported vector width");
4983 if (!isa<ConstantSDNode>(N->getOperand(2).getNode()))
4984 llvm_unreachable("Illegal insert subvector for VINSERT");
4987 cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
4989 MVT VecVT = N->getSimpleValueType(0);
4990 MVT ElVT = VecVT.getVectorElementType();
4992 unsigned NumElemsPerChunk = vecWidth / ElVT.getSizeInBits();
4993 return Index / NumElemsPerChunk;
4996 /// getExtractVEXTRACT128Immediate - Return the appropriate immediate
4997 /// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF128
4998 /// and VINSERTI128 instructions.
4999 unsigned X86::getExtractVEXTRACT128Immediate(SDNode *N) {
5000 return getExtractVEXTRACTImmediate(N, 128);
5003 /// getExtractVEXTRACT256Immediate - Return the appropriate immediate
5004 /// to extract the specified EXTRACT_SUBVECTOR index with VEXTRACTF64x4
5005 /// and VINSERTI64x4 instructions.
5006 unsigned X86::getExtractVEXTRACT256Immediate(SDNode *N) {
5007 return getExtractVEXTRACTImmediate(N, 256);
5010 /// getInsertVINSERT128Immediate - Return the appropriate immediate
5011 /// to insert at the specified INSERT_SUBVECTOR index with VINSERTF128
5012 /// and VINSERTI128 instructions.
5013 unsigned X86::getInsertVINSERT128Immediate(SDNode *N) {
5014 return getInsertVINSERTImmediate(N, 128);
5017 /// getInsertVINSERT256Immediate - Return the appropriate immediate
5018 /// to insert at the specified INSERT_SUBVECTOR index with VINSERTF46x4
5019 /// and VINSERTI64x4 instructions.
5020 unsigned X86::getInsertVINSERT256Immediate(SDNode *N) {
5021 return getInsertVINSERTImmediate(N, 256);
5024 /// isZero - Returns true if Elt is a constant integer zero
5025 static bool isZero(SDValue V) {
5026 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
5027 return C && C->isNullValue();
5030 /// isZeroNode - Returns true if Elt is a constant zero or a floating point
5032 bool X86::isZeroNode(SDValue Elt) {
5035 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Elt))
5036 return CFP->getValueAPF().isPosZero();
5040 /// ShouldXformToMOVHLPS - Return true if the node should be transformed to
5041 /// match movhlps. The lower half elements should come from upper half of
5042 /// V1 (and in order), and the upper half elements should come from the upper
5043 /// half of V2 (and in order).
5044 static bool ShouldXformToMOVHLPS(ArrayRef<int> Mask, MVT VT) {
5045 if (!VT.is128BitVector())
5047 if (VT.getVectorNumElements() != 4)
5049 for (unsigned i = 0, e = 2; i != e; ++i)
5050 if (!isUndefOrEqual(Mask[i], i+2))
5052 for (unsigned i = 2; i != 4; ++i)
5053 if (!isUndefOrEqual(Mask[i], i+4))
5058 /// isScalarLoadToVector - Returns true if the node is a scalar load that
5059 /// is promoted to a vector. It also returns the LoadSDNode by reference if
5061 static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = nullptr) {
5062 if (N->getOpcode() != ISD::SCALAR_TO_VECTOR)
5064 N = N->getOperand(0).getNode();
5065 if (!ISD::isNON_EXTLoad(N))
5068 *LD = cast<LoadSDNode>(N);
5072 // Test whether the given value is a vector value which will be legalized
5074 static bool WillBeConstantPoolLoad(SDNode *N) {
5075 if (N->getOpcode() != ISD::BUILD_VECTOR)
5078 // Check for any non-constant elements.
5079 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
5080 switch (N->getOperand(i).getNode()->getOpcode()) {
5082 case ISD::ConstantFP:
5089 // Vectors of all-zeros and all-ones are materialized with special
5090 // instructions rather than being loaded.
5091 return !ISD::isBuildVectorAllZeros(N) &&
5092 !ISD::isBuildVectorAllOnes(N);
5095 /// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to
5096 /// match movlp{s|d}. The lower half elements should come from lower half of
5097 /// V1 (and in order), and the upper half elements should come from the upper
5098 /// half of V2 (and in order). And since V1 will become the source of the
5099 /// MOVLP, it must be either a vector load or a scalar load to vector.
5100 static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
5101 ArrayRef<int> Mask, MVT VT) {
5102 if (!VT.is128BitVector())
5105 if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
5107 // Is V2 is a vector load, don't do this transformation. We will try to use
5108 // load folding shufps op.
5109 if (ISD::isNON_EXTLoad(V2) || WillBeConstantPoolLoad(V2))
5112 unsigned NumElems = VT.getVectorNumElements();
5114 if (NumElems != 2 && NumElems != 4)
5116 for (unsigned i = 0, e = NumElems/2; i != e; ++i)
5117 if (!isUndefOrEqual(Mask[i], i))
5119 for (unsigned i = NumElems/2, e = NumElems; i != e; ++i)
5120 if (!isUndefOrEqual(Mask[i], i+NumElems))
5125 /// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
5126 /// to an zero vector.
5127 /// FIXME: move to dag combiner / method on ShuffleVectorSDNode
5128 static bool isZeroShuffle(ShuffleVectorSDNode *N) {
5129 SDValue V1 = N->getOperand(0);
5130 SDValue V2 = N->getOperand(1);
5131 unsigned NumElems = N->getValueType(0).getVectorNumElements();
5132 for (unsigned i = 0; i != NumElems; ++i) {
5133 int Idx = N->getMaskElt(i);
5134 if (Idx >= (int)NumElems) {
5135 unsigned Opc = V2.getOpcode();
5136 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
5138 if (Opc != ISD::BUILD_VECTOR ||
5139 !X86::isZeroNode(V2.getOperand(Idx-NumElems)))
5141 } else if (Idx >= 0) {
5142 unsigned Opc = V1.getOpcode();
5143 if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode()))
5145 if (Opc != ISD::BUILD_VECTOR ||
5146 !X86::isZeroNode(V1.getOperand(Idx)))
5153 /// getZeroVector - Returns a vector of specified type with all zero elements.
5155 static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget,
5156 SelectionDAG &DAG, SDLoc dl) {
5157 assert(VT.isVector() && "Expected a vector type");
5159 // Always build SSE zero vectors as <4 x i32> bitcasted
5160 // to their dest type. This ensures they get CSE'd.
5162 if (VT.is128BitVector()) { // SSE
5163 if (Subtarget->hasSSE2()) { // SSE2
5164 SDValue Cst = DAG.getConstant(0, MVT::i32);
5165 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
5167 SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32);
5168 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst);
5170 } else if (VT.is256BitVector()) { // AVX
5171 if (Subtarget->hasInt256()) { // AVX2
5172 SDValue Cst = DAG.getConstant(0, MVT::i32);
5173 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5174 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
5176 // 256-bit logic and arithmetic instructions in AVX are all
5177 // floating-point, no support for integer ops. Emit fp zeroed vectors.
5178 SDValue Cst = DAG.getConstantFP(+0.0, MVT::f32);
5179 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5180 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops);
5182 } else if (VT.is512BitVector()) { // AVX-512
5183 SDValue Cst = DAG.getConstant(0, MVT::i32);
5184 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst,
5185 Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5186 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops);
5187 } else if (VT.getScalarType() == MVT::i1) {
5188 assert(VT.getVectorNumElements() <= 16 && "Unexpected vector type");
5189 SDValue Cst = DAG.getConstant(0, MVT::i1);
5190 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
5191 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
5193 llvm_unreachable("Unexpected vector type");
5195 return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
5198 /// getOnesVector - Returns a vector of specified type with all bits set.
5199 /// Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with
5200 /// no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately.
5201 /// Then bitcast to their original type, ensuring they get CSE'd.
5202 static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG,
5204 assert(VT.isVector() && "Expected a vector type");
5206 SDValue Cst = DAG.getConstant(~0U, MVT::i32);
5208 if (VT.is256BitVector()) {
5209 if (HasInt256) { // AVX2
5210 SDValue Ops[] = { Cst, Cst, Cst, Cst, Cst, Cst, Cst, Cst };
5211 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops);
5213 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
5214 Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl);
5216 } else if (VT.is128BitVector()) {
5217 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
5219 llvm_unreachable("Unexpected vector type");
5221 return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
5224 /// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements
5225 /// that point to V2 points to its first element.
5226 static void NormalizeMask(SmallVectorImpl<int> &Mask, unsigned NumElems) {
5227 for (unsigned i = 0; i != NumElems; ++i) {
5228 if (Mask[i] > (int)NumElems) {
5234 /// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
5235 /// operation of specified width.
5236 static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1,
5238 unsigned NumElems = VT.getVectorNumElements();
5239 SmallVector<int, 8> Mask;
5240 Mask.push_back(NumElems);
5241 for (unsigned i = 1; i != NumElems; ++i)
5243 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
5246 /// getUnpackl - Returns a vector_shuffle node for an unpackl operation.
5247 static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
5249 unsigned NumElems = VT.getVectorNumElements();
5250 SmallVector<int, 8> Mask;
5251 for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
5253 Mask.push_back(i + NumElems);
5255 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
5258 /// getUnpackh - Returns a vector_shuffle node for an unpackh operation.
5259 static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1,
5261 unsigned NumElems = VT.getVectorNumElements();
5262 SmallVector<int, 8> Mask;
5263 for (unsigned i = 0, Half = NumElems/2; i != Half; ++i) {
5264 Mask.push_back(i + Half);
5265 Mask.push_back(i + NumElems + Half);
5267 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
5270 // PromoteSplati8i16 - All i16 and i8 vector types can't be used directly by
5271 // a generic shuffle instruction because the target has no such instructions.
5272 // Generate shuffles which repeat i16 and i8 several times until they can be
5273 // represented by v4f32 and then be manipulated by target suported shuffles.
5274 static SDValue PromoteSplati8i16(SDValue V, SelectionDAG &DAG, int &EltNo) {
5275 MVT VT = V.getSimpleValueType();
5276 int NumElems = VT.getVectorNumElements();
5279 while (NumElems > 4) {
5280 if (EltNo < NumElems/2) {
5281 V = getUnpackl(DAG, dl, VT, V, V);
5283 V = getUnpackh(DAG, dl, VT, V, V);
5284 EltNo -= NumElems/2;
5291 /// getLegalSplat - Generate a legal splat with supported x86 shuffles
5292 static SDValue getLegalSplat(SelectionDAG &DAG, SDValue V, int EltNo) {
5293 MVT VT = V.getSimpleValueType();
5296 if (VT.is128BitVector()) {
5297 V = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V);
5298 int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
5299 V = DAG.getVectorShuffle(MVT::v4f32, dl, V, DAG.getUNDEF(MVT::v4f32),
5301 } else if (VT.is256BitVector()) {
5302 // To use VPERMILPS to splat scalars, the second half of indicies must
5303 // refer to the higher part, which is a duplication of the lower one,
5304 // because VPERMILPS can only handle in-lane permutations.
5305 int SplatMask[8] = { EltNo, EltNo, EltNo, EltNo,
5306 EltNo+4, EltNo+4, EltNo+4, EltNo+4 };
5308 V = DAG.getNode(ISD::BITCAST, dl, MVT::v8f32, V);
5309 V = DAG.getVectorShuffle(MVT::v8f32, dl, V, DAG.getUNDEF(MVT::v8f32),
5312 llvm_unreachable("Vector size not supported");
5314 return DAG.getNode(ISD::BITCAST, dl, VT, V);
5317 /// PromoteSplat - Splat is promoted to target supported vector shuffles.
5318 static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG) {
5319 MVT SrcVT = SV->getSimpleValueType(0);
5320 SDValue V1 = SV->getOperand(0);
5323 int EltNo = SV->getSplatIndex();
5324 int NumElems = SrcVT.getVectorNumElements();
5325 bool Is256BitVec = SrcVT.is256BitVector();
5327 assert(((SrcVT.is128BitVector() && NumElems > 4) || Is256BitVec) &&
5328 "Unknown how to promote splat for type");
5330 // Extract the 128-bit part containing the splat element and update
5331 // the splat element index when it refers to the higher register.
5333 V1 = Extract128BitVector(V1, EltNo, DAG, dl);
5334 if (EltNo >= NumElems/2)
5335 EltNo -= NumElems/2;
5338 // All i16 and i8 vector types can't be used directly by a generic shuffle
5339 // instruction because the target has no such instruction. Generate shuffles
5340 // which repeat i16 and i8 several times until they fit in i32, and then can
5341 // be manipulated by target suported shuffles.
5342 MVT EltVT = SrcVT.getVectorElementType();
5343 if (EltVT == MVT::i8 || EltVT == MVT::i16)
5344 V1 = PromoteSplati8i16(V1, DAG, EltNo);
5346 // Recreate the 256-bit vector and place the same 128-bit vector
5347 // into the low and high part. This is necessary because we want
5348 // to use VPERM* to shuffle the vectors
5350 V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, V1, V1);
5353 return getLegalSplat(DAG, V1, EltNo);
5356 /// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
5357 /// vector of zero or undef vector. This produces a shuffle where the low
5358 /// element of V2 is swizzled into the zero/undef vector, landing at element
5359 /// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
5360 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
5362 const X86Subtarget *Subtarget,
5363 SelectionDAG &DAG) {
5364 MVT VT = V2.getSimpleValueType();
5366 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
5367 unsigned NumElems = VT.getVectorNumElements();
5368 SmallVector<int, 16> MaskVec;
5369 for (unsigned i = 0; i != NumElems; ++i)
5370 // If this is the insertion idx, put the low elt of V2 here.
5371 MaskVec.push_back(i == Idx ? NumElems : i);
5372 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, &MaskVec[0]);
5375 /// getTargetShuffleMask - Calculates the shuffle mask corresponding to the
5376 /// target specific opcode. Returns true if the Mask could be calculated. Sets
5377 /// IsUnary to true if only uses one source. Note that this will set IsUnary for
5378 /// shuffles which use a single input multiple times, and in those cases it will
5379 /// adjust the mask to only have indices within that single input.
5380 static bool getTargetShuffleMask(SDNode *N, MVT VT,
5381 SmallVectorImpl<int> &Mask, bool &IsUnary) {
5382 unsigned NumElems = VT.getVectorNumElements();
5386 bool IsFakeUnary = false;
5387 switch(N->getOpcode()) {
5388 case X86ISD::BLENDI:
5389 ImmN = N->getOperand(N->getNumOperands()-1);
5390 DecodeBLENDMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5393 ImmN = N->getOperand(N->getNumOperands()-1);
5394 DecodeSHUFPMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5395 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5397 case X86ISD::UNPCKH:
5398 DecodeUNPCKHMask(VT, Mask);
5399 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5401 case X86ISD::UNPCKL:
5402 DecodeUNPCKLMask(VT, Mask);
5403 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5405 case X86ISD::MOVHLPS:
5406 DecodeMOVHLPSMask(NumElems, Mask);
5407 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5409 case X86ISD::MOVLHPS:
5410 DecodeMOVLHPSMask(NumElems, Mask);
5411 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
5413 case X86ISD::PALIGNR:
5414 ImmN = N->getOperand(N->getNumOperands()-1);
5415 DecodePALIGNRMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5417 case X86ISD::PSHUFD:
5418 case X86ISD::VPERMILPI:
5419 ImmN = N->getOperand(N->getNumOperands()-1);
5420 DecodePSHUFMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5423 case X86ISD::PSHUFHW:
5424 ImmN = N->getOperand(N->getNumOperands()-1);
5425 DecodePSHUFHWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5428 case X86ISD::PSHUFLW:
5429 ImmN = N->getOperand(N->getNumOperands()-1);
5430 DecodePSHUFLWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5433 case X86ISD::PSHUFB: {
5435 SDValue MaskNode = N->getOperand(1);
5436 while (MaskNode->getOpcode() == ISD::BITCAST)
5437 MaskNode = MaskNode->getOperand(0);
5439 if (MaskNode->getOpcode() == ISD::BUILD_VECTOR) {
5440 // If we have a build-vector, then things are easy.
5441 EVT VT = MaskNode.getValueType();
5442 assert(VT.isVector() &&
5443 "Can't produce a non-vector with a build_vector!");
5444 if (!VT.isInteger())
5447 int NumBytesPerElement = VT.getVectorElementType().getSizeInBits() / 8;
5449 SmallVector<uint64_t, 32> RawMask;
5450 for (int i = 0, e = MaskNode->getNumOperands(); i < e; ++i) {
5451 SDValue Op = MaskNode->getOperand(i);
5452 if (Op->getOpcode() == ISD::UNDEF) {
5453 RawMask.push_back((uint64_t)SM_SentinelUndef);
5456 auto *CN = dyn_cast<ConstantSDNode>(Op.getNode());
5459 APInt MaskElement = CN->getAPIntValue();
5461 // We now have to decode the element which could be any integer size and
5462 // extract each byte of it.
5463 for (int j = 0; j < NumBytesPerElement; ++j) {
5464 // Note that this is x86 and so always little endian: the low byte is
5465 // the first byte of the mask.
5466 RawMask.push_back(MaskElement.getLoBits(8).getZExtValue());
5467 MaskElement = MaskElement.lshr(8);
5470 DecodePSHUFBMask(RawMask, Mask);
5474 auto *MaskLoad = dyn_cast<LoadSDNode>(MaskNode);
5478 SDValue Ptr = MaskLoad->getBasePtr();
5479 if (Ptr->getOpcode() == X86ISD::Wrapper)
5480 Ptr = Ptr->getOperand(0);
5482 auto *MaskCP = dyn_cast<ConstantPoolSDNode>(Ptr);
5483 if (!MaskCP || MaskCP->isMachineConstantPoolEntry())
5486 if (auto *C = dyn_cast<Constant>(MaskCP->getConstVal())) {
5487 DecodePSHUFBMask(C, Mask);
5495 case X86ISD::VPERMI:
5496 ImmN = N->getOperand(N->getNumOperands()-1);
5497 DecodeVPERMMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5502 DecodeScalarMoveMask(VT, /* IsLoad */ false, Mask);
5504 case X86ISD::VPERM2X128:
5505 ImmN = N->getOperand(N->getNumOperands()-1);
5506 DecodeVPERM2X128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
5507 if (Mask.empty()) return false;
5509 case X86ISD::MOVSLDUP:
5510 DecodeMOVSLDUPMask(VT, Mask);
5513 case X86ISD::MOVSHDUP:
5514 DecodeMOVSHDUPMask(VT, Mask);
5517 case X86ISD::MOVDDUP:
5518 DecodeMOVDDUPMask(VT, Mask);
5521 case X86ISD::MOVLHPD:
5522 case X86ISD::MOVLPD:
5523 case X86ISD::MOVLPS:
5524 // Not yet implemented
5526 default: llvm_unreachable("unknown target shuffle node");
5529 // If we have a fake unary shuffle, the shuffle mask is spread across two
5530 // inputs that are actually the same node. Re-map the mask to always point
5531 // into the first input.
5534 if (M >= (int)Mask.size())
5540 /// getShuffleScalarElt - Returns the scalar element that will make up the ith
5541 /// element of the result of the vector shuffle.
5542 static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
5545 return SDValue(); // Limit search depth.
5547 SDValue V = SDValue(N, 0);
5548 EVT VT = V.getValueType();
5549 unsigned Opcode = V.getOpcode();
5551 // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
5552 if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
5553 int Elt = SV->getMaskElt(Index);
5556 return DAG.getUNDEF(VT.getVectorElementType());
5558 unsigned NumElems = VT.getVectorNumElements();
5559 SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0)
5560 : SV->getOperand(1);
5561 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1);
5564 // Recurse into target specific vector shuffles to find scalars.
5565 if (isTargetShuffle(Opcode)) {
5566 MVT ShufVT = V.getSimpleValueType();
5567 unsigned NumElems = ShufVT.getVectorNumElements();
5568 SmallVector<int, 16> ShuffleMask;
5571 if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary))
5574 int Elt = ShuffleMask[Index];
5576 return DAG.getUNDEF(ShufVT.getVectorElementType());
5578 SDValue NewV = (Elt < (int)NumElems) ? N->getOperand(0)
5580 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG,
5584 // Actual nodes that may contain scalar elements
5585 if (Opcode == ISD::BITCAST) {
5586 V = V.getOperand(0);
5587 EVT SrcVT = V.getValueType();
5588 unsigned NumElems = VT.getVectorNumElements();
5590 if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems)
5594 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
5595 return (Index == 0) ? V.getOperand(0)
5596 : DAG.getUNDEF(VT.getVectorElementType());
5598 if (V.getOpcode() == ISD::BUILD_VECTOR)
5599 return V.getOperand(Index);
5604 /// getNumOfConsecutiveZeros - Return the number of elements of a vector
5605 /// shuffle operation which come from a consecutively from a zero. The
5606 /// search can start in two different directions, from left or right.
5607 /// We count undefs as zeros until PreferredNum is reached.
5608 static unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp,
5609 unsigned NumElems, bool ZerosFromLeft,
5611 unsigned PreferredNum = -1U) {
5612 unsigned NumZeros = 0;
5613 for (unsigned i = 0; i != NumElems; ++i) {
5614 unsigned Index = ZerosFromLeft ? i : NumElems - i - 1;
5615 SDValue Elt = getShuffleScalarElt(SVOp, Index, DAG, 0);
5619 if (X86::isZeroNode(Elt))
5621 else if (Elt.getOpcode() == ISD::UNDEF) // Undef as zero up to PreferredNum.
5622 NumZeros = std::min(NumZeros + 1, PreferredNum);
5630 /// isShuffleMaskConsecutive - Check if the shuffle mask indicies [MaskI, MaskE)
5631 /// correspond consecutively to elements from one of the vector operands,
5632 /// starting from its index OpIdx. Also tell OpNum which source vector operand.
5634 bool isShuffleMaskConsecutive(ShuffleVectorSDNode *SVOp,
5635 unsigned MaskI, unsigned MaskE, unsigned OpIdx,
5636 unsigned NumElems, unsigned &OpNum) {
5637 bool SeenV1 = false;
5638 bool SeenV2 = false;
5640 for (unsigned i = MaskI; i != MaskE; ++i, ++OpIdx) {
5641 int Idx = SVOp->getMaskElt(i);
5642 // Ignore undef indicies
5646 if (Idx < (int)NumElems)
5651 // Only accept consecutive elements from the same vector
5652 if ((Idx % NumElems != OpIdx) || (SeenV1 && SeenV2))
5656 OpNum = SeenV1 ? 0 : 1;
5660 /// isVectorShiftRight - Returns true if the shuffle can be implemented as a
5661 /// logical left shift of a vector.
5662 static bool isVectorShiftRight(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
5663 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
5665 SVOp->getSimpleValueType(0).getVectorNumElements();
5666 unsigned NumZeros = getNumOfConsecutiveZeros(
5667 SVOp, NumElems, false /* check zeros from right */, DAG,
5668 SVOp->getMaskElt(0));
5674 // Considering the elements in the mask that are not consecutive zeros,
5675 // check if they consecutively come from only one of the source vectors.
5677 // V1 = {X, A, B, C} 0
5679 // vector_shuffle V1, V2 <1, 2, 3, X>
5681 if (!isShuffleMaskConsecutive(SVOp,
5682 0, // Mask Start Index
5683 NumElems-NumZeros, // Mask End Index(exclusive)
5684 NumZeros, // Where to start looking in the src vector
5685 NumElems, // Number of elements in vector
5686 OpSrc)) // Which source operand ?
5691 ShVal = SVOp->getOperand(OpSrc);
5695 /// isVectorShiftLeft - Returns true if the shuffle can be implemented as a
5696 /// logical left shift of a vector.
5697 static bool isVectorShiftLeft(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
5698 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
5700 SVOp->getSimpleValueType(0).getVectorNumElements();
5701 unsigned NumZeros = getNumOfConsecutiveZeros(
5702 SVOp, NumElems, true /* check zeros from left */, DAG,
5703 NumElems - SVOp->getMaskElt(NumElems - 1) - 1);
5709 // Considering the elements in the mask that are not consecutive zeros,
5710 // check if they consecutively come from only one of the source vectors.
5712 // 0 { A, B, X, X } = V2
5714 // vector_shuffle V1, V2 <X, X, 4, 5>
5716 if (!isShuffleMaskConsecutive(SVOp,
5717 NumZeros, // Mask Start Index
5718 NumElems, // Mask End Index(exclusive)
5719 0, // Where to start looking in the src vector
5720 NumElems, // Number of elements in vector
5721 OpSrc)) // Which source operand ?
5726 ShVal = SVOp->getOperand(OpSrc);
5730 /// isVectorShift - Returns true if the shuffle can be implemented as a
5731 /// logical left or right shift of a vector.
5732 static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
5733 bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
5734 // Although the logic below support any bitwidth size, there are no
5735 // shift instructions which handle more than 128-bit vectors.
5736 if (!SVOp->getSimpleValueType(0).is128BitVector())
5739 if (isVectorShiftLeft(SVOp, DAG, isLeft, ShVal, ShAmt) ||
5740 isVectorShiftRight(SVOp, DAG, isLeft, ShVal, ShAmt))
5746 /// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
5748 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
5749 unsigned NumNonZero, unsigned NumZero,
5751 const X86Subtarget* Subtarget,
5752 const TargetLowering &TLI) {
5759 for (unsigned i = 0; i < 16; ++i) {
5760 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
5761 if (ThisIsNonZero && First) {
5763 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
5765 V = DAG.getUNDEF(MVT::v8i16);
5770 SDValue ThisElt, LastElt;
5771 bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
5772 if (LastIsNonZero) {
5773 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl,
5774 MVT::i16, Op.getOperand(i-1));
5776 if (ThisIsNonZero) {
5777 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
5778 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16,
5779 ThisElt, DAG.getConstant(8, MVT::i8));
5781 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt);
5785 if (ThisElt.getNode())
5786 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt,
5787 DAG.getIntPtrConstant(i/2));
5791 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V);
5794 /// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.
5796 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
5797 unsigned NumNonZero, unsigned NumZero,
5799 const X86Subtarget* Subtarget,
5800 const TargetLowering &TLI) {
5807 for (unsigned i = 0; i < 8; ++i) {
5808 bool isNonZero = (NonZeros & (1 << i)) != 0;
5812 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
5814 V = DAG.getUNDEF(MVT::v8i16);
5817 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
5818 MVT::v8i16, V, Op.getOperand(i),
5819 DAG.getIntPtrConstant(i));
5826 /// LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32.
5827 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
5828 const X86Subtarget *Subtarget,
5829 const TargetLowering &TLI) {
5830 // Find all zeroable elements.
5832 for (int i=0; i < 4; ++i) {
5833 SDValue Elt = Op->getOperand(i);
5834 Zeroable[i] = (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt));
5836 assert(std::count_if(&Zeroable[0], &Zeroable[4],
5837 [](bool M) { return !M; }) > 1 &&
5838 "We expect at least two non-zero elements!");
5840 // We only know how to deal with build_vector nodes where elements are either
5841 // zeroable or extract_vector_elt with constant index.
5842 SDValue FirstNonZero;
5843 unsigned FirstNonZeroIdx;
5844 for (unsigned i=0; i < 4; ++i) {
5847 SDValue Elt = Op->getOperand(i);
5848 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5849 !isa<ConstantSDNode>(Elt.getOperand(1)))
5851 // Make sure that this node is extracting from a 128-bit vector.
5852 MVT VT = Elt.getOperand(0).getSimpleValueType();
5853 if (!VT.is128BitVector())
5855 if (!FirstNonZero.getNode()) {
5857 FirstNonZeroIdx = i;
5861 assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!");
5862 SDValue V1 = FirstNonZero.getOperand(0);
5863 MVT VT = V1.getSimpleValueType();
5865 // See if this build_vector can be lowered as a blend with zero.
5867 unsigned EltMaskIdx, EltIdx;
5869 for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
5870 if (Zeroable[EltIdx]) {
5871 // The zero vector will be on the right hand side.
5872 Mask[EltIdx] = EltIdx+4;
5876 Elt = Op->getOperand(EltIdx);
5877 // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index.
5878 EltMaskIdx = cast<ConstantSDNode>(Elt.getOperand(1))->getZExtValue();
5879 if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx)
5881 Mask[EltIdx] = EltIdx;
5885 // Let the shuffle legalizer deal with blend operations.
5886 SDValue VZero = getZeroVector(VT, Subtarget, DAG, SDLoc(Op));
5887 if (V1.getSimpleValueType() != VT)
5888 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), VT, V1);
5889 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZero, &Mask[0]);
5892 // See if we can lower this build_vector to a INSERTPS.
5893 if (!Subtarget->hasSSE41())
5896 SDValue V2 = Elt.getOperand(0);
5897 if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx)
5900 bool CanFold = true;
5901 for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) {
5905 SDValue Current = Op->getOperand(i);
5906 SDValue SrcVector = Current->getOperand(0);
5909 CanFold = SrcVector == V1 &&
5910 cast<ConstantSDNode>(Current.getOperand(1))->getZExtValue() == i;
5916 assert(V1.getNode() && "Expected at least two non-zero elements!");
5917 if (V1.getSimpleValueType() != MVT::v4f32)
5918 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1);
5919 if (V2.getSimpleValueType() != MVT::v4f32)
5920 V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2);
5922 // Ok, we can emit an INSERTPS instruction.
5924 for (int i = 0; i < 4; ++i)
5928 unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
5929 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
5930 SDValue Result = DAG.getNode(X86ISD::INSERTPS, SDLoc(Op), MVT::v4f32, V1, V2,
5931 DAG.getIntPtrConstant(InsertPSMask));
5932 return DAG.getNode(ISD::BITCAST, SDLoc(Op), VT, Result);
5935 /// Return a vector logical shift node.
5936 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
5937 unsigned NumBits, SelectionDAG &DAG,
5938 const TargetLowering &TLI, SDLoc dl) {
5939 assert(VT.is128BitVector() && "Unknown type for VShift");
5940 MVT ShVT = MVT::v2i64;
5941 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
5942 SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp);
5943 MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(SrcOp.getValueType());
5944 assert(NumBits % 8 == 0 && "Only support byte sized shifts");
5945 SDValue ShiftVal = DAG.getConstant(NumBits/8, ScalarShiftTy);
5946 return DAG.getNode(ISD::BITCAST, dl, VT,
5947 DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
5951 LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) {
5953 // Check if the scalar load can be widened into a vector load. And if
5954 // the address is "base + cst" see if the cst can be "absorbed" into
5955 // the shuffle mask.
5956 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
5957 SDValue Ptr = LD->getBasePtr();
5958 if (!ISD::isNormalLoad(LD) || LD->isVolatile())
5960 EVT PVT = LD->getValueType(0);
5961 if (PVT != MVT::i32 && PVT != MVT::f32)
5966 if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
5967 FI = FINode->getIndex();
5969 } else if (DAG.isBaseWithConstantOffset(Ptr) &&
5970 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
5971 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
5972 Offset = Ptr.getConstantOperandVal(1);
5973 Ptr = Ptr.getOperand(0);
5978 // FIXME: 256-bit vector instructions don't require a strict alignment,
5979 // improve this code to support it better.
5980 unsigned RequiredAlign = VT.getSizeInBits()/8;
5981 SDValue Chain = LD->getChain();
5982 // Make sure the stack object alignment is at least 16 or 32.
5983 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
5984 if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) {
5985 if (MFI->isFixedObjectIndex(FI)) {
5986 // Can't change the alignment. FIXME: It's possible to compute
5987 // the exact stack offset and reference FI + adjust offset instead.
5988 // If someone *really* cares about this. That's the way to implement it.
5991 MFI->setObjectAlignment(FI, RequiredAlign);
5995 // (Offset % 16 or 32) must be multiple of 4. Then address is then
5996 // Ptr + (Offset & ~15).
5999 if ((Offset % RequiredAlign) & 3)
6001 int64_t StartOffset = Offset & ~(RequiredAlign-1);
6003 Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(),
6004 Ptr,DAG.getConstant(StartOffset, Ptr.getValueType()));
6006 int EltNo = (Offset - StartOffset) >> 2;
6007 unsigned NumElems = VT.getVectorNumElements();
6009 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
6010 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
6011 LD->getPointerInfo().getWithOffset(StartOffset),
6012 false, false, false, 0);
6014 SmallVector<int, 8> Mask(NumElems, EltNo);
6016 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), &Mask[0]);
6022 /// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
6023 /// elements can be replaced by a single large load which has the same value as
6024 /// a build_vector or insert_subvector whose loaded operands are 'Elts'.
6026 /// Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a
6028 /// FIXME: we'd also like to handle the case where the last elements are zero
6029 /// rather than undef via VZEXT_LOAD, but we do not detect that case today.
6030 /// There's even a handy isZeroNode for that purpose.
6031 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
6032 SDLoc &DL, SelectionDAG &DAG,
6033 bool isAfterLegalize) {
6034 unsigned NumElems = Elts.size();
6036 LoadSDNode *LDBase = nullptr;
6037 unsigned LastLoadedElt = -1U;
6039 // For each element in the initializer, see if we've found a load or an undef.
6040 // If we don't find an initial load element, or later load elements are
6041 // non-consecutive, bail out.
6042 for (unsigned i = 0; i < NumElems; ++i) {
6043 SDValue Elt = Elts[i];
6044 // Look through a bitcast.
6045 if (Elt.getNode() && Elt.getOpcode() == ISD::BITCAST)
6046 Elt = Elt.getOperand(0);
6047 if (!Elt.getNode() ||
6048 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode())))
6051 if (Elt.getNode()->getOpcode() == ISD::UNDEF)
6053 LDBase = cast<LoadSDNode>(Elt.getNode());
6057 if (Elt.getOpcode() == ISD::UNDEF)
6060 LoadSDNode *LD = cast<LoadSDNode>(Elt);
6061 EVT LdVT = Elt.getValueType();
6062 // Each loaded element must be the correct fractional portion of the
6063 // requested vector load.
6064 if (LdVT.getSizeInBits() != VT.getSizeInBits() / NumElems)
6066 if (!DAG.isConsecutiveLoad(LD, LDBase, LdVT.getSizeInBits() / 8, i))
6071 // If we have found an entire vector of loads and undefs, then return a large
6072 // load of the entire vector width starting at the base pointer. If we found
6073 // consecutive loads for the low half, generate a vzext_load node.
6074 if (LastLoadedElt == NumElems - 1) {
6075 assert(LDBase && "Did not find base load for merging consecutive loads");
6076 EVT EltVT = LDBase->getValueType(0);
6077 // Ensure that the input vector size for the merged loads matches the
6078 // cumulative size of the input elements.
6079 if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems)
6082 if (isAfterLegalize &&
6083 !DAG.getTargetLoweringInfo().isOperationLegal(ISD::LOAD, VT))
6086 SDValue NewLd = SDValue();
6088 NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
6089 LDBase->getPointerInfo(), LDBase->isVolatile(),
6090 LDBase->isNonTemporal(), LDBase->isInvariant(),
6091 LDBase->getAlignment());
6093 if (LDBase->hasAnyUseOfValue(1)) {
6094 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
6096 SDValue(NewLd.getNode(), 1));
6097 DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
6098 DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
6099 SDValue(NewLd.getNode(), 1));
6105 //TODO: The code below fires only for for loading the low v2i32 / v2f32
6106 //of a v4i32 / v4f32. It's probably worth generalizing.
6107 EVT EltVT = VT.getVectorElementType();
6108 if (NumElems == 4 && LastLoadedElt == 1 && (EltVT.getSizeInBits() == 32) &&
6109 DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) {
6110 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
6111 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
6113 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64,
6114 LDBase->getPointerInfo(),
6115 LDBase->getAlignment(),
6116 false/*isVolatile*/, true/*ReadMem*/,
6119 // Make sure the newly-created LOAD is in the same position as LDBase in
6120 // terms of dependency. We create a TokenFactor for LDBase and ResNode, and
6121 // update uses of LDBase's output chain to use the TokenFactor.
6122 if (LDBase->hasAnyUseOfValue(1)) {
6123 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
6124 SDValue(LDBase, 1), SDValue(ResNode.getNode(), 1));
6125 DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
6126 DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
6127 SDValue(ResNode.getNode(), 1));
6130 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
6135 /// LowerVectorBroadcast - Attempt to use the vbroadcast instruction
6136 /// to generate a splat value for the following cases:
6137 /// 1. A splat BUILD_VECTOR which uses a single scalar load, or a constant.
6138 /// 2. A splat shuffle which uses a scalar_to_vector node which comes from
6139 /// a scalar load, or a constant.
6140 /// The VBROADCAST node is returned when a pattern is found,
6141 /// or SDValue() otherwise.
6142 static SDValue LowerVectorBroadcast(SDValue Op, const X86Subtarget* Subtarget,
6143 SelectionDAG &DAG) {
6144 // VBROADCAST requires AVX.
6145 // TODO: Splats could be generated for non-AVX CPUs using SSE
6146 // instructions, but there's less potential gain for only 128-bit vectors.
6147 if (!Subtarget->hasAVX())
6150 MVT VT = Op.getSimpleValueType();
6153 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&
6154 "Unsupported vector type for broadcast.");
6159 switch (Op.getOpcode()) {
6161 // Unknown pattern found.
6164 case ISD::BUILD_VECTOR: {
6165 auto *BVOp = cast<BuildVectorSDNode>(Op.getNode());
6166 BitVector UndefElements;
6167 SDValue Splat = BVOp->getSplatValue(&UndefElements);
6169 // We need a splat of a single value to use broadcast, and it doesn't
6170 // make any sense if the value is only in one element of the vector.
6171 if (!Splat || (VT.getVectorNumElements() - UndefElements.count()) <= 1)
6175 ConstSplatVal = (Ld.getOpcode() == ISD::Constant ||
6176 Ld.getOpcode() == ISD::ConstantFP);
6178 // Make sure that all of the users of a non-constant load are from the
6179 // BUILD_VECTOR node.
6180 if (!ConstSplatVal && !BVOp->isOnlyUserOf(Ld.getNode()))
6185 case ISD::VECTOR_SHUFFLE: {
6186 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
6188 // Shuffles must have a splat mask where the first element is
6190 if ((!SVOp->isSplat()) || SVOp->getMaskElt(0) != 0)
6193 SDValue Sc = Op.getOperand(0);
6194 if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR &&
6195 Sc.getOpcode() != ISD::BUILD_VECTOR) {
6197 if (!Subtarget->hasInt256())
6200 // Use the register form of the broadcast instruction available on AVX2.
6201 if (VT.getSizeInBits() >= 256)
6202 Sc = Extract128BitVector(Sc, 0, DAG, dl);
6203 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc);
6206 Ld = Sc.getOperand(0);
6207 ConstSplatVal = (Ld.getOpcode() == ISD::Constant ||
6208 Ld.getOpcode() == ISD::ConstantFP);
6210 // The scalar_to_vector node and the suspected
6211 // load node must have exactly one user.
6212 // Constants may have multiple users.
6214 // AVX-512 has register version of the broadcast
6215 bool hasRegVer = Subtarget->hasAVX512() && VT.is512BitVector() &&
6216 Ld.getValueType().getSizeInBits() >= 32;
6217 if (!ConstSplatVal && ((!Sc.hasOneUse() || !Ld.hasOneUse()) &&
6224 unsigned ScalarSize = Ld.getValueType().getSizeInBits();
6225 bool IsGE256 = (VT.getSizeInBits() >= 256);
6227 // When optimizing for size, generate up to 5 extra bytes for a broadcast
6228 // instruction to save 8 or more bytes of constant pool data.
6229 // TODO: If multiple splats are generated to load the same constant,
6230 // it may be detrimental to overall size. There needs to be a way to detect
6231 // that condition to know if this is truly a size win.
6232 const Function *F = DAG.getMachineFunction().getFunction();
6233 bool OptForSize = F->hasFnAttribute(Attribute::OptimizeForSize);
6235 // Handle broadcasting a single constant scalar from the constant pool
6237 // On Sandybridge (no AVX2), it is still better to load a constant vector
6238 // from the constant pool and not to broadcast it from a scalar.
6239 // But override that restriction when optimizing for size.
6240 // TODO: Check if splatting is recommended for other AVX-capable CPUs.
6241 if (ConstSplatVal && (Subtarget->hasAVX2() || OptForSize)) {
6242 EVT CVT = Ld.getValueType();
6243 assert(!CVT.isVector() && "Must not broadcast a vector type");
6245 // Splat f32, i32, v4f64, v4i64 in all cases with AVX2.
6246 // For size optimization, also splat v2f64 and v2i64, and for size opt
6247 // with AVX2, also splat i8 and i16.
6248 // With pattern matching, the VBROADCAST node may become a VMOVDDUP.
6249 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
6250 (OptForSize && (ScalarSize == 64 || Subtarget->hasAVX2()))) {
6251 const Constant *C = nullptr;
6252 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
6253 C = CI->getConstantIntValue();
6254 else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
6255 C = CF->getConstantFPValue();
6257 assert(C && "Invalid constant type");
6259 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6260 SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
6261 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
6262 Ld = DAG.getLoad(CVT, dl, DAG.getEntryNode(), CP,
6263 MachinePointerInfo::getConstantPool(),
6264 false, false, false, Alignment);
6266 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6270 bool IsLoad = ISD::isNormalLoad(Ld.getNode());
6272 // Handle AVX2 in-register broadcasts.
6273 if (!IsLoad && Subtarget->hasInt256() &&
6274 (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
6275 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6277 // The scalar source must be a normal load.
6281 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
6282 (Subtarget->hasVLX() && ScalarSize == 64))
6283 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6285 // The integer check is needed for the 64-bit into 128-bit so it doesn't match
6286 // double since there is no vbroadcastsd xmm
6287 if (Subtarget->hasInt256() && Ld.getValueType().isInteger()) {
6288 if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)
6289 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6292 // Unsupported broadcast.
6296 /// \brief For an EXTRACT_VECTOR_ELT with a constant index return the real
6297 /// underlying vector and index.
6299 /// Modifies \p ExtractedFromVec to the real vector and returns the real
6301 static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
6303 int Idx = cast<ConstantSDNode>(ExtIdx)->getZExtValue();
6304 if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
6307 // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
6309 // (extract_vector_elt (v8f32 %vreg1), Constant<6>)
6311 // (extract_vector_elt (vector_shuffle<2,u,u,u>
6312 // (extract_subvector (v8f32 %vreg0), Constant<4>),
6315 // In this case the vector is the extract_subvector expression and the index
6316 // is 2, as specified by the shuffle.
6317 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
6318 SDValue ShuffleVec = SVOp->getOperand(0);
6319 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
6320 assert(ShuffleVecVT.getVectorElementType() ==
6321 ExtractedFromVec.getSimpleValueType().getVectorElementType());
6323 int ShuffleIdx = SVOp->getMaskElt(Idx);
6324 if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
6325 ExtractedFromVec = ShuffleVec;
6331 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
6332 MVT VT = Op.getSimpleValueType();
6334 // Skip if insert_vec_elt is not supported.
6335 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6336 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
6340 unsigned NumElems = Op.getNumOperands();
6344 SmallVector<unsigned, 4> InsertIndices;
6345 SmallVector<int, 8> Mask(NumElems, -1);
6347 for (unsigned i = 0; i != NumElems; ++i) {
6348 unsigned Opc = Op.getOperand(i).getOpcode();
6350 if (Opc == ISD::UNDEF)
6353 if (Opc != ISD::EXTRACT_VECTOR_ELT) {
6354 // Quit if more than 1 elements need inserting.
6355 if (InsertIndices.size() > 1)
6358 InsertIndices.push_back(i);
6362 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
6363 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
6364 // Quit if non-constant index.
6365 if (!isa<ConstantSDNode>(ExtIdx))
6367 int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
6369 // Quit if extracted from vector of different type.
6370 if (ExtractedFromVec.getValueType() != VT)
6373 if (!VecIn1.getNode())
6374 VecIn1 = ExtractedFromVec;
6375 else if (VecIn1 != ExtractedFromVec) {
6376 if (!VecIn2.getNode())
6377 VecIn2 = ExtractedFromVec;
6378 else if (VecIn2 != ExtractedFromVec)
6379 // Quit if more than 2 vectors to shuffle
6383 if (ExtractedFromVec == VecIn1)
6385 else if (ExtractedFromVec == VecIn2)
6386 Mask[i] = Idx + NumElems;
6389 if (!VecIn1.getNode())
6392 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
6393 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]);
6394 for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) {
6395 unsigned Idx = InsertIndices[i];
6396 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
6397 DAG.getIntPtrConstant(Idx));
6403 // Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
6405 X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const {
6407 MVT VT = Op.getSimpleValueType();
6408 assert((VT.getVectorElementType() == MVT::i1) && (VT.getSizeInBits() <= 16) &&
6409 "Unexpected type in LowerBUILD_VECTORvXi1!");
6412 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
6413 SDValue Cst = DAG.getTargetConstant(0, MVT::i1);
6414 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
6415 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
6418 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
6419 SDValue Cst = DAG.getTargetConstant(1, MVT::i1);
6420 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Cst);
6421 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
6424 bool AllContants = true;
6425 uint64_t Immediate = 0;
6426 int NonConstIdx = -1;
6427 bool IsSplat = true;
6428 unsigned NumNonConsts = 0;
6429 unsigned NumConsts = 0;
6430 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
6431 SDValue In = Op.getOperand(idx);
6432 if (In.getOpcode() == ISD::UNDEF)
6434 if (!isa<ConstantSDNode>(In)) {
6435 AllContants = false;
6440 if (cast<ConstantSDNode>(In)->getZExtValue())
6441 Immediate |= (1ULL << idx);
6443 if (In != Op.getOperand(0))
6448 SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1,
6449 DAG.getConstant(Immediate, MVT::i16));
6450 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, FullMask,
6451 DAG.getIntPtrConstant(0));
6454 if (NumNonConsts == 1 && NonConstIdx != 0) {
6457 SDValue VecAsImm = DAG.getConstant(Immediate,
6458 MVT::getIntegerVT(VT.getSizeInBits()));
6459 DstVec = DAG.getNode(ISD::BITCAST, dl, VT, VecAsImm);
6462 DstVec = DAG.getUNDEF(VT);
6463 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
6464 Op.getOperand(NonConstIdx),
6465 DAG.getIntPtrConstant(NonConstIdx));
6467 if (!IsSplat && (NonConstIdx != 0))
6468 llvm_unreachable("Unsupported BUILD_VECTOR operation");
6469 MVT SelectVT = (VT == MVT::v16i1)? MVT::i16 : MVT::i8;
6472 Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0),
6473 DAG.getConstant(-1, SelectVT),
6474 DAG.getConstant(0, SelectVT));
6476 Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0),
6477 DAG.getConstant((Immediate | 1), SelectVT),
6478 DAG.getConstant(Immediate, SelectVT));
6479 return DAG.getNode(ISD::BITCAST, dl, VT, Select);
6482 /// \brief Return true if \p N implements a horizontal binop and return the
6483 /// operands for the horizontal binop into V0 and V1.
6485 /// This is a helper function of PerformBUILD_VECTORCombine.
6486 /// This function checks that the build_vector \p N in input implements a
6487 /// horizontal operation. Parameter \p Opcode defines the kind of horizontal
6488 /// operation to match.
6489 /// For example, if \p Opcode is equal to ISD::ADD, then this function
6490 /// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
6491 /// is equal to ISD::SUB, then this function checks if this is a horizontal
6494 /// This function only analyzes elements of \p N whose indices are
6495 /// in range [BaseIdx, LastIdx).
6496 static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode,
6498 unsigned BaseIdx, unsigned LastIdx,
6499 SDValue &V0, SDValue &V1) {
6500 EVT VT = N->getValueType(0);
6502 assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
6503 assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
6504 "Invalid Vector in input!");
6506 bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
6507 bool CanFold = true;
6508 unsigned ExpectedVExtractIdx = BaseIdx;
6509 unsigned NumElts = LastIdx - BaseIdx;
6510 V0 = DAG.getUNDEF(VT);
6511 V1 = DAG.getUNDEF(VT);
6513 // Check if N implements a horizontal binop.
6514 for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
6515 SDValue Op = N->getOperand(i + BaseIdx);
6518 if (Op->getOpcode() == ISD::UNDEF) {
6519 // Update the expected vector extract index.
6520 if (i * 2 == NumElts)
6521 ExpectedVExtractIdx = BaseIdx;
6522 ExpectedVExtractIdx += 2;
6526 CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
6531 SDValue Op0 = Op.getOperand(0);
6532 SDValue Op1 = Op.getOperand(1);
6534 // Try to match the following pattern:
6535 // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
6536 CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6537 Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6538 Op0.getOperand(0) == Op1.getOperand(0) &&
6539 isa<ConstantSDNode>(Op0.getOperand(1)) &&
6540 isa<ConstantSDNode>(Op1.getOperand(1)));
6544 unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
6545 unsigned I1 = cast<ConstantSDNode>(Op1.getOperand(1))->getZExtValue();
6547 if (i * 2 < NumElts) {
6548 if (V0.getOpcode() == ISD::UNDEF)
6549 V0 = Op0.getOperand(0);
6551 if (V1.getOpcode() == ISD::UNDEF)
6552 V1 = Op0.getOperand(0);
6553 if (i * 2 == NumElts)
6554 ExpectedVExtractIdx = BaseIdx;
6557 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
6558 if (I0 == ExpectedVExtractIdx)
6559 CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
6560 else if (IsCommutable && I1 == ExpectedVExtractIdx) {
6561 // Try to match the following dag sequence:
6562 // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
6563 CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
6567 ExpectedVExtractIdx += 2;
6573 /// \brief Emit a sequence of two 128-bit horizontal add/sub followed by
6574 /// a concat_vector.
6576 /// This is a helper function of PerformBUILD_VECTORCombine.
6577 /// This function expects two 256-bit vectors called V0 and V1.
6578 /// At first, each vector is split into two separate 128-bit vectors.
6579 /// Then, the resulting 128-bit vectors are used to implement two
6580 /// horizontal binary operations.
6582 /// The kind of horizontal binary operation is defined by \p X86Opcode.
6584 /// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
6585 /// the two new horizontal binop.
6586 /// When Mode is set, the first horizontal binop dag node would take as input
6587 /// the lower 128-bit of V0 and the upper 128-bit of V0. The second
6588 /// horizontal binop dag node would take as input the lower 128-bit of V1
6589 /// and the upper 128-bit of V1.
6591 /// HADD V0_LO, V0_HI
6592 /// HADD V1_LO, V1_HI
6594 /// Otherwise, the first horizontal binop dag node takes as input the lower
6595 /// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
6596 /// dag node takes the the upper 128-bit of V0 and the upper 128-bit of V1.
6598 /// HADD V0_LO, V1_LO
6599 /// HADD V0_HI, V1_HI
6601 /// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
6602 /// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
6603 /// the upper 128-bits of the result.
6604 static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
6605 SDLoc DL, SelectionDAG &DAG,
6606 unsigned X86Opcode, bool Mode,
6607 bool isUndefLO, bool isUndefHI) {
6608 EVT VT = V0.getValueType();
6609 assert(VT.is256BitVector() && VT == V1.getValueType() &&
6610 "Invalid nodes in input!");
6612 unsigned NumElts = VT.getVectorNumElements();
6613 SDValue V0_LO = Extract128BitVector(V0, 0, DAG, DL);
6614 SDValue V0_HI = Extract128BitVector(V0, NumElts/2, DAG, DL);
6615 SDValue V1_LO = Extract128BitVector(V1, 0, DAG, DL);
6616 SDValue V1_HI = Extract128BitVector(V1, NumElts/2, DAG, DL);
6617 EVT NewVT = V0_LO.getValueType();
6619 SDValue LO = DAG.getUNDEF(NewVT);
6620 SDValue HI = DAG.getUNDEF(NewVT);
6623 // Don't emit a horizontal binop if the result is expected to be UNDEF.
6624 if (!isUndefLO && V0->getOpcode() != ISD::UNDEF)
6625 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
6626 if (!isUndefHI && V1->getOpcode() != ISD::UNDEF)
6627 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
6629 // Don't emit a horizontal binop if the result is expected to be UNDEF.
6630 if (!isUndefLO && (V0_LO->getOpcode() != ISD::UNDEF ||
6631 V1_LO->getOpcode() != ISD::UNDEF))
6632 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
6634 if (!isUndefHI && (V0_HI->getOpcode() != ISD::UNDEF ||
6635 V1_HI->getOpcode() != ISD::UNDEF))
6636 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
6639 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
6642 /// \brief Try to fold a build_vector that performs an 'addsub' into the
6643 /// sequence of 'vadd + vsub + blendi'.
6644 static SDValue matchAddSub(const BuildVectorSDNode *BV, SelectionDAG &DAG,
6645 const X86Subtarget *Subtarget) {
6647 EVT VT = BV->getValueType(0);
6648 unsigned NumElts = VT.getVectorNumElements();
6649 SDValue InVec0 = DAG.getUNDEF(VT);
6650 SDValue InVec1 = DAG.getUNDEF(VT);
6652 assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 ||
6653 VT == MVT::v2f64) && "build_vector with an invalid type found!");
6655 // Odd-numbered elements in the input build vector are obtained from
6656 // adding two integer/float elements.
6657 // Even-numbered elements in the input build vector are obtained from
6658 // subtracting two integer/float elements.
6659 unsigned ExpectedOpcode = ISD::FSUB;
6660 unsigned NextExpectedOpcode = ISD::FADD;
6661 bool AddFound = false;
6662 bool SubFound = false;
6664 for (unsigned i = 0, e = NumElts; i != e; ++i) {
6665 SDValue Op = BV->getOperand(i);
6667 // Skip 'undef' values.
6668 unsigned Opcode = Op.getOpcode();
6669 if (Opcode == ISD::UNDEF) {
6670 std::swap(ExpectedOpcode, NextExpectedOpcode);
6674 // Early exit if we found an unexpected opcode.
6675 if (Opcode != ExpectedOpcode)
6678 SDValue Op0 = Op.getOperand(0);
6679 SDValue Op1 = Op.getOperand(1);
6681 // Try to match the following pattern:
6682 // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
6683 // Early exit if we cannot match that sequence.
6684 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6685 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6686 !isa<ConstantSDNode>(Op0.getOperand(1)) ||
6687 !isa<ConstantSDNode>(Op1.getOperand(1)) ||
6688 Op0.getOperand(1) != Op1.getOperand(1))
6691 unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
6695 // We found a valid add/sub node. Update the information accordingly.
6701 // Update InVec0 and InVec1.
6702 if (InVec0.getOpcode() == ISD::UNDEF)
6703 InVec0 = Op0.getOperand(0);
6704 if (InVec1.getOpcode() == ISD::UNDEF)
6705 InVec1 = Op1.getOperand(0);
6707 // Make sure that operands in input to each add/sub node always
6708 // come from a same pair of vectors.
6709 if (InVec0 != Op0.getOperand(0)) {
6710 if (ExpectedOpcode == ISD::FSUB)
6713 // FADD is commutable. Try to commute the operands
6714 // and then test again.
6715 std::swap(Op0, Op1);
6716 if (InVec0 != Op0.getOperand(0))
6720 if (InVec1 != Op1.getOperand(0))
6723 // Update the pair of expected opcodes.
6724 std::swap(ExpectedOpcode, NextExpectedOpcode);
6727 // Don't try to fold this build_vector into an ADDSUB if the inputs are undef.
6728 if (AddFound && SubFound && InVec0.getOpcode() != ISD::UNDEF &&
6729 InVec1.getOpcode() != ISD::UNDEF)
6730 return DAG.getNode(X86ISD::ADDSUB, DL, VT, InVec0, InVec1);
6735 static SDValue PerformBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
6736 const X86Subtarget *Subtarget) {
6738 EVT VT = N->getValueType(0);
6739 unsigned NumElts = VT.getVectorNumElements();
6740 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(N);
6741 SDValue InVec0, InVec1;
6743 // Try to match an ADDSUB.
6744 if ((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
6745 (Subtarget->hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {
6746 SDValue Value = matchAddSub(BV, DAG, Subtarget);
6747 if (Value.getNode())
6751 // Try to match horizontal ADD/SUB.
6752 unsigned NumUndefsLO = 0;
6753 unsigned NumUndefsHI = 0;
6754 unsigned Half = NumElts/2;
6756 // Count the number of UNDEF operands in the build_vector in input.
6757 for (unsigned i = 0, e = Half; i != e; ++i)
6758 if (BV->getOperand(i)->getOpcode() == ISD::UNDEF)
6761 for (unsigned i = Half, e = NumElts; i != e; ++i)
6762 if (BV->getOperand(i)->getOpcode() == ISD::UNDEF)
6765 // Early exit if this is either a build_vector of all UNDEFs or all the
6766 // operands but one are UNDEF.
6767 if (NumUndefsLO + NumUndefsHI + 1 >= NumElts)
6770 if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) {
6771 // Try to match an SSE3 float HADD/HSUB.
6772 if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
6773 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6775 if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
6776 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6777 } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) {
6778 // Try to match an SSSE3 integer HADD/HSUB.
6779 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
6780 return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
6782 if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
6783 return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
6786 if (!Subtarget->hasAVX())
6789 if ((VT == MVT::v8f32 || VT == MVT::v4f64)) {
6790 // Try to match an AVX horizontal add/sub of packed single/double
6791 // precision floating point values from 256-bit vectors.
6792 SDValue InVec2, InVec3;
6793 if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) &&
6794 isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) &&
6795 ((InVec0.getOpcode() == ISD::UNDEF ||
6796 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6797 ((InVec1.getOpcode() == ISD::UNDEF ||
6798 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6799 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6801 if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) &&
6802 isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) &&
6803 ((InVec0.getOpcode() == ISD::UNDEF ||
6804 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6805 ((InVec1.getOpcode() == ISD::UNDEF ||
6806 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6807 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6808 } else if (VT == MVT::v8i32 || VT == MVT::v16i16) {
6809 // Try to match an AVX2 horizontal add/sub of signed integers.
6810 SDValue InVec2, InVec3;
6812 bool CanFold = true;
6814 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
6815 isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) &&
6816 ((InVec0.getOpcode() == ISD::UNDEF ||
6817 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6818 ((InVec1.getOpcode() == ISD::UNDEF ||
6819 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6820 X86Opcode = X86ISD::HADD;
6821 else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) &&
6822 isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) &&
6823 ((InVec0.getOpcode() == ISD::UNDEF ||
6824 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) &&
6825 ((InVec1.getOpcode() == ISD::UNDEF ||
6826 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3))
6827 X86Opcode = X86ISD::HSUB;
6832 // Fold this build_vector into a single horizontal add/sub.
6833 // Do this only if the target has AVX2.
6834 if (Subtarget->hasAVX2())
6835 return DAG.getNode(X86Opcode, DL, VT, InVec0, InVec1);
6837 // Do not try to expand this build_vector into a pair of horizontal
6838 // add/sub if we can emit a pair of scalar add/sub.
6839 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
6842 // Convert this build_vector into a pair of horizontal binop followed by
6844 bool isUndefLO = NumUndefsLO == Half;
6845 bool isUndefHI = NumUndefsHI == Half;
6846 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, false,
6847 isUndefLO, isUndefHI);
6851 if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
6852 VT == MVT::v16i16) && Subtarget->hasAVX()) {
6854 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
6855 X86Opcode = X86ISD::HADD;
6856 else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
6857 X86Opcode = X86ISD::HSUB;
6858 else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
6859 X86Opcode = X86ISD::FHADD;
6860 else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
6861 X86Opcode = X86ISD::FHSUB;
6865 // Don't try to expand this build_vector into a pair of horizontal add/sub
6866 // if we can simply emit a pair of scalar add/sub.
6867 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
6870 // Convert this build_vector into two horizontal add/sub followed by
6872 bool isUndefLO = NumUndefsLO == Half;
6873 bool isUndefHI = NumUndefsHI == Half;
6874 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
6875 isUndefLO, isUndefHI);
6882 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
6885 MVT VT = Op.getSimpleValueType();
6886 MVT ExtVT = VT.getVectorElementType();
6887 unsigned NumElems = Op.getNumOperands();
6889 // Generate vectors for predicate vectors.
6890 if (VT.getScalarType() == MVT::i1 && Subtarget->hasAVX512())
6891 return LowerBUILD_VECTORvXi1(Op, DAG);
6893 // Vectors containing all zeros can be matched by pxor and xorps later
6894 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
6895 // Canonicalize this to <4 x i32> to 1) ensure the zero vectors are CSE'd
6896 // and 2) ensure that i64 scalars are eliminated on x86-32 hosts.
6897 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)
6900 return getZeroVector(VT, Subtarget, DAG, dl);
6903 // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
6904 // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use
6905 // vpcmpeqd on 256-bit vectors.
6906 if (Subtarget->hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
6907 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256()))
6910 if (!VT.is512BitVector())
6911 return getOnesVector(VT, Subtarget->hasInt256(), DAG, dl);
6914 SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG);
6915 if (Broadcast.getNode())
6918 unsigned EVTBits = ExtVT.getSizeInBits();
6920 unsigned NumZero = 0;
6921 unsigned NumNonZero = 0;
6922 unsigned NonZeros = 0;
6923 bool IsAllConstants = true;
6924 SmallSet<SDValue, 8> Values;
6925 for (unsigned i = 0; i < NumElems; ++i) {
6926 SDValue Elt = Op.getOperand(i);
6927 if (Elt.getOpcode() == ISD::UNDEF)
6930 if (Elt.getOpcode() != ISD::Constant &&
6931 Elt.getOpcode() != ISD::ConstantFP)
6932 IsAllConstants = false;
6933 if (X86::isZeroNode(Elt))
6936 NonZeros |= (1 << i);
6941 // All undef vector. Return an UNDEF. All zero vectors were handled above.
6942 if (NumNonZero == 0)
6943 return DAG.getUNDEF(VT);
6945 // Special case for single non-zero, non-undef, element.
6946 if (NumNonZero == 1) {
6947 unsigned Idx = countTrailingZeros(NonZeros);
6948 SDValue Item = Op.getOperand(Idx);
6950 // If this is an insertion of an i64 value on x86-32, and if the top bits of
6951 // the value are obviously zero, truncate the value to i32 and do the
6952 // insertion that way. Only do this if the value is non-constant or if the
6953 // value is a constant being inserted into element 0. It is cheaper to do
6954 // a constant pool load than it is to do a movd + shuffle.
6955 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() &&
6956 (!IsAllConstants || Idx == 0)) {
6957 if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) {
6959 assert(VT == MVT::v2i64 && "Expected an SSE value type!");
6960 EVT VecVT = MVT::v4i32;
6961 unsigned VecElts = 4;
6963 // Truncate the value (which may itself be a constant) to i32, and
6964 // convert it to a vector with movd (S2V+shuffle to zero extend).
6965 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item);
6966 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item);
6968 // If using the new shuffle lowering, just directly insert this.
6969 if (ExperimentalVectorShuffleLowering)
6971 ISD::BITCAST, dl, VT,
6972 getShuffleVectorZeroOrUndef(Item, Idx * 2, true, Subtarget, DAG));
6974 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
6976 // Now we have our 32-bit value zero extended in the low element of
6977 // a vector. If Idx != 0, swizzle it into place.
6979 SmallVector<int, 4> Mask;
6980 Mask.push_back(Idx);
6981 for (unsigned i = 1; i != VecElts; ++i)
6983 Item = DAG.getVectorShuffle(VecVT, dl, Item, DAG.getUNDEF(VecVT),
6986 return DAG.getNode(ISD::BITCAST, dl, VT, Item);
6990 // If we have a constant or non-constant insertion into the low element of
6991 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
6992 // the rest of the elements. This will be matched as movd/movq/movss/movsd
6993 // depending on what the source datatype is.
6996 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
6998 if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 ||
6999 (ExtVT == MVT::i64 && Subtarget->is64Bit())) {
7000 if (VT.is256BitVector() || VT.is512BitVector()) {
7001 SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl);
7002 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec,
7003 Item, DAG.getIntPtrConstant(0));
7005 assert(VT.is128BitVector() && "Expected an SSE value type!");
7006 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
7007 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
7008 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
7011 if (ExtVT == MVT::i16 || ExtVT == MVT::i8) {
7012 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
7013 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item);
7014 if (VT.is256BitVector()) {
7015 SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl);
7016 Item = Insert128BitVector(ZeroVec, Item, 0, DAG, dl);
7018 assert(VT.is128BitVector() && "Expected an SSE value type!");
7019 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
7021 return DAG.getNode(ISD::BITCAST, dl, VT, Item);
7025 // Is it a vector logical left shift?
7026 if (NumElems == 2 && Idx == 1 &&
7027 X86::isZeroNode(Op.getOperand(0)) &&
7028 !X86::isZeroNode(Op.getOperand(1))) {
7029 unsigned NumBits = VT.getSizeInBits();
7030 return getVShift(true, VT,
7031 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
7032 VT, Op.getOperand(1)),
7033 NumBits/2, DAG, *this, dl);
7036 if (IsAllConstants) // Otherwise, it's better to do a constpool load.
7039 // Otherwise, if this is a vector with i32 or f32 elements, and the element
7040 // is a non-constant being inserted into an element other than the low one,
7041 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka
7042 // movd/movss) to move this into the low element, then shuffle it into
7044 if (EVTBits == 32) {
7045 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
7047 // If using the new shuffle lowering, just directly insert this.
7048 if (ExperimentalVectorShuffleLowering)
7049 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
7051 // Turn it into a shuffle of zero and zero-extended scalar to vector.
7052 Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, Subtarget, DAG);
7053 SmallVector<int, 8> MaskVec;
7054 for (unsigned i = 0; i != NumElems; ++i)
7055 MaskVec.push_back(i == Idx ? 0 : 1);
7056 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]);
7060 // Splat is obviously ok. Let legalizer expand it to a shuffle.
7061 if (Values.size() == 1) {
7062 if (EVTBits == 32) {
7063 // Instead of a shuffle like this:
7064 // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
7065 // Check if it's possible to issue this instead.
7066 // shuffle (vload ptr)), undef, <1, 1, 1, 1>
7067 unsigned Idx = countTrailingZeros(NonZeros);
7068 SDValue Item = Op.getOperand(Idx);
7069 if (Op.getNode()->isOnlyUserOf(Item.getNode()))
7070 return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
7075 // A vector full of immediates; various special cases are already
7076 // handled, so this is best done with a single constant-pool load.
7080 // For AVX-length vectors, see if we can use a vector load to get all of the
7081 // elements, otherwise build the individual 128-bit pieces and use
7082 // shuffles to put them in place.
7083 if (VT.is256BitVector() || VT.is512BitVector()) {
7084 SmallVector<SDValue, 64> V(Op->op_begin(), Op->op_begin() + NumElems);
7086 // Check for a build vector of consecutive loads.
7087 if (SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false))
7090 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElems/2);
7092 // Build both the lower and upper subvector.
7093 SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT,
7094 makeArrayRef(&V[0], NumElems/2));
7095 SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT,
7096 makeArrayRef(&V[NumElems / 2], NumElems/2));
7098 // Recreate the wider vector with the lower and upper part.
7099 if (VT.is256BitVector())
7100 return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl);
7101 return Concat256BitVectors(Lower, Upper, VT, NumElems, DAG, dl);
7104 // Let legalizer expand 2-wide build_vectors.
7105 if (EVTBits == 64) {
7106 if (NumNonZero == 1) {
7107 // One half is zero or undef.
7108 unsigned Idx = countTrailingZeros(NonZeros);
7109 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
7110 Op.getOperand(Idx));
7111 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
7116 // If element VT is < 32 bits, convert it to inserts into a zero vector.
7117 if (EVTBits == 8 && NumElems == 16) {
7118 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
7120 if (V.getNode()) return V;
7123 if (EVTBits == 16 && NumElems == 8) {
7124 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
7126 if (V.getNode()) return V;
7129 // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS
7130 if (EVTBits == 32 && NumElems == 4) {
7131 SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget, *this);
7136 // If element VT is == 32 bits, turn it into a number of shuffles.
7137 SmallVector<SDValue, 8> V(NumElems);
7138 if (NumElems == 4 && NumZero > 0) {
7139 for (unsigned i = 0; i < 4; ++i) {
7140 bool isZero = !(NonZeros & (1 << i));
7142 V[i] = getZeroVector(VT, Subtarget, DAG, dl);
7144 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
7147 for (unsigned i = 0; i < 2; ++i) {
7148 switch ((NonZeros & (0x3 << i*2)) >> (i*2)) {
7151 V[i] = V[i*2]; // Must be a zero vector.
7154 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]);
7157 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]);
7160 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]);
7165 bool Reverse1 = (NonZeros & 0x3) == 2;
7166 bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2;
7170 static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
7171 static_cast<int>(Reverse2 ? NumElems : NumElems+1)
7173 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]);
7176 if (Values.size() > 1 && VT.is128BitVector()) {
7177 // Check for a build vector of consecutive loads.
7178 for (unsigned i = 0; i < NumElems; ++i)
7179 V[i] = Op.getOperand(i);
7181 // Check for elements which are consecutive loads.
7182 SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG, false);
7186 // Check for a build vector from mostly shuffle plus few inserting.
7187 SDValue Sh = buildFromShuffleMostly(Op, DAG);
7191 // For SSE 4.1, use insertps to put the high elements into the low element.
7192 if (Subtarget->hasSSE41()) {
7194 if (Op.getOperand(0).getOpcode() != ISD::UNDEF)
7195 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
7197 Result = DAG.getUNDEF(VT);
7199 for (unsigned i = 1; i < NumElems; ++i) {
7200 if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue;
7201 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
7202 Op.getOperand(i), DAG.getIntPtrConstant(i));
7207 // Otherwise, expand into a number of unpckl*, start by extending each of
7208 // our (non-undef) elements to the full vector width with the element in the
7209 // bottom slot of the vector (which generates no code for SSE).
7210 for (unsigned i = 0; i < NumElems; ++i) {
7211 if (Op.getOperand(i).getOpcode() != ISD::UNDEF)
7212 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
7214 V[i] = DAG.getUNDEF(VT);
7217 // Next, we iteratively mix elements, e.g. for v4f32:
7218 // Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0>
7219 // : unpcklps 1, 3 ==> Y: <?, ?, 3, 1>
7220 // Step 2: unpcklps X, Y ==> <3, 2, 1, 0>
7221 unsigned EltStride = NumElems >> 1;
7222 while (EltStride != 0) {
7223 for (unsigned i = 0; i < EltStride; ++i) {
7224 // If V[i+EltStride] is undef and this is the first round of mixing,
7225 // then it is safe to just drop this shuffle: V[i] is already in the
7226 // right place, the one element (since it's the first round) being
7227 // inserted as undef can be dropped. This isn't safe for successive
7228 // rounds because they will permute elements within both vectors.
7229 if (V[i+EltStride].getOpcode() == ISD::UNDEF &&
7230 EltStride == NumElems/2)
7233 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]);
7242 // LowerAVXCONCAT_VECTORS - 256-bit AVX can use the vinsertf128 instruction
7243 // to create 256-bit vectors from two other 128-bit ones.
7244 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
7246 MVT ResVT = Op.getSimpleValueType();
7248 assert((ResVT.is256BitVector() ||
7249 ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");
7251 SDValue V1 = Op.getOperand(0);
7252 SDValue V2 = Op.getOperand(1);
7253 unsigned NumElems = ResVT.getVectorNumElements();
7254 if(ResVT.is256BitVector())
7255 return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
7257 if (Op.getNumOperands() == 4) {
7258 MVT HalfVT = MVT::getVectorVT(ResVT.getScalarType(),
7259 ResVT.getVectorNumElements()/2);
7260 SDValue V3 = Op.getOperand(2);
7261 SDValue V4 = Op.getOperand(3);
7262 return Concat256BitVectors(Concat128BitVectors(V1, V2, HalfVT, NumElems/2, DAG, dl),
7263 Concat128BitVectors(V3, V4, HalfVT, NumElems/2, DAG, dl), ResVT, NumElems, DAG, dl);
7265 return Concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
7268 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
7269 MVT LLVM_ATTRIBUTE_UNUSED VT = Op.getSimpleValueType();
7270 assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
7271 (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
7272 Op.getNumOperands() == 4)));
7274 // AVX can use the vinsertf128 instruction to create 256-bit vectors
7275 // from two other 128-bit ones.
7277 // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
7278 return LowerAVXCONCAT_VECTORS(Op, DAG);
7282 //===----------------------------------------------------------------------===//
7283 // Vector shuffle lowering
7285 // This is an experimental code path for lowering vector shuffles on x86. It is
7286 // designed to handle arbitrary vector shuffles and blends, gracefully
7287 // degrading performance as necessary. It works hard to recognize idiomatic
7288 // shuffles and lower them to optimal instruction patterns without leaving
7289 // a framework that allows reasonably efficient handling of all vector shuffle
7291 //===----------------------------------------------------------------------===//
7293 /// \brief Tiny helper function to identify a no-op mask.
7295 /// This is a somewhat boring predicate function. It checks whether the mask
7296 /// array input, which is assumed to be a single-input shuffle mask of the kind
7297 /// used by the X86 shuffle instructions (not a fully general
7298 /// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an
7299 /// in-place shuffle are 'no-op's.
7300 static bool isNoopShuffleMask(ArrayRef<int> Mask) {
7301 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7302 if (Mask[i] != -1 && Mask[i] != i)
7307 /// \brief Helper function to classify a mask as a single-input mask.
7309 /// This isn't a generic single-input test because in the vector shuffle
7310 /// lowering we canonicalize single inputs to be the first input operand. This
7311 /// means we can more quickly test for a single input by only checking whether
7312 /// an input from the second operand exists. We also assume that the size of
7313 /// mask corresponds to the size of the input vectors which isn't true in the
7314 /// fully general case.
7315 static bool isSingleInputShuffleMask(ArrayRef<int> Mask) {
7317 if (M >= (int)Mask.size())
7322 /// \brief Test whether there are elements crossing 128-bit lanes in this
7325 /// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
7326 /// and we routinely test for these.
7327 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {
7328 int LaneSize = 128 / VT.getScalarSizeInBits();
7329 int Size = Mask.size();
7330 for (int i = 0; i < Size; ++i)
7331 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
7336 /// \brief Test whether a shuffle mask is equivalent within each 128-bit lane.
7338 /// This checks a shuffle mask to see if it is performing the same
7339 /// 128-bit lane-relative shuffle in each 128-bit lane. This trivially implies
7340 /// that it is also not lane-crossing. It may however involve a blend from the
7341 /// same lane of a second vector.
7343 /// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
7344 /// non-trivial to compute in the face of undef lanes. The representation is
7345 /// *not* suitable for use with existing 128-bit shuffles as it will contain
7346 /// entries from both V1 and V2 inputs to the wider mask.
7348 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
7349 SmallVectorImpl<int> &RepeatedMask) {
7350 int LaneSize = 128 / VT.getScalarSizeInBits();
7351 RepeatedMask.resize(LaneSize, -1);
7352 int Size = Mask.size();
7353 for (int i = 0; i < Size; ++i) {
7356 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
7357 // This entry crosses lanes, so there is no way to model this shuffle.
7360 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
7361 if (RepeatedMask[i % LaneSize] == -1)
7362 // This is the first non-undef entry in this slot of a 128-bit lane.
7363 RepeatedMask[i % LaneSize] =
7364 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + Size;
7365 else if (RepeatedMask[i % LaneSize] + (i / LaneSize) * LaneSize != Mask[i])
7366 // Found a mismatch with the repeated mask.
7372 /// \brief Base case helper for testing a single mask element.
7373 static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2,
7374 BuildVectorSDNode *BV1,
7375 BuildVectorSDNode *BV2, ArrayRef<int> Mask,
7377 int Size = Mask.size();
7378 if (Mask[i] != -1 && Mask[i] != Arg) {
7379 auto *MaskBV = Mask[i] < Size ? BV1 : BV2;
7380 auto *ArgsBV = Arg < Size ? BV1 : BV2;
7381 if (!MaskBV || !ArgsBV ||
7382 MaskBV->getOperand(Mask[i] % Size) != ArgsBV->getOperand(Arg % Size))
7388 /// \brief Recursive helper to peel off and test each mask element.
7389 template <typename... Ts>
7390 static bool isShuffleEquivalentImpl(SDValue V1, SDValue V2,
7391 BuildVectorSDNode *BV1,
7392 BuildVectorSDNode *BV2, ArrayRef<int> Mask,
7393 int i, int Arg, Ts... Args) {
7394 if (!isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i, Arg))
7397 return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, i + 1, Args...);
7400 /// \brief Checks whether a shuffle mask is equivalent to an explicit list of
7403 /// This is a fast way to test a shuffle mask against a fixed pattern:
7405 /// if (isShuffleEquivalent(Mask, 3, 2, 1, 0)) { ... }
7407 /// It returns true if the mask is exactly as wide as the argument list, and
7408 /// each element of the mask is either -1 (signifying undef) or the value given
7409 /// in the argument.
7410 template <typename... Ts>
7411 static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
7413 if (Mask.size() != sizeof...(Args))
7416 // If the values are build vectors, we can look through them to find
7417 // equivalent inputs that make the shuffles equivalent.
7418 auto *BV1 = dyn_cast<BuildVectorSDNode>(V1);
7419 auto *BV2 = dyn_cast<BuildVectorSDNode>(V2);
7421 // Recursively peel off arguments and test them against the mask.
7422 return isShuffleEquivalentImpl(V1, V2, BV1, BV2, Mask, 0, Args...);
7425 /// \brief Get a 4-lane 8-bit shuffle immediate for a mask.
7427 /// This helper function produces an 8-bit shuffle immediate corresponding to
7428 /// the ubiquitous shuffle encoding scheme used in x86 instructions for
7429 /// shuffling 4 lanes. It can be used with most of the PSHUF instructions for
7432 /// NB: We rely heavily on "undef" masks preserving the input lane.
7433 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask,
7434 SelectionDAG &DAG) {
7435 assert(Mask.size() == 4 && "Only 4-lane shuffle masks");
7436 assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!");
7437 assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!");
7438 assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!");
7439 assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!");
7442 Imm |= (Mask[0] == -1 ? 0 : Mask[0]) << 0;
7443 Imm |= (Mask[1] == -1 ? 1 : Mask[1]) << 2;
7444 Imm |= (Mask[2] == -1 ? 2 : Mask[2]) << 4;
7445 Imm |= (Mask[3] == -1 ? 3 : Mask[3]) << 6;
7446 return DAG.getConstant(Imm, MVT::i8);
7449 /// \brief Try to emit a blend instruction for a shuffle.
7451 /// This doesn't do any checks for the availability of instructions for blending
7452 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
7453 /// be matched in the backend with the type given. What it does check for is
7454 /// that the shuffle mask is in fact a blend.
7455 static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1,
7456 SDValue V2, ArrayRef<int> Mask,
7457 const X86Subtarget *Subtarget,
7458 SelectionDAG &DAG) {
7460 unsigned BlendMask = 0;
7461 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7462 if (Mask[i] >= Size) {
7463 if (Mask[i] != i + Size)
7464 return SDValue(); // Shuffled V2 input!
7465 BlendMask |= 1u << i;
7468 if (Mask[i] >= 0 && Mask[i] != i)
7469 return SDValue(); // Shuffled V1 input!
7471 switch (VT.SimpleTy) {
7476 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
7477 DAG.getConstant(BlendMask, MVT::i8));
7481 assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!");
7485 // If we have AVX2 it is faster to use VPBLENDD when the shuffle fits into
7486 // that instruction.
7487 if (Subtarget->hasAVX2()) {
7488 // Scale the blend by the number of 32-bit dwords per element.
7489 int Scale = VT.getScalarSizeInBits() / 32;
7491 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7492 if (Mask[i] >= Size)
7493 for (int j = 0; j < Scale; ++j)
7494 BlendMask |= 1u << (i * Scale + j);
7496 MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32;
7497 V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1);
7498 V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2);
7499 return DAG.getNode(ISD::BITCAST, DL, VT,
7500 DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
7501 DAG.getConstant(BlendMask, MVT::i8)));
7505 // For integer shuffles we need to expand the mask and cast the inputs to
7506 // v8i16s prior to blending.
7507 int Scale = 8 / VT.getVectorNumElements();
7509 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7510 if (Mask[i] >= Size)
7511 for (int j = 0; j < Scale; ++j)
7512 BlendMask |= 1u << (i * Scale + j);
7514 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1);
7515 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2);
7516 return DAG.getNode(ISD::BITCAST, DL, VT,
7517 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
7518 DAG.getConstant(BlendMask, MVT::i8)));
7522 assert(Subtarget->hasAVX2() && "256-bit integer blends require AVX2!");
7523 SmallVector<int, 8> RepeatedMask;
7524 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
7525 // We can lower these with PBLENDW which is mirrored across 128-bit lanes.
7526 assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!");
7528 for (int i = 0; i < 8; ++i)
7529 if (RepeatedMask[i] >= 16)
7530 BlendMask |= 1u << i;
7531 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
7532 DAG.getConstant(BlendMask, MVT::i8));
7538 // Scale the blend by the number of bytes per element.
7539 int Scale = VT.getScalarSizeInBits() / 8;
7541 // This form of blend is always done on bytes. Compute the byte vector
7543 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
7545 // Compute the VSELECT mask. Note that VSELECT is really confusing in the
7546 // mix of LLVM's code generator and the x86 backend. We tell the code
7547 // generator that boolean values in the elements of an x86 vector register
7548 // are -1 for true and 0 for false. We then use the LLVM semantics of 'true'
7549 // mapping a select to operand #1, and 'false' mapping to operand #2. The
7550 // reality in x86 is that vector masks (pre-AVX-512) use only the high bit
7551 // of the element (the remaining are ignored) and 0 in that high bit would
7552 // mean operand #1 while 1 in the high bit would mean operand #2. So while
7553 // the LLVM model for boolean values in vector elements gets the relevant
7554 // bit set, it is set backwards and over constrained relative to x86's
7556 SmallVector<SDValue, 32> VSELECTMask;
7557 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7558 for (int j = 0; j < Scale; ++j)
7559 VSELECTMask.push_back(
7560 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
7561 : DAG.getConstant(Mask[i] < Size ? -1 : 0, MVT::i8));
7563 V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1);
7564 V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2);
7566 ISD::BITCAST, DL, VT,
7567 DAG.getNode(ISD::VSELECT, DL, BlendVT,
7568 DAG.getNode(ISD::BUILD_VECTOR, DL, BlendVT, VSELECTMask),
7573 llvm_unreachable("Not a supported integer vector type!");
7577 /// \brief Try to lower as a blend of elements from two inputs followed by
7578 /// a single-input permutation.
7580 /// This matches the pattern where we can blend elements from two inputs and
7581 /// then reduce the shuffle to a single-input permutation.
7582 static SDValue lowerVectorShuffleAsBlendAndPermute(SDLoc DL, MVT VT, SDValue V1,
7585 SelectionDAG &DAG) {
7586 // We build up the blend mask while checking whether a blend is a viable way
7587 // to reduce the shuffle.
7588 SmallVector<int, 32> BlendMask(Mask.size(), -1);
7589 SmallVector<int, 32> PermuteMask(Mask.size(), -1);
7591 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7595 assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds.");
7597 if (BlendMask[Mask[i] % Size] == -1)
7598 BlendMask[Mask[i] % Size] = Mask[i];
7599 else if (BlendMask[Mask[i] % Size] != Mask[i])
7600 return SDValue(); // Can't blend in the needed input!
7602 PermuteMask[i] = Mask[i] % Size;
7605 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
7606 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
7609 /// \brief Generic routine to decompose a shuffle and blend into indepndent
7610 /// blends and permutes.
7612 /// This matches the extremely common pattern for handling combined
7613 /// shuffle+blend operations on newer X86 ISAs where we have very fast blend
7614 /// operations. It will try to pick the best arrangement of shuffles and
7616 static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(SDLoc DL, MVT VT,
7620 SelectionDAG &DAG) {
7621 // Shuffle the input elements into the desired positions in V1 and V2 and
7622 // blend them together.
7623 SmallVector<int, 32> V1Mask(Mask.size(), -1);
7624 SmallVector<int, 32> V2Mask(Mask.size(), -1);
7625 SmallVector<int, 32> BlendMask(Mask.size(), -1);
7626 for (int i = 0, Size = Mask.size(); i < Size; ++i)
7627 if (Mask[i] >= 0 && Mask[i] < Size) {
7628 V1Mask[i] = Mask[i];
7630 } else if (Mask[i] >= Size) {
7631 V2Mask[i] = Mask[i] - Size;
7632 BlendMask[i] = i + Size;
7635 // Try to lower with the simpler initial blend strategy unless one of the
7636 // input shuffles would be a no-op. We prefer to shuffle inputs as the
7637 // shuffle may be able to fold with a load or other benefit. However, when
7638 // we'll have to do 2x as many shuffles in order to achieve this, blending
7639 // first is a better strategy.
7640 if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask))
7641 if (SDValue BlendPerm =
7642 lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))
7645 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
7646 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
7647 return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
7650 /// \brief Try to lower a vector shuffle as a byte rotation.
7652 /// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary
7653 /// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
7654 /// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will
7655 /// try to generically lower a vector shuffle through such an pattern. It
7656 /// does not check for the profitability of lowering either as PALIGNR or
7657 /// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form.
7658 /// This matches shuffle vectors that look like:
7660 /// v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
7662 /// Essentially it concatenates V1 and V2, shifts right by some number of
7663 /// elements, and takes the low elements as the result. Note that while this is
7664 /// specified as a *right shift* because x86 is little-endian, it is a *left
7665 /// rotate* of the vector lanes.
7666 static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1,
7669 const X86Subtarget *Subtarget,
7670 SelectionDAG &DAG) {
7671 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
7673 int NumElts = Mask.size();
7674 int NumLanes = VT.getSizeInBits() / 128;
7675 int NumLaneElts = NumElts / NumLanes;
7677 // We need to detect various ways of spelling a rotation:
7678 // [11, 12, 13, 14, 15, 0, 1, 2]
7679 // [-1, 12, 13, 14, -1, -1, 1, -1]
7680 // [-1, -1, -1, -1, -1, -1, 1, 2]
7681 // [ 3, 4, 5, 6, 7, 8, 9, 10]
7682 // [-1, 4, 5, 6, -1, -1, 9, -1]
7683 // [-1, 4, 5, 6, -1, -1, -1, -1]
7686 for (int l = 0; l < NumElts; l += NumLaneElts) {
7687 for (int i = 0; i < NumLaneElts; ++i) {
7688 if (Mask[l + i] == -1)
7690 assert(Mask[l + i] >= 0 && "Only -1 is a valid negative mask element!");
7692 // Get the mod-Size index and lane correct it.
7693 int LaneIdx = (Mask[l + i] % NumElts) - l;
7694 // Make sure it was in this lane.
7695 if (LaneIdx < 0 || LaneIdx >= NumLaneElts)
7698 // Determine where a rotated vector would have started.
7699 int StartIdx = i - LaneIdx;
7701 // The identity rotation isn't interesting, stop.
7704 // If we found the tail of a vector the rotation must be the missing
7705 // front. If we found the head of a vector, it must be how much of the
7707 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumLaneElts - StartIdx;
7710 Rotation = CandidateRotation;
7711 else if (Rotation != CandidateRotation)
7712 // The rotations don't match, so we can't match this mask.
7715 // Compute which value this mask is pointing at.
7716 SDValue MaskV = Mask[l + i] < NumElts ? V1 : V2;
7718 // Compute which of the two target values this index should be assigned
7719 // to. This reflects whether the high elements are remaining or the low
7720 // elements are remaining.
7721 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
7723 // Either set up this value if we've not encountered it before, or check
7724 // that it remains consistent.
7727 else if (TargetV != MaskV)
7728 // This may be a rotation, but it pulls from the inputs in some
7729 // unsupported interleaving.
7734 // Check that we successfully analyzed the mask, and normalize the results.
7735 assert(Rotation != 0 && "Failed to locate a viable rotation!");
7736 assert((Lo || Hi) && "Failed to find a rotated input vector!");
7742 // The actual rotate instruction rotates bytes, so we need to scale the
7743 // rotation based on how many bytes are in the vector lane.
7744 int Scale = 16 / NumLaneElts;
7746 // SSSE3 targets can use the palignr instruction.
7747 if (Subtarget->hasSSSE3()) {
7748 // Cast the inputs to i8 vector of correct length to match PALIGNR.
7749 MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes);
7750 Lo = DAG.getNode(ISD::BITCAST, DL, AlignVT, Lo);
7751 Hi = DAG.getNode(ISD::BITCAST, DL, AlignVT, Hi);
7753 return DAG.getNode(ISD::BITCAST, DL, VT,
7754 DAG.getNode(X86ISD::PALIGNR, DL, AlignVT, Hi, Lo,
7755 DAG.getConstant(Rotation * Scale, MVT::i8)));
7758 assert(VT.getSizeInBits() == 128 &&
7759 "Rotate-based lowering only supports 128-bit lowering!");
7760 assert(Mask.size() <= 16 &&
7761 "Can shuffle at most 16 bytes in a 128-bit vector!");
7763 // Default SSE2 implementation
7764 int LoByteShift = 16 - Rotation * Scale;
7765 int HiByteShift = Rotation * Scale;
7767 // Cast the inputs to v2i64 to match PSLLDQ/PSRLDQ.
7768 Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo);
7769 Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi);
7771 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo,
7772 DAG.getConstant(LoByteShift, MVT::i8));
7773 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi,
7774 DAG.getConstant(HiByteShift, MVT::i8));
7775 return DAG.getNode(ISD::BITCAST, DL, VT,
7776 DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift));
7779 /// \brief Compute whether each element of a shuffle is zeroable.
7781 /// A "zeroable" vector shuffle element is one which can be lowered to zero.
7782 /// Either it is an undef element in the shuffle mask, the element of the input
7783 /// referenced is undef, or the element of the input referenced is known to be
7784 /// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
7785 /// as many lanes with this technique as possible to simplify the remaining
7787 static SmallBitVector computeZeroableShuffleElements(ArrayRef<int> Mask,
7788 SDValue V1, SDValue V2) {
7789 SmallBitVector Zeroable(Mask.size(), false);
7791 while (V1.getOpcode() == ISD::BITCAST)
7792 V1 = V1->getOperand(0);
7793 while (V2.getOpcode() == ISD::BITCAST)
7794 V2 = V2->getOperand(0);
7796 bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
7797 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
7799 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7801 // Handle the easy cases.
7802 if (M < 0 || (M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
7807 // If this is an index into a build_vector node (which has the same number
7808 // of elements), dig out the input value and use it.
7809 SDValue V = M < Size ? V1 : V2;
7810 if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands())
7813 SDValue Input = V.getOperand(M % Size);
7814 // The UNDEF opcode check really should be dead code here, but not quite
7815 // worth asserting on (it isn't invalid, just unexpected).
7816 if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input))
7823 /// \brief Try to emit a bitmask instruction for a shuffle.
7825 /// This handles cases where we can model a blend exactly as a bitmask due to
7826 /// one of the inputs being zeroable.
7827 static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1,
7828 SDValue V2, ArrayRef<int> Mask,
7829 SelectionDAG &DAG) {
7830 MVT EltVT = VT.getScalarType();
7831 int NumEltBits = EltVT.getSizeInBits();
7832 MVT IntEltVT = MVT::getIntegerVT(NumEltBits);
7833 SDValue Zero = DAG.getConstant(0, IntEltVT);
7834 SDValue AllOnes = DAG.getConstant(APInt::getAllOnesValue(NumEltBits), IntEltVT);
7835 if (EltVT.isFloatingPoint()) {
7836 Zero = DAG.getNode(ISD::BITCAST, DL, EltVT, Zero);
7837 AllOnes = DAG.getNode(ISD::BITCAST, DL, EltVT, AllOnes);
7839 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
7840 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7842 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
7845 if (Mask[i] % Size != i)
7846 return SDValue(); // Not a blend.
7848 V = Mask[i] < Size ? V1 : V2;
7849 else if (V != (Mask[i] < Size ? V1 : V2))
7850 return SDValue(); // Can only let one input through the mask.
7852 VMaskOps[i] = AllOnes;
7855 return SDValue(); // No non-zeroable elements!
7857 SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps);
7858 V = DAG.getNode(VT.isFloatingPoint()
7859 ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND,
7864 /// \brief Try to lower a vector shuffle as a byte shift (shifts in zeros).
7866 /// Attempts to match a shuffle mask against the PSRLDQ and PSLLDQ
7867 /// byte-shift instructions. The mask must consist of a shifted sequential
7868 /// shuffle from one of the input vectors and zeroable elements for the
7869 /// remaining 'shifted in' elements.
7870 static SDValue lowerVectorShuffleAsByteShift(SDLoc DL, MVT VT, SDValue V1,
7871 SDValue V2, ArrayRef<int> Mask,
7872 SelectionDAG &DAG) {
7873 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
7875 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7877 int NumElts = VT.getVectorNumElements();
7878 int NumLanes = VT.getSizeInBits() / 128;
7879 int NumLaneElts = NumElts / NumLanes;
7880 int Scale = 16 / NumLaneElts;
7881 MVT ShiftVT = MVT::getVectorVT(MVT::i64, 2 * NumLanes);
7883 // PSLLDQ : (little-endian) left byte shift
7884 // [ zz, 0, 1, 2, 3, 4, 5, 6]
7885 // [ zz, zz, -1, -1, 2, 3, 4, -1]
7886 // [ zz, zz, zz, zz, zz, zz, -1, 1]
7887 // PSRLDQ : (little-endian) right byte shift
7888 // [ 5, 6, 7, zz, zz, zz, zz, zz]
7889 // [ -1, 5, 6, 7, zz, zz, zz, zz]
7890 // [ 1, 2, -1, -1, -1, -1, zz, zz]
7891 auto MatchByteShift = [&](int Shift) -> SDValue {
7892 bool MatchLeft = true, MatchRight = true;
7893 for (int l = 0; l < NumElts; l += NumLaneElts) {
7894 for (int i = 0; i < Shift; ++i)
7895 MatchLeft &= Zeroable[l + i];
7896 for (int i = NumLaneElts - Shift; i < NumLaneElts; ++i)
7897 MatchRight &= Zeroable[l + i];
7899 if (!(MatchLeft || MatchRight))
7902 bool MatchV1 = true, MatchV2 = true;
7903 for (int l = 0; l < NumElts; l += NumLaneElts) {
7904 unsigned Pos = MatchLeft ? Shift + l : l;
7905 unsigned Low = MatchLeft ? l : Shift + l;
7906 unsigned Len = NumLaneElts - Shift;
7907 MatchV1 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low);
7908 MatchV2 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low + NumElts);
7910 if (!(MatchV1 || MatchV2))
7913 int ByteShift = Shift * Scale;
7914 unsigned Op = MatchRight ? X86ISD::VSRLDQ : X86ISD::VSHLDQ;
7915 SDValue V = MatchV1 ? V1 : V2;
7916 V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V);
7917 V = DAG.getNode(Op, DL, ShiftVT, V,
7918 DAG.getConstant(ByteShift, MVT::i8));
7919 return DAG.getNode(ISD::BITCAST, DL, VT, V);
7922 for (int Shift = 1; Shift < NumLaneElts; ++Shift)
7923 if (SDValue S = MatchByteShift(Shift))
7930 /// \brief Try to lower a vector shuffle as a bit shift (shifts in zeros).
7932 /// Attempts to match a shuffle mask against the PSRL(W/D/Q) and PSLL(W/D/Q)
7933 /// SSE2 and AVX2 logical bit-shift instructions. The function matches
7934 /// elements from one of the input vectors shuffled to the left or right
7935 /// with zeroable elements 'shifted in'.
7936 static SDValue lowerVectorShuffleAsBitShift(SDLoc DL, MVT VT, SDValue V1,
7937 SDValue V2, ArrayRef<int> Mask,
7938 SelectionDAG &DAG) {
7939 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7941 int Size = Mask.size();
7942 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
7944 // PSRL : (little-endian) right bit shift.
7947 // PSHL : (little-endian) left bit shift.
7949 // [ -1, 4, zz, -1 ]
7950 auto MatchBitShift = [&](int Shift, int Scale) -> SDValue {
7951 MVT ShiftSVT = MVT::getIntegerVT(VT.getScalarSizeInBits() * Scale);
7952 MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale);
7953 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
7954 "Illegal integer vector type");
7956 bool MatchLeft = true, MatchRight = true;
7957 for (int i = 0; i != Size; i += Scale) {
7958 for (int j = 0; j != Shift; ++j) {
7959 MatchLeft &= Zeroable[i + j];
7961 for (int j = Scale - Shift; j != Scale; ++j) {
7962 MatchRight &= Zeroable[i + j];
7965 if (!(MatchLeft || MatchRight))
7968 bool MatchV1 = true, MatchV2 = true;
7969 for (int i = 0; i != Size; i += Scale) {
7970 unsigned Pos = MatchLeft ? i + Shift : i;
7971 unsigned Low = MatchLeft ? i : i + Shift;
7972 unsigned Len = Scale - Shift;
7973 MatchV1 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low);
7974 MatchV2 &= isSequentialOrUndefInRange(Mask, Pos, Len, Low + Size);
7976 if (!(MatchV1 || MatchV2))
7979 // Cast the inputs to ShiftVT to match VSRLI/VSHLI and back again.
7980 unsigned OpCode = MatchLeft ? X86ISD::VSHLI : X86ISD::VSRLI;
7981 int ShiftAmt = Shift * VT.getScalarSizeInBits();
7982 SDValue V = MatchV1 ? V1 : V2;
7983 V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V);
7984 V = DAG.getNode(OpCode, DL, ShiftVT, V, DAG.getConstant(ShiftAmt, MVT::i8));
7985 return DAG.getNode(ISD::BITCAST, DL, VT, V);
7988 // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
7989 // keep doubling the size of the integer elements up to that. We can
7990 // then shift the elements of the integer vector by whole multiples of
7991 // their width within the elements of the larger integer vector. Test each
7992 // multiple to see if we can find a match with the moved element indices
7993 // and that the shifted in elements are all zeroable.
7994 for (int Scale = 2; Scale * VT.getScalarSizeInBits() <= 64; Scale *= 2)
7995 for (int Shift = 1; Shift != Scale; ++Shift)
7996 if (SDValue BitShift = MatchBitShift(Shift, Scale))
8003 /// \brief Lower a vector shuffle as a zero or any extension.
8005 /// Given a specific number of elements, element bit width, and extension
8006 /// stride, produce either a zero or any extension based on the available
8007 /// features of the subtarget.
8008 static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
8009 SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV,
8010 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
8011 assert(Scale > 1 && "Need a scale to extend.");
8012 int NumElements = VT.getVectorNumElements();
8013 int EltBits = VT.getScalarSizeInBits();
8014 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
8015 "Only 8, 16, and 32 bit elements can be extended.");
8016 assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits.");
8018 // Found a valid zext mask! Try various lowering strategies based on the
8019 // input type and available ISA extensions.
8020 if (Subtarget->hasSSE41()) {
8021 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
8022 NumElements / Scale);
8023 return DAG.getNode(ISD::BITCAST, DL, VT,
8024 DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV));
8027 // For any extends we can cheat for larger element sizes and use shuffle
8028 // instructions that can fold with a load and/or copy.
8029 if (AnyExt && EltBits == 32) {
8030 int PSHUFDMask[4] = {0, -1, 1, -1};
8032 ISD::BITCAST, DL, VT,
8033 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
8034 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV),
8035 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
8037 if (AnyExt && EltBits == 16 && Scale > 2) {
8038 int PSHUFDMask[4] = {0, -1, 0, -1};
8039 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
8040 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV),
8041 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG));
8042 int PSHUFHWMask[4] = {1, -1, -1, -1};
8044 ISD::BITCAST, DL, VT,
8045 DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16,
8046 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV),
8047 getV4X86ShuffleImm8ForMask(PSHUFHWMask, DAG)));
8050 // If this would require more than 2 unpack instructions to expand, use
8051 // pshufb when available. We can only use more than 2 unpack instructions
8052 // when zero extending i8 elements which also makes it easier to use pshufb.
8053 if (Scale > 4 && EltBits == 8 && Subtarget->hasSSSE3()) {
8054 assert(NumElements == 16 && "Unexpected byte vector width!");
8055 SDValue PSHUFBMask[16];
8056 for (int i = 0; i < 16; ++i)
8058 DAG.getConstant((i % Scale == 0) ? i / Scale : 0x80, MVT::i8);
8059 InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV);
8060 return DAG.getNode(ISD::BITCAST, DL, VT,
8061 DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
8062 DAG.getNode(ISD::BUILD_VECTOR, DL,
8063 MVT::v16i8, PSHUFBMask)));
8066 // Otherwise emit a sequence of unpacks.
8068 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
8069 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
8070 : getZeroVector(InputVT, Subtarget, DAG, DL);
8071 InputV = DAG.getNode(ISD::BITCAST, DL, InputVT, InputV);
8072 InputV = DAG.getNode(X86ISD::UNPCKL, DL, InputVT, InputV, Ext);
8076 } while (Scale > 1);
8077 return DAG.getNode(ISD::BITCAST, DL, VT, InputV);
8080 /// \brief Try to lower a vector shuffle as a zero extension on any microarch.
8082 /// This routine will try to do everything in its power to cleverly lower
8083 /// a shuffle which happens to match the pattern of a zero extend. It doesn't
8084 /// check for the profitability of this lowering, it tries to aggressively
8085 /// match this pattern. It will use all of the micro-architectural details it
8086 /// can to emit an efficient lowering. It handles both blends with all-zero
8087 /// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
8088 /// masking out later).
8090 /// The reason we have dedicated lowering for zext-style shuffles is that they
8091 /// are both incredibly common and often quite performance sensitive.
8092 static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
8093 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
8094 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
8095 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8097 int Bits = VT.getSizeInBits();
8098 int NumElements = VT.getVectorNumElements();
8099 assert(VT.getScalarSizeInBits() <= 32 &&
8100 "Exceeds 32-bit integer zero extension limit");
8101 assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size");
8103 // Define a helper function to check a particular ext-scale and lower to it if
8105 auto Lower = [&](int Scale) -> SDValue {
8108 for (int i = 0; i < NumElements; ++i) {
8110 continue; // Valid anywhere but doesn't tell us anything.
8111 if (i % Scale != 0) {
8112 // Each of the extended elements need to be zeroable.
8116 // We no longer are in the anyext case.
8121 // Each of the base elements needs to be consecutive indices into the
8122 // same input vector.
8123 SDValue V = Mask[i] < NumElements ? V1 : V2;
8126 else if (InputV != V)
8127 return SDValue(); // Flip-flopping inputs.
8129 if (Mask[i] % NumElements != i / Scale)
8130 return SDValue(); // Non-consecutive strided elements.
8133 // If we fail to find an input, we have a zero-shuffle which should always
8134 // have already been handled.
8135 // FIXME: Maybe handle this here in case during blending we end up with one?
8139 return lowerVectorShuffleAsSpecificZeroOrAnyExtend(
8140 DL, VT, Scale, AnyExt, InputV, Subtarget, DAG);
8143 // The widest scale possible for extending is to a 64-bit integer.
8144 assert(Bits % 64 == 0 &&
8145 "The number of bits in a vector must be divisible by 64 on x86!");
8146 int NumExtElements = Bits / 64;
8148 // Each iteration, try extending the elements half as much, but into twice as
8150 for (; NumExtElements < NumElements; NumExtElements *= 2) {
8151 assert(NumElements % NumExtElements == 0 &&
8152 "The input vector size must be divisible by the extended size.");
8153 if (SDValue V = Lower(NumElements / NumExtElements))
8157 // General extends failed, but 128-bit vectors may be able to use MOVQ.
8161 // Returns one of the source operands if the shuffle can be reduced to a
8162 // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits.
8163 auto CanZExtLowHalf = [&]() {
8164 for (int i = NumElements / 2; i != NumElements; ++i)
8167 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0))
8169 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements))
8174 if (SDValue V = CanZExtLowHalf()) {
8175 V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V);
8176 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
8177 return DAG.getNode(ISD::BITCAST, DL, VT, V);
8180 // No viable ext lowering found.
8184 /// \brief Try to get a scalar value for a specific element of a vector.
8186 /// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
8187 static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
8188 SelectionDAG &DAG) {
8189 MVT VT = V.getSimpleValueType();
8190 MVT EltVT = VT.getVectorElementType();
8191 while (V.getOpcode() == ISD::BITCAST)
8192 V = V.getOperand(0);
8193 // If the bitcasts shift the element size, we can't extract an equivalent
8195 MVT NewVT = V.getSimpleValueType();
8196 if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
8199 if (V.getOpcode() == ISD::BUILD_VECTOR ||
8200 (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR))
8201 return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, V.getOperand(Idx));
8206 /// \brief Helper to test for a load that can be folded with x86 shuffles.
8208 /// This is particularly important because the set of instructions varies
8209 /// significantly based on whether the operand is a load or not.
8210 static bool isShuffleFoldableLoad(SDValue V) {
8211 while (V.getOpcode() == ISD::BITCAST)
8212 V = V.getOperand(0);
8214 return ISD::isNON_EXTLoad(V.getNode());
8217 /// \brief Try to lower insertion of a single element into a zero vector.
8219 /// This is a common pattern that we have especially efficient patterns to lower
8220 /// across all subtarget feature sets.
8221 static SDValue lowerVectorShuffleAsElementInsertion(
8222 MVT VT, SDLoc DL, SDValue V1, SDValue V2, ArrayRef<int> Mask,
8223 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
8224 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8226 MVT EltVT = VT.getVectorElementType();
8228 int V2Index = std::find_if(Mask.begin(), Mask.end(),
8229 [&Mask](int M) { return M >= (int)Mask.size(); }) -
8231 bool IsV1Zeroable = true;
8232 for (int i = 0, Size = Mask.size(); i < Size; ++i)
8233 if (i != V2Index && !Zeroable[i]) {
8234 IsV1Zeroable = false;
8238 // Check for a single input from a SCALAR_TO_VECTOR node.
8239 // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and
8240 // all the smarts here sunk into that routine. However, the current
8241 // lowering of BUILD_VECTOR makes that nearly impossible until the old
8242 // vector shuffle lowering is dead.
8243 if (SDValue V2S = getScalarValueForVectorElement(
8244 V2, Mask[V2Index] - Mask.size(), DAG)) {
8245 // We need to zext the scalar if it is smaller than an i32.
8246 V2S = DAG.getNode(ISD::BITCAST, DL, EltVT, V2S);
8247 if (EltVT == MVT::i8 || EltVT == MVT::i16) {
8248 // Using zext to expand a narrow element won't work for non-zero
8253 // Zero-extend directly to i32.
8255 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
8257 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
8258 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||
8259 EltVT == MVT::i16) {
8260 // Either not inserting from the low element of the input or the input
8261 // element size is too small to use VZEXT_MOVL to clear the high bits.
8265 if (!IsV1Zeroable) {
8266 // If V1 can't be treated as a zero vector we have fewer options to lower
8267 // this. We can't support integer vectors or non-zero targets cheaply, and
8268 // the V1 elements can't be permuted in any way.
8269 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");
8270 if (!VT.isFloatingPoint() || V2Index != 0)
8272 SmallVector<int, 8> V1Mask(Mask.begin(), Mask.end());
8273 V1Mask[V2Index] = -1;
8274 if (!isNoopShuffleMask(V1Mask))
8276 // This is essentially a special case blend operation, but if we have
8277 // general purpose blend operations, they are always faster. Bail and let
8278 // the rest of the lowering handle these as blends.
8279 if (Subtarget->hasSSE41())
8282 // Otherwise, use MOVSD or MOVSS.
8283 assert((EltVT == MVT::f32 || EltVT == MVT::f64) &&
8284 "Only two types of floating point element types to handle!");
8285 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL,
8289 // This lowering only works for the low element with floating point vectors.
8290 if (VT.isFloatingPoint() && V2Index != 0)
8293 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
8295 V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2);
8298 // If we have 4 or fewer lanes we can cheaply shuffle the element into
8299 // the desired position. Otherwise it is more efficient to do a vector
8300 // shift left. We know that we can do a vector shift left because all
8301 // the inputs are zero.
8302 if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) {
8303 SmallVector<int, 4> V2Shuffle(Mask.size(), 1);
8304 V2Shuffle[V2Index] = 0;
8305 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
8307 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2);
8309 X86ISD::VSHLDQ, DL, MVT::v2i64, V2,
8311 V2Index * EltVT.getSizeInBits()/8,
8312 DAG.getTargetLoweringInfo().getScalarShiftAmountTy(MVT::v2i64)));
8313 V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2);
8319 /// \brief Try to lower broadcast of a single element.
8321 /// For convenience, this code also bundles all of the subtarget feature set
8322 /// filtering. While a little annoying to re-dispatch on type here, there isn't
8323 /// a convenient way to factor it out.
8324 static SDValue lowerVectorShuffleAsBroadcast(MVT VT, SDLoc DL, SDValue V,
8326 const X86Subtarget *Subtarget,
8327 SelectionDAG &DAG) {
8328 if (!Subtarget->hasAVX())
8330 if (VT.isInteger() && !Subtarget->hasAVX2())
8333 // Check that the mask is a broadcast.
8334 int BroadcastIdx = -1;
8336 if (M >= 0 && BroadcastIdx == -1)
8338 else if (M >= 0 && M != BroadcastIdx)
8341 assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
8342 "a sorted mask where the broadcast "
8345 // Go up the chain of (vector) values to try and find a scalar load that
8346 // we can combine with the broadcast.
8348 switch (V.getOpcode()) {
8349 case ISD::CONCAT_VECTORS: {
8350 int OperandSize = Mask.size() / V.getNumOperands();
8351 V = V.getOperand(BroadcastIdx / OperandSize);
8352 BroadcastIdx %= OperandSize;
8356 case ISD::INSERT_SUBVECTOR: {
8357 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
8358 auto ConstantIdx = dyn_cast<ConstantSDNode>(V.getOperand(2));
8362 int BeginIdx = (int)ConstantIdx->getZExtValue();
8364 BeginIdx + (int)VInner.getValueType().getVectorNumElements();
8365 if (BroadcastIdx >= BeginIdx && BroadcastIdx < EndIdx) {
8366 BroadcastIdx -= BeginIdx;
8377 // Check if this is a broadcast of a scalar. We special case lowering
8378 // for scalars so that we can more effectively fold with loads.
8379 if (V.getOpcode() == ISD::BUILD_VECTOR ||
8380 (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0)) {
8381 V = V.getOperand(BroadcastIdx);
8383 // If the scalar isn't a load we can't broadcast from it in AVX1, only with
8385 if (!Subtarget->hasAVX2() && !isShuffleFoldableLoad(V))
8387 } else if (BroadcastIdx != 0 || !Subtarget->hasAVX2()) {
8388 // We can't broadcast from a vector register w/o AVX2, and we can only
8389 // broadcast from the zero-element of a vector register.
8393 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, V);
8396 // Check for whether we can use INSERTPS to perform the shuffle. We only use
8397 // INSERTPS when the V1 elements are already in the correct locations
8398 // because otherwise we can just always use two SHUFPS instructions which
8399 // are much smaller to encode than a SHUFPS and an INSERTPS. We can also
8400 // perform INSERTPS if a single V1 element is out of place and all V2
8401 // elements are zeroable.
8402 static SDValue lowerVectorShuffleAsInsertPS(SDValue Op, SDValue V1, SDValue V2,
8404 SelectionDAG &DAG) {
8405 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!");
8406 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8407 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8408 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
8410 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8413 int V1DstIndex = -1;
8414 int V2DstIndex = -1;
8415 bool V1UsedInPlace = false;
8417 for (int i = 0; i < 4; ++i) {
8418 // Synthesize a zero mask from the zeroable elements (includes undefs).
8424 // Flag if we use any V1 inputs in place.
8426 V1UsedInPlace = true;
8430 // We can only insert a single non-zeroable element.
8431 if (V1DstIndex != -1 || V2DstIndex != -1)
8435 // V1 input out of place for insertion.
8438 // V2 input for insertion.
8443 // Don't bother if we have no (non-zeroable) element for insertion.
8444 if (V1DstIndex == -1 && V2DstIndex == -1)
8447 // Determine element insertion src/dst indices. The src index is from the
8448 // start of the inserted vector, not the start of the concatenated vector.
8449 unsigned V2SrcIndex = 0;
8450 if (V1DstIndex != -1) {
8451 // If we have a V1 input out of place, we use V1 as the V2 element insertion
8452 // and don't use the original V2 at all.
8453 V2SrcIndex = Mask[V1DstIndex];
8454 V2DstIndex = V1DstIndex;
8457 V2SrcIndex = Mask[V2DstIndex] - 4;
8460 // If no V1 inputs are used in place, then the result is created only from
8461 // the zero mask and the V2 insertion - so remove V1 dependency.
8463 V1 = DAG.getUNDEF(MVT::v4f32);
8465 unsigned InsertPSMask = V2SrcIndex << 6 | V2DstIndex << 4 | ZMask;
8466 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
8468 // Insert the V2 element into the desired position.
8470 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
8471 DAG.getConstant(InsertPSMask, MVT::i8));
8474 /// \brief Try to lower a shuffle as a permute of the inputs followed by an
8475 /// UNPCK instruction.
8477 /// This specifically targets cases where we end up with alternating between
8478 /// the two inputs, and so can permute them into something that feeds a single
8479 /// UNPCK instruction. Note that this routine only targets integer vectors
8480 /// because for floating point vectors we have a generalized SHUFPS lowering
8481 /// strategy that handles everything that doesn't *exactly* match an unpack,
8482 /// making this clever lowering unnecessary.
8483 static SDValue lowerVectorShuffleAsUnpack(MVT VT, SDLoc DL, SDValue V1,
8484 SDValue V2, ArrayRef<int> Mask,
8485 SelectionDAG &DAG) {
8486 assert(!VT.isFloatingPoint() &&
8487 "This routine only supports integer vectors.");
8488 assert(!isSingleInputShuffleMask(Mask) &&
8489 "This routine should only be used when blending two inputs.");
8490 assert(Mask.size() >= 2 && "Single element masks are invalid.");
8492 int Size = Mask.size();
8494 int NumLoInputs = std::count_if(Mask.begin(), Mask.end(), [Size](int M) {
8495 return M >= 0 && M % Size < Size / 2;
8497 int NumHiInputs = std::count_if(
8498 Mask.begin(), Mask.end(), [Size](int M) { return M % Size > Size / 2; });
8500 bool UnpackLo = NumLoInputs >= NumHiInputs;
8502 auto TryUnpack = [&](MVT UnpackVT, int Scale) {
8503 SmallVector<int, 32> V1Mask(Mask.size(), -1);
8504 SmallVector<int, 32> V2Mask(Mask.size(), -1);
8506 for (int i = 0; i < Size; ++i) {
8510 // Each element of the unpack contains Scale elements from this mask.
8511 int UnpackIdx = i / Scale;
8513 // We only handle the case where V1 feeds the first slots of the unpack.
8514 // We rely on canonicalization to ensure this is the case.
8515 if ((UnpackIdx % 2 == 0) != (Mask[i] < Size))
8518 // Setup the mask for this input. The indexing is tricky as we have to
8519 // handle the unpack stride.
8520 SmallVectorImpl<int> &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask;
8521 VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] =
8525 // Shuffle the inputs into place.
8526 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
8527 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
8529 // Cast the inputs to the type we will use to unpack them.
8530 V1 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V1);
8531 V2 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V2);
8533 // Unpack the inputs and cast the result back to the desired type.
8534 return DAG.getNode(ISD::BITCAST, DL, VT,
8535 DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
8536 DL, UnpackVT, V1, V2));
8539 // We try each unpack from the largest to the smallest to try and find one
8540 // that fits this mask.
8541 int OrigNumElements = VT.getVectorNumElements();
8542 int OrigScalarSize = VT.getScalarSizeInBits();
8543 for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2) {
8544 int Scale = ScalarSize / OrigScalarSize;
8545 int NumElements = OrigNumElements / Scale;
8546 MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), NumElements);
8547 if (SDValue Unpack = TryUnpack(UnpackVT, Scale))
8554 /// \brief Handle lowering of 2-lane 64-bit floating point shuffles.
8556 /// This is the basis function for the 2-lane 64-bit shuffles as we have full
8557 /// support for floating point shuffles but not integer shuffles. These
8558 /// instructions will incur a domain crossing penalty on some chips though so
8559 /// it is better to avoid lowering through this for integer vectors where
8561 static SDValue lowerV2F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8562 const X86Subtarget *Subtarget,
8563 SelectionDAG &DAG) {
8565 assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!");
8566 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
8567 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
8568 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8569 ArrayRef<int> Mask = SVOp->getMask();
8570 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
8572 if (isSingleInputShuffleMask(Mask)) {
8573 // Use low duplicate instructions for masks that match their pattern.
8574 if (Subtarget->hasSSE3())
8575 if (isShuffleEquivalent(V1, V2, Mask, 0, 0))
8576 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, V1);
8578 // Straight shuffle of a single input vector. Simulate this by using the
8579 // single input as both of the "inputs" to this instruction..
8580 unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
8582 if (Subtarget->hasAVX()) {
8583 // If we have AVX, we can use VPERMILPS which will allow folding a load
8584 // into the shuffle.
8585 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
8586 DAG.getConstant(SHUFPDMask, MVT::i8));
8589 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1,
8590 DAG.getConstant(SHUFPDMask, MVT::i8));
8592 assert(Mask[0] >= 0 && Mask[0] < 2 && "Non-canonicalized blend!");
8593 assert(Mask[1] >= 2 && "Non-canonicalized blend!");
8595 // If we have a single input, insert that into V1 if we can do so cheaply.
8596 if ((Mask[0] >= 2) + (Mask[1] >= 2) == 1) {
8597 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8598 MVT::v2f64, DL, V1, V2, Mask, Subtarget, DAG))
8600 // Try inverting the insertion since for v2 masks it is easy to do and we
8601 // can't reliably sort the mask one way or the other.
8602 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
8603 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
8604 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8605 MVT::v2f64, DL, V2, V1, InverseMask, Subtarget, DAG))
8609 // Try to use one of the special instruction patterns to handle two common
8610 // blend patterns if a zero-blend above didn't work.
8611 if (isShuffleEquivalent(V1, V2, Mask, 0, 3) || isShuffleEquivalent(V1, V2, Mask, 1, 3))
8612 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
8613 // We can either use a special instruction to load over the low double or
8614 // to move just the low double.
8616 isShuffleFoldableLoad(V1S) ? X86ISD::MOVLPD : X86ISD::MOVSD,
8618 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
8620 if (Subtarget->hasSSE41())
8621 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
8625 // Use dedicated unpack instructions for masks that match their pattern.
8626 if (isShuffleEquivalent(V1, V2, Mask, 0, 2))
8627 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2f64, V1, V2);
8628 if (isShuffleEquivalent(V1, V2, Mask, 1, 3))
8629 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2f64, V1, V2);
8631 unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
8632 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2,
8633 DAG.getConstant(SHUFPDMask, MVT::i8));
8636 /// \brief Handle lowering of 2-lane 64-bit integer shuffles.
8638 /// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
8639 /// the integer unit to minimize domain crossing penalties. However, for blends
8640 /// it falls back to the floating point shuffle operation with appropriate bit
8642 static SDValue lowerV2I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8643 const X86Subtarget *Subtarget,
8644 SelectionDAG &DAG) {
8646 assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!");
8647 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
8648 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
8649 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8650 ArrayRef<int> Mask = SVOp->getMask();
8651 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
8653 if (isSingleInputShuffleMask(Mask)) {
8654 // Check for being able to broadcast a single element.
8655 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v2i64, DL, V1,
8656 Mask, Subtarget, DAG))
8659 // Straight shuffle of a single input vector. For everything from SSE2
8660 // onward this has a single fast instruction with no scary immediates.
8661 // We have to map the mask as it is actually a v4i32 shuffle instruction.
8662 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1);
8663 int WidenedMask[4] = {
8664 std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1,
8665 std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1};
8667 ISD::BITCAST, DL, MVT::v2i64,
8668 DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1,
8669 getV4X86ShuffleImm8ForMask(WidenedMask, DAG)));
8672 // Try to use byte shift instructions.
8673 if (SDValue Shift = lowerVectorShuffleAsByteShift(
8674 DL, MVT::v2i64, V1, V2, Mask, DAG))
8677 // If we have a single input from V2 insert that into V1 if we can do so
8679 if ((Mask[0] >= 2) + (Mask[1] >= 2) == 1) {
8680 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8681 MVT::v2i64, DL, V1, V2, Mask, Subtarget, DAG))
8683 // Try inverting the insertion since for v2 masks it is easy to do and we
8684 // can't reliably sort the mask one way or the other.
8685 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
8686 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
8687 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
8688 MVT::v2i64, DL, V2, V1, InverseMask, Subtarget, DAG))
8692 // We have different paths for blend lowering, but they all must use the
8693 // *exact* same predicate.
8694 bool IsBlendSupported = Subtarget->hasSSE41();
8695 if (IsBlendSupported)
8696 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
8700 // Use dedicated unpack instructions for masks that match their pattern.
8701 if (isShuffleEquivalent(V1, V2, Mask, 0, 2))
8702 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, V1, V2);
8703 if (isShuffleEquivalent(V1, V2, Mask, 1, 3))
8704 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v2i64, V1, V2);
8706 // Try to use byte rotation instructions.
8707 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
8708 if (Subtarget->hasSSSE3())
8709 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
8710 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
8713 // If we have direct support for blends, we should lower by decomposing into
8714 // a permute. That will be faster than the domain cross.
8715 if (IsBlendSupported)
8716 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2,
8719 // We implement this with SHUFPD which is pretty lame because it will likely
8720 // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
8721 // However, all the alternatives are still more cycles and newer chips don't
8722 // have this problem. It would be really nice if x86 had better shuffles here.
8723 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1);
8724 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2);
8725 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
8726 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
8729 /// \brief Test whether this can be lowered with a single SHUFPS instruction.
8731 /// This is used to disable more specialized lowerings when the shufps lowering
8732 /// will happen to be efficient.
8733 static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
8734 // This routine only handles 128-bit shufps.
8735 assert(Mask.size() == 4 && "Unsupported mask size!");
8737 // To lower with a single SHUFPS we need to have the low half and high half
8738 // each requiring a single input.
8739 if (Mask[0] != -1 && Mask[1] != -1 && (Mask[0] < 4) != (Mask[1] < 4))
8741 if (Mask[2] != -1 && Mask[3] != -1 && (Mask[2] < 4) != (Mask[3] < 4))
8747 /// \brief Lower a vector shuffle using the SHUFPS instruction.
8749 /// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
8750 /// It makes no assumptions about whether this is the *best* lowering, it simply
8752 static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT,
8753 ArrayRef<int> Mask, SDValue V1,
8754 SDValue V2, SelectionDAG &DAG) {
8755 SDValue LowV = V1, HighV = V2;
8756 int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]};
8759 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
8761 if (NumV2Elements == 1) {
8763 std::find_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; }) -
8766 // Compute the index adjacent to V2Index and in the same half by toggling
8768 int V2AdjIndex = V2Index ^ 1;
8770 if (Mask[V2AdjIndex] == -1) {
8771 // Handles all the cases where we have a single V2 element and an undef.
8772 // This will only ever happen in the high lanes because we commute the
8773 // vector otherwise.
8775 std::swap(LowV, HighV);
8776 NewMask[V2Index] -= 4;
8778 // Handle the case where the V2 element ends up adjacent to a V1 element.
8779 // To make this work, blend them together as the first step.
8780 int V1Index = V2AdjIndex;
8781 int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
8782 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
8783 getV4X86ShuffleImm8ForMask(BlendMask, DAG));
8785 // Now proceed to reconstruct the final blend as we have the necessary
8786 // high or low half formed.
8793 NewMask[V1Index] = 2; // We put the V1 element in V2[2].
8794 NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
8796 } else if (NumV2Elements == 2) {
8797 if (Mask[0] < 4 && Mask[1] < 4) {
8798 // Handle the easy case where we have V1 in the low lanes and V2 in the
8802 } else if (Mask[2] < 4 && Mask[3] < 4) {
8803 // We also handle the reversed case because this utility may get called
8804 // when we detect a SHUFPS pattern but can't easily commute the shuffle to
8805 // arrange things in the right direction.
8811 // We have a mixture of V1 and V2 in both low and high lanes. Rather than
8812 // trying to place elements directly, just blend them and set up the final
8813 // shuffle to place them.
8815 // The first two blend mask elements are for V1, the second two are for
8817 int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
8818 Mask[2] < 4 ? Mask[2] : Mask[3],
8819 (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
8820 (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
8821 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
8822 getV4X86ShuffleImm8ForMask(BlendMask, DAG));
8824 // Now we do a normal shuffle of V1 by giving V1 as both operands to
8827 NewMask[0] = Mask[0] < 4 ? 0 : 2;
8828 NewMask[1] = Mask[0] < 4 ? 2 : 0;
8829 NewMask[2] = Mask[2] < 4 ? 1 : 3;
8830 NewMask[3] = Mask[2] < 4 ? 3 : 1;
8833 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
8834 getV4X86ShuffleImm8ForMask(NewMask, DAG));
8837 /// \brief Lower 4-lane 32-bit floating point shuffles.
8839 /// Uses instructions exclusively from the floating point unit to minimize
8840 /// domain crossing penalties, as these are sufficient to implement all v4f32
8842 static SDValue lowerV4F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8843 const X86Subtarget *Subtarget,
8844 SelectionDAG &DAG) {
8846 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!");
8847 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8848 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8849 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8850 ArrayRef<int> Mask = SVOp->getMask();
8851 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
8854 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
8856 if (NumV2Elements == 0) {
8857 // Check for being able to broadcast a single element.
8858 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f32, DL, V1,
8859 Mask, Subtarget, DAG))
8862 // Use even/odd duplicate instructions for masks that match their pattern.
8863 if (Subtarget->hasSSE3()) {
8864 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2))
8865 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
8866 if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3))
8867 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
8870 if (Subtarget->hasAVX()) {
8871 // If we have AVX, we can use VPERMILPS which will allow folding a load
8872 // into the shuffle.
8873 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
8874 getV4X86ShuffleImm8ForMask(Mask, DAG));
8877 // Otherwise, use a straight shuffle of a single input vector. We pass the
8878 // input vector to both operands to simulate this with a SHUFPS.
8879 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
8880 getV4X86ShuffleImm8ForMask(Mask, DAG));
8883 // There are special ways we can lower some single-element blends. However, we
8884 // have custom ways we can lower more complex single-element blends below that
8885 // we defer to if both this and BLENDPS fail to match, so restrict this to
8886 // when the V2 input is targeting element 0 of the mask -- that is the fast
8888 if (NumV2Elements == 1 && Mask[0] >= 4)
8889 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4f32, DL, V1, V2,
8890 Mask, Subtarget, DAG))
8893 if (Subtarget->hasSSE41()) {
8894 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
8898 // Use INSERTPS if we can complete the shuffle efficiently.
8899 if (SDValue V = lowerVectorShuffleAsInsertPS(Op, V1, V2, Mask, DAG))
8902 if (!isSingleSHUFPSMask(Mask))
8903 if (SDValue BlendPerm = lowerVectorShuffleAsBlendAndPermute(
8904 DL, MVT::v4f32, V1, V2, Mask, DAG))
8908 // Use dedicated unpack instructions for masks that match their pattern.
8909 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5))
8910 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V1, V2);
8911 if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7))
8912 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V1, V2);
8913 if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 5, 1))
8914 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f32, V2, V1);
8915 if (isShuffleEquivalent(V1, V2, Mask, 6, 2, 7, 3))
8916 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f32, V2, V1);
8918 // Otherwise fall back to a SHUFPS lowering strategy.
8919 return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
8922 /// \brief Lower 4-lane i32 vector shuffles.
8924 /// We try to handle these with integer-domain shuffles where we can, but for
8925 /// blends we use the floating point domain blend instructions.
8926 static SDValue lowerV4I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
8927 const X86Subtarget *Subtarget,
8928 SelectionDAG &DAG) {
8930 assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!");
8931 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
8932 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
8933 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
8934 ArrayRef<int> Mask = SVOp->getMask();
8935 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
8937 // Whenever we can lower this as a zext, that instruction is strictly faster
8938 // than any alternative. It also allows us to fold memory operands into the
8939 // shuffle in many cases.
8940 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2,
8941 Mask, Subtarget, DAG))
8945 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
8947 if (NumV2Elements == 0) {
8948 // Check for being able to broadcast a single element.
8949 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i32, DL, V1,
8950 Mask, Subtarget, DAG))
8953 // Straight shuffle of a single input vector. For everything from SSE2
8954 // onward this has a single fast instruction with no scary immediates.
8955 // We coerce the shuffle pattern to be compatible with UNPCK instructions
8956 // but we aren't actually going to use the UNPCK instruction because doing
8957 // so prevents folding a load into this instruction or making a copy.
8958 const int UnpackLoMask[] = {0, 0, 1, 1};
8959 const int UnpackHiMask[] = {2, 2, 3, 3};
8960 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 1, 1))
8961 Mask = UnpackLoMask;
8962 else if (isShuffleEquivalent(V1, V2, Mask, 2, 2, 3, 3))
8963 Mask = UnpackHiMask;
8965 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
8966 getV4X86ShuffleImm8ForMask(Mask, DAG));
8969 // Try to use bit shift instructions.
8970 if (SDValue Shift = lowerVectorShuffleAsBitShift(
8971 DL, MVT::v4i32, V1, V2, Mask, DAG))
8974 // Try to use byte shift instructions.
8975 if (SDValue Shift = lowerVectorShuffleAsByteShift(
8976 DL, MVT::v4i32, V1, V2, Mask, DAG))
8979 // There are special ways we can lower some single-element blends.
8980 if (NumV2Elements == 1)
8981 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v4i32, DL, V1, V2,
8982 Mask, Subtarget, DAG))
8985 // We have different paths for blend lowering, but they all must use the
8986 // *exact* same predicate.
8987 bool IsBlendSupported = Subtarget->hasSSE41();
8988 if (IsBlendSupported)
8989 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
8993 if (SDValue Masked =
8994 lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG))
8997 // Use dedicated unpack instructions for masks that match their pattern.
8998 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 1, 5))
8999 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V1, V2);
9000 if (isShuffleEquivalent(V1, V2, Mask, 2, 6, 3, 7))
9001 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V1, V2);
9002 if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 5, 1))
9003 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i32, V2, V1);
9004 if (isShuffleEquivalent(V1, V2, Mask, 6, 2, 7, 3))
9005 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i32, V2, V1);
9007 // Try to use byte rotation instructions.
9008 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
9009 if (Subtarget->hasSSSE3())
9010 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9011 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG))
9014 // If we have direct support for blends, we should lower by decomposing into
9015 // a permute. That will be faster than the domain cross.
9016 if (IsBlendSupported)
9017 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2,
9020 // Try to lower by permuting the inputs into an unpack instruction.
9021 if (SDValue Unpack =
9022 lowerVectorShuffleAsUnpack(MVT::v4i32, DL, V1, V2, Mask, DAG))
9025 // We implement this with SHUFPS because it can blend from two vectors.
9026 // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
9027 // up the inputs, bypassing domain shift penalties that we would encur if we
9028 // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
9030 return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32,
9031 DAG.getVectorShuffle(
9033 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1),
9034 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask));
9037 /// \brief Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
9038 /// shuffle lowering, and the most complex part.
9040 /// The lowering strategy is to try to form pairs of input lanes which are
9041 /// targeted at the same half of the final vector, and then use a dword shuffle
9042 /// to place them onto the right half, and finally unpack the paired lanes into
9043 /// their final position.
9045 /// The exact breakdown of how to form these dword pairs and align them on the
9046 /// correct sides is really tricky. See the comments within the function for
9047 /// more of the details.
9048 static SDValue lowerV8I16SingleInputVectorShuffle(
9049 SDLoc DL, SDValue V, MutableArrayRef<int> Mask,
9050 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
9051 assert(V.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
9052 MutableArrayRef<int> LoMask = Mask.slice(0, 4);
9053 MutableArrayRef<int> HiMask = Mask.slice(4, 4);
9055 SmallVector<int, 4> LoInputs;
9056 std::copy_if(LoMask.begin(), LoMask.end(), std::back_inserter(LoInputs),
9057 [](int M) { return M >= 0; });
9058 std::sort(LoInputs.begin(), LoInputs.end());
9059 LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end());
9060 SmallVector<int, 4> HiInputs;
9061 std::copy_if(HiMask.begin(), HiMask.end(), std::back_inserter(HiInputs),
9062 [](int M) { return M >= 0; });
9063 std::sort(HiInputs.begin(), HiInputs.end());
9064 HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end());
9066 std::lower_bound(LoInputs.begin(), LoInputs.end(), 4) - LoInputs.begin();
9067 int NumHToL = LoInputs.size() - NumLToL;
9069 std::lower_bound(HiInputs.begin(), HiInputs.end(), 4) - HiInputs.begin();
9070 int NumHToH = HiInputs.size() - NumLToH;
9071 MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
9072 MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
9073 MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
9074 MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
9076 // Check for being able to broadcast a single element.
9077 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i16, DL, V,
9078 Mask, Subtarget, DAG))
9081 // Try to use bit shift instructions.
9082 if (SDValue Shift = lowerVectorShuffleAsBitShift(
9083 DL, MVT::v8i16, V, V, Mask, DAG))
9086 // Try to use byte shift instructions.
9087 if (SDValue Shift = lowerVectorShuffleAsByteShift(
9088 DL, MVT::v8i16, V, V, Mask, DAG))
9091 // Use dedicated unpack instructions for masks that match their pattern.
9092 if (isShuffleEquivalent(V, V, Mask, 0, 0, 1, 1, 2, 2, 3, 3))
9093 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V, V);
9094 if (isShuffleEquivalent(V, V, Mask, 4, 4, 5, 5, 6, 6, 7, 7))
9095 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V, V);
9097 // Try to use byte rotation instructions.
9098 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9099 DL, MVT::v8i16, V, V, Mask, Subtarget, DAG))
9102 // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
9103 // such inputs we can swap two of the dwords across the half mark and end up
9104 // with <=2 inputs to each half in each half. Once there, we can fall through
9105 // to the generic code below. For example:
9107 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
9108 // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
9110 // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half
9111 // and an existing 2-into-2 on the other half. In this case we may have to
9112 // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or
9113 // 1-into-3 which could cause us to cycle endlessly fixing each side in turn.
9114 // Fortunately, we don't have to handle anything but a 2-into-2 pattern
9115 // because any other situation (including a 3-into-1 or 1-into-3 in the other
9116 // half than the one we target for fixing) will be fixed when we re-enter this
9117 // path. We will also combine away any sequence of PSHUFD instructions that
9118 // result into a single instruction. Here is an example of the tricky case:
9120 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
9121 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3]
9123 // This now has a 1-into-3 in the high half! Instead, we do two shuffles:
9125 // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h]
9126 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6]
9128 // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h]
9129 // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6]
9131 // The result is fine to be handled by the generic logic.
9132 auto balanceSides = [&](ArrayRef<int> AToAInputs, ArrayRef<int> BToAInputs,
9133 ArrayRef<int> BToBInputs, ArrayRef<int> AToBInputs,
9134 int AOffset, int BOffset) {
9135 assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
9136 "Must call this with A having 3 or 1 inputs from the A half.");
9137 assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
9138 "Must call this with B having 1 or 3 inputs from the B half.");
9139 assert(AToAInputs.size() + BToAInputs.size() == 4 &&
9140 "Must call this with either 3:1 or 1:3 inputs (summing to 4).");
9142 // Compute the index of dword with only one word among the three inputs in
9143 // a half by taking the sum of the half with three inputs and subtracting
9144 // the sum of the actual three inputs. The difference is the remaining
9147 int &TripleDWord = AToAInputs.size() == 3 ? ADWord : BDWord;
9148 int &OneInputDWord = AToAInputs.size() == 3 ? BDWord : ADWord;
9149 int TripleInputOffset = AToAInputs.size() == 3 ? AOffset : BOffset;
9150 ArrayRef<int> TripleInputs = AToAInputs.size() == 3 ? AToAInputs : BToAInputs;
9151 int OneInput = AToAInputs.size() == 3 ? BToAInputs[0] : AToAInputs[0];
9152 int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
9153 int TripleNonInputIdx =
9154 TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
9155 TripleDWord = TripleNonInputIdx / 2;
9157 // We use xor with one to compute the adjacent DWord to whichever one the
9159 OneInputDWord = (OneInput / 2) ^ 1;
9161 // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA
9162 // and BToA inputs. If there is also such a problem with the BToB and AToB
9163 // inputs, we don't try to fix it necessarily -- we'll recurse and see it in
9164 // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it
9165 // is essential that we don't *create* a 3<-1 as then we might oscillate.
9166 if (BToBInputs.size() == 2 && AToBInputs.size() == 2) {
9167 // Compute how many inputs will be flipped by swapping these DWords. We
9169 // to balance this to ensure we don't form a 3-1 shuffle in the other
9171 int NumFlippedAToBInputs =
9172 std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) +
9173 std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1);
9174 int NumFlippedBToBInputs =
9175 std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) +
9176 std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1);
9177 if ((NumFlippedAToBInputs == 1 &&
9178 (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
9179 (NumFlippedBToBInputs == 1 &&
9180 (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) {
9181 // We choose whether to fix the A half or B half based on whether that
9182 // half has zero flipped inputs. At zero, we may not be able to fix it
9183 // with that half. We also bias towards fixing the B half because that
9184 // will more commonly be the high half, and we have to bias one way.
9185 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
9186 ArrayRef<int> Inputs) {
9187 int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot.
9188 bool IsFixIdxInput = std::find(Inputs.begin(), Inputs.end(),
9189 PinnedIdx ^ 1) != Inputs.end();
9190 // Determine whether the free index is in the flipped dword or the
9191 // unflipped dword based on where the pinned index is. We use this bit
9192 // in an xor to conditionally select the adjacent dword.
9193 int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord));
9194 bool IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(),
9195 FixFreeIdx) != Inputs.end();
9196 if (IsFixIdxInput == IsFixFreeIdxInput)
9198 IsFixFreeIdxInput = std::find(Inputs.begin(), Inputs.end(),
9199 FixFreeIdx) != Inputs.end();
9200 assert(IsFixIdxInput != IsFixFreeIdxInput &&
9201 "We need to be changing the number of flipped inputs!");
9202 int PSHUFHalfMask[] = {0, 1, 2, 3};
9203 std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
9204 V = DAG.getNode(FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
9206 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DAG));
9209 if (M != -1 && M == FixIdx)
9211 else if (M != -1 && M == FixFreeIdx)
9214 if (NumFlippedBToBInputs != 0) {
9216 BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
9217 FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs);
9219 assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!");
9221 AToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
9222 FixFlippedInputs(APinnedIdx, ADWord, AToBInputs);
9227 int PSHUFDMask[] = {0, 1, 2, 3};
9228 PSHUFDMask[ADWord] = BDWord;
9229 PSHUFDMask[BDWord] = ADWord;
9230 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
9231 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
9232 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V),
9233 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
9235 // Adjust the mask to match the new locations of A and B.
9237 if (M != -1 && M/2 == ADWord)
9238 M = 2 * BDWord + M % 2;
9239 else if (M != -1 && M/2 == BDWord)
9240 M = 2 * ADWord + M % 2;
9242 // Recurse back into this routine to re-compute state now that this isn't
9243 // a 3 and 1 problem.
9244 return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16),
9247 if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
9248 return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
9249 else if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3))
9250 return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0);
9252 // At this point there are at most two inputs to the low and high halves from
9253 // each half. That means the inputs can always be grouped into dwords and
9254 // those dwords can then be moved to the correct half with a dword shuffle.
9255 // We use at most one low and one high word shuffle to collect these paired
9256 // inputs into dwords, and finally a dword shuffle to place them.
9257 int PSHUFLMask[4] = {-1, -1, -1, -1};
9258 int PSHUFHMask[4] = {-1, -1, -1, -1};
9259 int PSHUFDMask[4] = {-1, -1, -1, -1};
9261 // First fix the masks for all the inputs that are staying in their
9262 // original halves. This will then dictate the targets of the cross-half
9264 auto fixInPlaceInputs =
9265 [&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
9266 MutableArrayRef<int> SourceHalfMask,
9267 MutableArrayRef<int> HalfMask, int HalfOffset) {
9268 if (InPlaceInputs.empty())
9270 if (InPlaceInputs.size() == 1) {
9271 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
9272 InPlaceInputs[0] - HalfOffset;
9273 PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
9276 if (IncomingInputs.empty()) {
9277 // Just fix all of the in place inputs.
9278 for (int Input : InPlaceInputs) {
9279 SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
9280 PSHUFDMask[Input / 2] = Input / 2;
9285 assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!");
9286 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
9287 InPlaceInputs[0] - HalfOffset;
9288 // Put the second input next to the first so that they are packed into
9289 // a dword. We find the adjacent index by toggling the low bit.
9290 int AdjIndex = InPlaceInputs[0] ^ 1;
9291 SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
9292 std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
9293 PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
9295 fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
9296 fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
9298 // Now gather the cross-half inputs and place them into a free dword of
9299 // their target half.
9300 // FIXME: This operation could almost certainly be simplified dramatically to
9301 // look more like the 3-1 fixing operation.
9302 auto moveInputsToRightHalf = [&PSHUFDMask](
9303 MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
9304 MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
9305 MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
9307 auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
9308 return SourceHalfMask[Word] != -1 && SourceHalfMask[Word] != Word;
9310 auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
9312 int LowWord = Word & ~1;
9313 int HighWord = Word | 1;
9314 return isWordClobbered(SourceHalfMask, LowWord) ||
9315 isWordClobbered(SourceHalfMask, HighWord);
9318 if (IncomingInputs.empty())
9321 if (ExistingInputs.empty()) {
9322 // Map any dwords with inputs from them into the right half.
9323 for (int Input : IncomingInputs) {
9324 // If the source half mask maps over the inputs, turn those into
9325 // swaps and use the swapped lane.
9326 if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
9327 if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == -1) {
9328 SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
9329 Input - SourceOffset;
9330 // We have to swap the uses in our half mask in one sweep.
9331 for (int &M : HalfMask)
9332 if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset)
9334 else if (M == Input)
9335 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
9337 assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==
9338 Input - SourceOffset &&
9339 "Previous placement doesn't match!");
9341 // Note that this correctly re-maps both when we do a swap and when
9342 // we observe the other side of the swap above. We rely on that to
9343 // avoid swapping the members of the input list directly.
9344 Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
9347 // Map the input's dword into the correct half.
9348 if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == -1)
9349 PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
9351 assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==
9353 "Previous placement doesn't match!");
9356 // And just directly shift any other-half mask elements to be same-half
9357 // as we will have mirrored the dword containing the element into the
9358 // same position within that half.
9359 for (int &M : HalfMask)
9360 if (M >= SourceOffset && M < SourceOffset + 4) {
9361 M = M - SourceOffset + DestOffset;
9362 assert(M >= 0 && "This should never wrap below zero!");
9367 // Ensure we have the input in a viable dword of its current half. This
9368 // is particularly tricky because the original position may be clobbered
9369 // by inputs being moved and *staying* in that half.
9370 if (IncomingInputs.size() == 1) {
9371 if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
9372 int InputFixed = std::find(std::begin(SourceHalfMask),
9373 std::end(SourceHalfMask), -1) -
9374 std::begin(SourceHalfMask) + SourceOffset;
9375 SourceHalfMask[InputFixed - SourceOffset] =
9376 IncomingInputs[0] - SourceOffset;
9377 std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
9379 IncomingInputs[0] = InputFixed;
9381 } else if (IncomingInputs.size() == 2) {
9382 if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
9383 isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
9384 // We have two non-adjacent or clobbered inputs we need to extract from
9385 // the source half. To do this, we need to map them into some adjacent
9386 // dword slot in the source mask.
9387 int InputsFixed[2] = {IncomingInputs[0] - SourceOffset,
9388 IncomingInputs[1] - SourceOffset};
9390 // If there is a free slot in the source half mask adjacent to one of
9391 // the inputs, place the other input in it. We use (Index XOR 1) to
9392 // compute an adjacent index.
9393 if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) &&
9394 SourceHalfMask[InputsFixed[0] ^ 1] == -1) {
9395 SourceHalfMask[InputsFixed[0]] = InputsFixed[0];
9396 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
9397 InputsFixed[1] = InputsFixed[0] ^ 1;
9398 } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) &&
9399 SourceHalfMask[InputsFixed[1] ^ 1] == -1) {
9400 SourceHalfMask[InputsFixed[1]] = InputsFixed[1];
9401 SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0];
9402 InputsFixed[0] = InputsFixed[1] ^ 1;
9403 } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] == -1 &&
9404 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] == -1) {
9405 // The two inputs are in the same DWord but it is clobbered and the
9406 // adjacent DWord isn't used at all. Move both inputs to the free
9408 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0];
9409 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1];
9410 InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1);
9411 InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1;
9413 // The only way we hit this point is if there is no clobbering
9414 // (because there are no off-half inputs to this half) and there is no
9415 // free slot adjacent to one of the inputs. In this case, we have to
9416 // swap an input with a non-input.
9417 for (int i = 0; i < 4; ++i)
9418 assert((SourceHalfMask[i] == -1 || SourceHalfMask[i] == i) &&
9419 "We can't handle any clobbers here!");
9420 assert(InputsFixed[1] != (InputsFixed[0] ^ 1) &&
9421 "Cannot have adjacent inputs here!");
9423 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
9424 SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1;
9426 // We also have to update the final source mask in this case because
9427 // it may need to undo the above swap.
9428 for (int &M : FinalSourceHalfMask)
9429 if (M == (InputsFixed[0] ^ 1) + SourceOffset)
9430 M = InputsFixed[1] + SourceOffset;
9431 else if (M == InputsFixed[1] + SourceOffset)
9432 M = (InputsFixed[0] ^ 1) + SourceOffset;
9434 InputsFixed[1] = InputsFixed[0] ^ 1;
9437 // Point everything at the fixed inputs.
9438 for (int &M : HalfMask)
9439 if (M == IncomingInputs[0])
9440 M = InputsFixed[0] + SourceOffset;
9441 else if (M == IncomingInputs[1])
9442 M = InputsFixed[1] + SourceOffset;
9444 IncomingInputs[0] = InputsFixed[0] + SourceOffset;
9445 IncomingInputs[1] = InputsFixed[1] + SourceOffset;
9448 llvm_unreachable("Unhandled input size!");
9451 // Now hoist the DWord down to the right half.
9452 int FreeDWord = (PSHUFDMask[DestOffset / 2] == -1 ? 0 : 1) + DestOffset / 2;
9453 assert(PSHUFDMask[FreeDWord] == -1 && "DWord not free");
9454 PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
9455 for (int &M : HalfMask)
9456 for (int Input : IncomingInputs)
9458 M = FreeDWord * 2 + Input % 2;
9460 moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask,
9461 /*SourceOffset*/ 4, /*DestOffset*/ 0);
9462 moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask,
9463 /*SourceOffset*/ 0, /*DestOffset*/ 4);
9465 // Now enact all the shuffles we've computed to move the inputs into their
9467 if (!isNoopShuffleMask(PSHUFLMask))
9468 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
9469 getV4X86ShuffleImm8ForMask(PSHUFLMask, DAG));
9470 if (!isNoopShuffleMask(PSHUFHMask))
9471 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
9472 getV4X86ShuffleImm8ForMask(PSHUFHMask, DAG));
9473 if (!isNoopShuffleMask(PSHUFDMask))
9474 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
9475 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
9476 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V),
9477 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
9479 // At this point, each half should contain all its inputs, and we can then
9480 // just shuffle them into their final position.
9481 assert(std::count_if(LoMask.begin(), LoMask.end(),
9482 [](int M) { return M >= 4; }) == 0 &&
9483 "Failed to lift all the high half inputs to the low mask!");
9484 assert(std::count_if(HiMask.begin(), HiMask.end(),
9485 [](int M) { return M >= 0 && M < 4; }) == 0 &&
9486 "Failed to lift all the low half inputs to the high mask!");
9488 // Do a half shuffle for the low mask.
9489 if (!isNoopShuffleMask(LoMask))
9490 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
9491 getV4X86ShuffleImm8ForMask(LoMask, DAG));
9493 // Do a half shuffle with the high mask after shifting its values down.
9494 for (int &M : HiMask)
9497 if (!isNoopShuffleMask(HiMask))
9498 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
9499 getV4X86ShuffleImm8ForMask(HiMask, DAG));
9504 /// \brief Detect whether the mask pattern should be lowered through
9507 /// This essentially tests whether viewing the mask as an interleaving of two
9508 /// sub-sequences reduces the cross-input traffic of a blend operation. If so,
9509 /// lowering it through interleaving is a significantly better strategy.
9510 static bool shouldLowerAsInterleaving(ArrayRef<int> Mask) {
9511 int NumEvenInputs[2] = {0, 0};
9512 int NumOddInputs[2] = {0, 0};
9513 int NumLoInputs[2] = {0, 0};
9514 int NumHiInputs[2] = {0, 0};
9515 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
9519 int InputIdx = Mask[i] >= Size;
9522 ++NumLoInputs[InputIdx];
9524 ++NumHiInputs[InputIdx];
9527 ++NumEvenInputs[InputIdx];
9529 ++NumOddInputs[InputIdx];
9532 // The minimum number of cross-input results for both the interleaved and
9533 // split cases. If interleaving results in fewer cross-input results, return
9535 int InterleavedCrosses = std::min(NumEvenInputs[1] + NumOddInputs[0],
9536 NumEvenInputs[0] + NumOddInputs[1]);
9537 int SplitCrosses = std::min(NumLoInputs[1] + NumHiInputs[0],
9538 NumLoInputs[0] + NumHiInputs[1]);
9539 return InterleavedCrosses < SplitCrosses;
9542 /// \brief Blend two v8i16 vectors using a naive unpack strategy.
9544 /// This strategy only works when the inputs from each vector fit into a single
9545 /// half of that vector, and generally there are not so many inputs as to leave
9546 /// the in-place shuffles required highly constrained (and thus expensive). It
9547 /// shifts all the inputs into a single side of both input vectors and then
9548 /// uses an unpack to interleave these inputs in a single vector. At that
9549 /// point, we will fall back on the generic single input shuffle lowering.
9550 static SDValue lowerV8I16BasicBlendVectorShuffle(SDLoc DL, SDValue V1,
9552 MutableArrayRef<int> Mask,
9553 const X86Subtarget *Subtarget,
9554 SelectionDAG &DAG) {
9555 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
9556 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad input type!");
9557 SmallVector<int, 3> LoV1Inputs, HiV1Inputs, LoV2Inputs, HiV2Inputs;
9558 for (int i = 0; i < 8; ++i)
9559 if (Mask[i] >= 0 && Mask[i] < 4)
9560 LoV1Inputs.push_back(i);
9561 else if (Mask[i] >= 4 && Mask[i] < 8)
9562 HiV1Inputs.push_back(i);
9563 else if (Mask[i] >= 8 && Mask[i] < 12)
9564 LoV2Inputs.push_back(i);
9565 else if (Mask[i] >= 12)
9566 HiV2Inputs.push_back(i);
9568 int NumV1Inputs = LoV1Inputs.size() + HiV1Inputs.size();
9569 int NumV2Inputs = LoV2Inputs.size() + HiV2Inputs.size();
9572 assert(NumV1Inputs > 0 && NumV1Inputs <= 3 && "At most 3 inputs supported");
9573 assert(NumV2Inputs > 0 && NumV2Inputs <= 3 && "At most 3 inputs supported");
9574 assert(NumV1Inputs + NumV2Inputs <= 4 && "At most 4 combined inputs");
9576 bool MergeFromLo = LoV1Inputs.size() + LoV2Inputs.size() >=
9577 HiV1Inputs.size() + HiV2Inputs.size();
9579 auto moveInputsToHalf = [&](SDValue V, ArrayRef<int> LoInputs,
9580 ArrayRef<int> HiInputs, bool MoveToLo,
9582 ArrayRef<int> GoodInputs = MoveToLo ? LoInputs : HiInputs;
9583 ArrayRef<int> BadInputs = MoveToLo ? HiInputs : LoInputs;
9584 if (BadInputs.empty())
9587 int MoveMask[] = {-1, -1, -1, -1, -1, -1, -1, -1};
9588 int MoveOffset = MoveToLo ? 0 : 4;
9590 if (GoodInputs.empty()) {
9591 for (int BadInput : BadInputs) {
9592 MoveMask[Mask[BadInput] % 4 + MoveOffset] = Mask[BadInput] - MaskOffset;
9593 Mask[BadInput] = Mask[BadInput] % 4 + MoveOffset + MaskOffset;
9596 if (GoodInputs.size() == 2) {
9597 // If the low inputs are spread across two dwords, pack them into
9599 MoveMask[MoveOffset] = Mask[GoodInputs[0]] - MaskOffset;
9600 MoveMask[MoveOffset + 1] = Mask[GoodInputs[1]] - MaskOffset;
9601 Mask[GoodInputs[0]] = MoveOffset + MaskOffset;
9602 Mask[GoodInputs[1]] = MoveOffset + 1 + MaskOffset;
9604 // Otherwise pin the good inputs.
9605 for (int GoodInput : GoodInputs)
9606 MoveMask[Mask[GoodInput] - MaskOffset] = Mask[GoodInput] - MaskOffset;
9609 if (BadInputs.size() == 2) {
9610 // If we have two bad inputs then there may be either one or two good
9611 // inputs fixed in place. Find a fixed input, and then find the *other*
9612 // two adjacent indices by using modular arithmetic.
9614 std::find_if(std::begin(MoveMask) + MoveOffset, std::end(MoveMask),
9615 [](int M) { return M >= 0; }) -
9616 std::begin(MoveMask);
9618 ((((GoodMaskIdx - MoveOffset) & ~1) + 2) % 4) + MoveOffset;
9619 assert(MoveMask[MoveMaskIdx] == -1 && "Expected empty slot");
9620 assert(MoveMask[MoveMaskIdx + 1] == -1 && "Expected empty slot");
9621 MoveMask[MoveMaskIdx] = Mask[BadInputs[0]] - MaskOffset;
9622 MoveMask[MoveMaskIdx + 1] = Mask[BadInputs[1]] - MaskOffset;
9623 Mask[BadInputs[0]] = MoveMaskIdx + MaskOffset;
9624 Mask[BadInputs[1]] = MoveMaskIdx + 1 + MaskOffset;
9626 assert(BadInputs.size() == 1 && "All sizes handled");
9627 int MoveMaskIdx = std::find(std::begin(MoveMask) + MoveOffset,
9628 std::end(MoveMask), -1) -
9629 std::begin(MoveMask);
9630 MoveMask[MoveMaskIdx] = Mask[BadInputs[0]] - MaskOffset;
9631 Mask[BadInputs[0]] = MoveMaskIdx + MaskOffset;
9635 return DAG.getVectorShuffle(MVT::v8i16, DL, V, DAG.getUNDEF(MVT::v8i16),
9638 V1 = moveInputsToHalf(V1, LoV1Inputs, HiV1Inputs, MergeFromLo,
9640 V2 = moveInputsToHalf(V2, LoV2Inputs, HiV2Inputs, MergeFromLo,
9643 // FIXME: Select an interleaving of the merge of V1 and V2 that minimizes
9644 // cross-half traffic in the final shuffle.
9646 // Munge the mask to be a single-input mask after the unpack merges the
9650 M = 2 * (M % 4) + (M / 8);
9652 return DAG.getVectorShuffle(
9653 MVT::v8i16, DL, DAG.getNode(MergeFromLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
9654 DL, MVT::v8i16, V1, V2),
9655 DAG.getUNDEF(MVT::v8i16), Mask);
9658 /// \brief Generic lowering of 8-lane i16 shuffles.
9660 /// This handles both single-input shuffles and combined shuffle/blends with
9661 /// two inputs. The single input shuffles are immediately delegated to
9662 /// a dedicated lowering routine.
9664 /// The blends are lowered in one of three fundamental ways. If there are few
9665 /// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
9666 /// of the input is significantly cheaper when lowered as an interleaving of
9667 /// the two inputs, try to interleave them. Otherwise, blend the low and high
9668 /// halves of the inputs separately (making them have relatively few inputs)
9669 /// and then concatenate them.
9670 static SDValue lowerV8I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
9671 const X86Subtarget *Subtarget,
9672 SelectionDAG &DAG) {
9674 assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!");
9675 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
9676 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
9677 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
9678 ArrayRef<int> OrigMask = SVOp->getMask();
9679 int MaskStorage[8] = {OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3],
9680 OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7]};
9681 MutableArrayRef<int> Mask(MaskStorage);
9683 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
9685 // Whenever we can lower this as a zext, that instruction is strictly faster
9686 // than any alternative.
9687 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
9688 DL, MVT::v8i16, V1, V2, OrigMask, Subtarget, DAG))
9691 auto isV1 = [](int M) { return M >= 0 && M < 8; };
9692 auto isV2 = [](int M) { return M >= 8; };
9694 int NumV1Inputs = std::count_if(Mask.begin(), Mask.end(), isV1);
9695 int NumV2Inputs = std::count_if(Mask.begin(), Mask.end(), isV2);
9697 if (NumV2Inputs == 0)
9698 return lowerV8I16SingleInputVectorShuffle(DL, V1, Mask, Subtarget, DAG);
9700 assert(NumV1Inputs > 0 && "All single-input shuffles should be canonicalized "
9701 "to be V1-input shuffles.");
9703 // Try to use bit shift instructions.
9704 if (SDValue Shift = lowerVectorShuffleAsBitShift(
9705 DL, MVT::v8i16, V1, V2, Mask, DAG))
9708 // Try to use byte shift instructions.
9709 if (SDValue Shift = lowerVectorShuffleAsByteShift(
9710 DL, MVT::v8i16, V1, V2, Mask, DAG))
9713 // There are special ways we can lower some single-element blends.
9714 if (NumV2Inputs == 1)
9715 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v8i16, DL, V1, V2,
9716 Mask, Subtarget, DAG))
9719 // We have different paths for blend lowering, but they all must use the
9720 // *exact* same predicate.
9721 bool IsBlendSupported = Subtarget->hasSSE41();
9722 if (IsBlendSupported)
9723 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
9727 if (SDValue Masked =
9728 lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG))
9731 // Use dedicated unpack instructions for masks that match their pattern.
9732 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 2, 10, 3, 11))
9733 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, V1, V2);
9734 if (isShuffleEquivalent(V1, V2, Mask, 4, 12, 5, 13, 6, 14, 7, 15))
9735 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i16, V1, V2);
9737 // Try to use byte rotation instructions.
9738 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9739 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
9742 if (NumV1Inputs + NumV2Inputs <= 4)
9743 return lowerV8I16BasicBlendVectorShuffle(DL, V1, V2, Mask, Subtarget, DAG);
9745 // Check whether an interleaving lowering is likely to be more efficient.
9746 // This isn't perfect but it is a strong heuristic that tends to work well on
9747 // the kinds of shuffles that show up in practice.
9749 // FIXME: Handle 1x, 2x, and 4x interleaving.
9750 if (shouldLowerAsInterleaving(Mask)) {
9751 // FIXME: Figure out whether we should pack these into the low or high
9754 int EMask[8], OMask[8];
9755 for (int i = 0; i < 4; ++i) {
9756 EMask[i] = Mask[2*i];
9757 OMask[i] = Mask[2*i + 1];
9762 SDValue Evens = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, EMask);
9763 SDValue Odds = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, OMask);
9765 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, Evens, Odds);
9768 // If we have direct support for blends, we should lower by decomposing into
9770 if (IsBlendSupported)
9771 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
9774 // Try to lower by permuting the inputs into an unpack instruction.
9775 if (SDValue Unpack =
9776 lowerVectorShuffleAsUnpack(MVT::v8i16, DL, V1, V2, Mask, DAG))
9779 int LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
9780 int HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
9782 for (int i = 0; i < 4; ++i) {
9783 LoBlendMask[i] = Mask[i];
9784 HiBlendMask[i] = Mask[i + 4];
9787 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask);
9788 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask);
9789 LoV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, LoV);
9790 HiV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, HiV);
9792 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
9793 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, LoV, HiV));
9796 /// \brief Check whether a compaction lowering can be done by dropping even
9797 /// elements and compute how many times even elements must be dropped.
9799 /// This handles shuffles which take every Nth element where N is a power of
9800 /// two. Example shuffle masks:
9802 /// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14
9803 /// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
9804 /// N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12
9805 /// N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28
9806 /// N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8
9807 /// N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
9809 /// Any of these lanes can of course be undef.
9811 /// This routine only supports N <= 3.
9812 /// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
9815 /// \returns N above, or the number of times even elements must be dropped if
9816 /// there is such a number. Otherwise returns zero.
9817 static int canLowerByDroppingEvenElements(ArrayRef<int> Mask) {
9818 // Figure out whether we're looping over two inputs or just one.
9819 bool IsSingleInput = isSingleInputShuffleMask(Mask);
9821 // The modulus for the shuffle vector entries is based on whether this is
9822 // a single input or not.
9823 int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
9824 assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&
9825 "We should only be called with masks with a power-of-2 size!");
9827 uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
9829 // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
9830 // and 2^3 simultaneously. This is because we may have ambiguity with
9831 // partially undef inputs.
9832 bool ViableForN[3] = {true, true, true};
9834 for (int i = 0, e = Mask.size(); i < e; ++i) {
9835 // Ignore undef lanes, we'll optimistically collapse them to the pattern we
9840 bool IsAnyViable = false;
9841 for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
9842 if (ViableForN[j]) {
9845 // The shuffle mask must be equal to (i * 2^N) % M.
9846 if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask))
9849 ViableForN[j] = false;
9851 // Early exit if we exhaust the possible powers of two.
9856 for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
9860 // Return 0 as there is no viable power of two.
9864 /// \brief Generic lowering of v16i8 shuffles.
9866 /// This is a hybrid strategy to lower v16i8 vectors. It first attempts to
9867 /// detect any complexity reducing interleaving. If that doesn't help, it uses
9868 /// UNPCK to spread the i8 elements across two i16-element vectors, and uses
9869 /// the existing lowering for v8i16 blends on each half, finally PACK-ing them
9871 static SDValue lowerV16I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
9872 const X86Subtarget *Subtarget,
9873 SelectionDAG &DAG) {
9875 assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!");
9876 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
9877 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
9878 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
9879 ArrayRef<int> OrigMask = SVOp->getMask();
9880 assert(OrigMask.size() == 16 && "Unexpected mask size for v16 shuffle!");
9882 // Try to use bit shift instructions.
9883 if (SDValue Shift = lowerVectorShuffleAsBitShift(
9884 DL, MVT::v16i8, V1, V2, OrigMask, DAG))
9887 // Try to use byte shift instructions.
9888 if (SDValue Shift = lowerVectorShuffleAsByteShift(
9889 DL, MVT::v16i8, V1, V2, OrigMask, DAG))
9892 // Try to use byte rotation instructions.
9893 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
9894 DL, MVT::v16i8, V1, V2, OrigMask, Subtarget, DAG))
9897 // Try to use a zext lowering.
9898 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
9899 DL, MVT::v16i8, V1, V2, OrigMask, Subtarget, DAG))
9902 int MaskStorage[16] = {
9903 OrigMask[0], OrigMask[1], OrigMask[2], OrigMask[3],
9904 OrigMask[4], OrigMask[5], OrigMask[6], OrigMask[7],
9905 OrigMask[8], OrigMask[9], OrigMask[10], OrigMask[11],
9906 OrigMask[12], OrigMask[13], OrigMask[14], OrigMask[15]};
9907 MutableArrayRef<int> Mask(MaskStorage);
9908 MutableArrayRef<int> LoMask = Mask.slice(0, 8);
9909 MutableArrayRef<int> HiMask = Mask.slice(8, 8);
9912 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 16; });
9914 // For single-input shuffles, there are some nicer lowering tricks we can use.
9915 if (NumV2Elements == 0) {
9916 // Check for being able to broadcast a single element.
9917 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i8, DL, V1,
9918 Mask, Subtarget, DAG))
9921 // Check whether we can widen this to an i16 shuffle by duplicating bytes.
9922 // Notably, this handles splat and partial-splat shuffles more efficiently.
9923 // However, it only makes sense if the pre-duplication shuffle simplifies
9924 // things significantly. Currently, this means we need to be able to
9925 // express the pre-duplication shuffle as an i16 shuffle.
9927 // FIXME: We should check for other patterns which can be widened into an
9928 // i16 shuffle as well.
9929 auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
9930 for (int i = 0; i < 16; i += 2)
9931 if (Mask[i] != -1 && Mask[i + 1] != -1 && Mask[i] != Mask[i + 1])
9936 auto tryToWidenViaDuplication = [&]() -> SDValue {
9937 if (!canWidenViaDuplication(Mask))
9939 SmallVector<int, 4> LoInputs;
9940 std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(LoInputs),
9941 [](int M) { return M >= 0 && M < 8; });
9942 std::sort(LoInputs.begin(), LoInputs.end());
9943 LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()),
9945 SmallVector<int, 4> HiInputs;
9946 std::copy_if(Mask.begin(), Mask.end(), std::back_inserter(HiInputs),
9947 [](int M) { return M >= 8; });
9948 std::sort(HiInputs.begin(), HiInputs.end());
9949 HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()),
9952 bool TargetLo = LoInputs.size() >= HiInputs.size();
9953 ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
9954 ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
9956 int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
9957 SmallDenseMap<int, int, 8> LaneMap;
9958 for (int I : InPlaceInputs) {
9959 PreDupI16Shuffle[I/2] = I/2;
9962 int j = TargetLo ? 0 : 4, je = j + 4;
9963 for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
9964 // Check if j is already a shuffle of this input. This happens when
9965 // there are two adjacent bytes after we move the low one.
9966 if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
9967 // If we haven't yet mapped the input, search for a slot into which
9969 while (j < je && PreDupI16Shuffle[j] != -1)
9973 // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
9976 // Map this input with the i16 shuffle.
9977 PreDupI16Shuffle[j] = MovingInputs[i] / 2;
9980 // Update the lane map based on the mapping we ended up with.
9981 LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
9984 ISD::BITCAST, DL, MVT::v16i8,
9985 DAG.getVectorShuffle(MVT::v8i16, DL,
9986 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
9987 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
9989 // Unpack the bytes to form the i16s that will be shuffled into place.
9990 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
9991 MVT::v16i8, V1, V1);
9993 int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
9994 for (int i = 0; i < 16; ++i)
9995 if (Mask[i] != -1) {
9996 int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
9997 assert(MappedMask < 8 && "Invalid v8 shuffle mask!");
9998 if (PostDupI16Shuffle[i / 2] == -1)
9999 PostDupI16Shuffle[i / 2] = MappedMask;
10001 assert(PostDupI16Shuffle[i / 2] == MappedMask &&
10002 "Conflicting entrties in the original shuffle!");
10004 return DAG.getNode(
10005 ISD::BITCAST, DL, MVT::v16i8,
10006 DAG.getVectorShuffle(MVT::v8i16, DL,
10007 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1),
10008 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
10010 if (SDValue V = tryToWidenViaDuplication())
10014 // Check whether an interleaving lowering is likely to be more efficient.
10015 // This isn't perfect but it is a strong heuristic that tends to work well on
10016 // the kinds of shuffles that show up in practice.
10018 // FIXME: We need to handle other interleaving widths (i16, i32, ...).
10019 if (shouldLowerAsInterleaving(Mask)) {
10020 int NumLoHalf = std::count_if(Mask.begin(), Mask.end(), [](int M) {
10021 return (M >= 0 && M < 8) || (M >= 16 && M < 24);
10023 int NumHiHalf = std::count_if(Mask.begin(), Mask.end(), [](int M) {
10024 return (M >= 8 && M < 16) || M >= 24;
10026 int EMask[16] = {-1, -1, -1, -1, -1, -1, -1, -1,
10027 -1, -1, -1, -1, -1, -1, -1, -1};
10028 int OMask[16] = {-1, -1, -1, -1, -1, -1, -1, -1,
10029 -1, -1, -1, -1, -1, -1, -1, -1};
10030 bool UnpackLo = NumLoHalf >= NumHiHalf;
10031 MutableArrayRef<int> TargetEMask(UnpackLo ? EMask : EMask + 8, 8);
10032 MutableArrayRef<int> TargetOMask(UnpackLo ? OMask : OMask + 8, 8);
10033 for (int i = 0; i < 8; ++i) {
10034 TargetEMask[i] = Mask[2 * i];
10035 TargetOMask[i] = Mask[2 * i + 1];
10038 SDValue Evens = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, EMask);
10039 SDValue Odds = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2, OMask);
10041 return DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
10042 MVT::v16i8, Evens, Odds);
10045 // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
10046 // with PSHUFB. It is important to do this before we attempt to generate any
10047 // blends but after all of the single-input lowerings. If the single input
10048 // lowerings can find an instruction sequence that is faster than a PSHUFB, we
10049 // want to preserve that and we can DAG combine any longer sequences into
10050 // a PSHUFB in the end. But once we start blending from multiple inputs,
10051 // the complexity of DAG combining bad patterns back into PSHUFB is too high,
10052 // and there are *very* few patterns that would actually be faster than the
10053 // PSHUFB approach because of its ability to zero lanes.
10055 // FIXME: The only exceptions to the above are blends which are exact
10056 // interleavings with direct instructions supporting them. We currently don't
10057 // handle those well here.
10058 if (Subtarget->hasSSSE3()) {
10059 SDValue V1Mask[16];
10060 SDValue V2Mask[16];
10061 bool V1InUse = false;
10062 bool V2InUse = false;
10063 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
10065 for (int i = 0; i < 16; ++i) {
10066 if (Mask[i] == -1) {
10067 V1Mask[i] = V2Mask[i] = DAG.getUNDEF(MVT::i8);
10069 const int ZeroMask = 0x80;
10070 int V1Idx = (Mask[i] < 16 ? Mask[i] : ZeroMask);
10071 int V2Idx = (Mask[i] < 16 ? ZeroMask : Mask[i] - 16);
10073 V1Idx = V2Idx = ZeroMask;
10074 V1Mask[i] = DAG.getConstant(V1Idx, MVT::i8);
10075 V2Mask[i] = DAG.getConstant(V2Idx, MVT::i8);
10076 V1InUse |= (ZeroMask != V1Idx);
10077 V2InUse |= (ZeroMask != V2Idx);
10081 // If both V1 and V2 are in use and we can use a direct blend, do so. This
10082 // avoids using blends to handle blends-with-zero which is important as
10083 // a single pshufb is significantly faster for that.
10084 if (V1InUse && V2InUse && Subtarget->hasSSE41())
10085 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
10091 V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, V1,
10092 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask));
10094 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, V2,
10095 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask));
10097 // If we need shuffled inputs from both, blend the two.
10098 if (V1InUse && V2InUse)
10099 return DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2);
10101 return V1; // Single inputs are easy.
10103 return V2; // Single inputs are easy.
10104 // Shuffling to a zeroable vector.
10105 return getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
10108 // There are special ways we can lower some single-element blends.
10109 if (NumV2Elements == 1)
10110 if (SDValue V = lowerVectorShuffleAsElementInsertion(MVT::v16i8, DL, V1, V2,
10111 Mask, Subtarget, DAG))
10114 // Check whether a compaction lowering can be done. This handles shuffles
10115 // which take every Nth element for some even N. See the helper function for
10118 // We special case these as they can be particularly efficiently handled with
10119 // the PACKUSB instruction on x86 and they show up in common patterns of
10120 // rearranging bytes to truncate wide elements.
10121 if (int NumEvenDrops = canLowerByDroppingEvenElements(Mask)) {
10122 // NumEvenDrops is the power of two stride of the elements. Another way of
10123 // thinking about it is that we need to drop the even elements this many
10124 // times to get the original input.
10125 bool IsSingleInput = isSingleInputShuffleMask(Mask);
10127 // First we need to zero all the dropped bytes.
10128 assert(NumEvenDrops <= 3 &&
10129 "No support for dropping even elements more than 3 times.");
10130 // We use the mask type to pick which bytes are preserved based on how many
10131 // elements are dropped.
10132 MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 };
10133 SDValue ByteClearMask =
10134 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8,
10135 DAG.getConstant(0xFF, MaskVTs[NumEvenDrops - 1]));
10136 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask);
10137 if (!IsSingleInput)
10138 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask);
10140 // Now pack things back together.
10141 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1);
10142 V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2);
10143 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
10144 for (int i = 1; i < NumEvenDrops; ++i) {
10145 Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result);
10146 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
10152 int V1LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10153 int V1HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10154 int V2LoBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10155 int V2HiBlendMask[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
10157 auto buildBlendMasks = [](MutableArrayRef<int> HalfMask,
10158 MutableArrayRef<int> V1HalfBlendMask,
10159 MutableArrayRef<int> V2HalfBlendMask) {
10160 for (int i = 0; i < 8; ++i)
10161 if (HalfMask[i] >= 0 && HalfMask[i] < 16) {
10162 V1HalfBlendMask[i] = HalfMask[i];
10164 } else if (HalfMask[i] >= 16) {
10165 V2HalfBlendMask[i] = HalfMask[i] - 16;
10166 HalfMask[i] = i + 8;
10169 buildBlendMasks(LoMask, V1LoBlendMask, V2LoBlendMask);
10170 buildBlendMasks(HiMask, V1HiBlendMask, V2HiBlendMask);
10172 SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL);
10174 auto buildLoAndHiV8s = [&](SDValue V, MutableArrayRef<int> LoBlendMask,
10175 MutableArrayRef<int> HiBlendMask) {
10177 // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
10178 // them out and avoid using UNPCK{L,H} to extract the elements of V as
10180 if (std::none_of(LoBlendMask.begin(), LoBlendMask.end(),
10181 [](int M) { return M >= 0 && M % 2 == 1; }) &&
10182 std::none_of(HiBlendMask.begin(), HiBlendMask.end(),
10183 [](int M) { return M >= 0 && M % 2 == 1; })) {
10184 // Use a mask to drop the high bytes.
10185 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
10186 V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, V1,
10187 DAG.getConstant(0x00FF, MVT::v8i16));
10189 // This will be a single vector shuffle instead of a blend so nuke V2.
10190 V2 = DAG.getUNDEF(MVT::v8i16);
10192 // Squash the masks to point directly into V1.
10193 for (int &M : LoBlendMask)
10196 for (int &M : HiBlendMask)
10200 // Otherwise just unpack the low half of V into V1 and the high half into
10201 // V2 so that we can blend them as i16s.
10202 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
10203 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
10204 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16,
10205 DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
10208 SDValue BlendedLo = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, LoBlendMask);
10209 SDValue BlendedHi = DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, HiBlendMask);
10210 return std::make_pair(BlendedLo, BlendedHi);
10212 SDValue V1Lo, V1Hi, V2Lo, V2Hi;
10213 std::tie(V1Lo, V1Hi) = buildLoAndHiV8s(V1, V1LoBlendMask, V1HiBlendMask);
10214 std::tie(V2Lo, V2Hi) = buildLoAndHiV8s(V2, V2LoBlendMask, V2HiBlendMask);
10216 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Lo, V2Lo, LoMask);
10217 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, V1Hi, V2Hi, HiMask);
10219 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
10222 /// \brief Dispatching routine to lower various 128-bit x86 vector shuffles.
10224 /// This routine breaks down the specific type of 128-bit shuffle and
10225 /// dispatches to the lowering routines accordingly.
10226 static SDValue lower128BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10227 MVT VT, const X86Subtarget *Subtarget,
10228 SelectionDAG &DAG) {
10229 switch (VT.SimpleTy) {
10231 return lowerV2I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
10233 return lowerV2F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
10235 return lowerV4I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
10237 return lowerV4F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
10239 return lowerV8I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
10241 return lowerV16I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
10244 llvm_unreachable("Unimplemented!");
10248 /// \brief Helper function to test whether a shuffle mask could be
10249 /// simplified by widening the elements being shuffled.
10251 /// Appends the mask for wider elements in WidenedMask if valid. Otherwise
10252 /// leaves it in an unspecified state.
10254 /// NOTE: This must handle normal vector shuffle masks and *target* vector
10255 /// shuffle masks. The latter have the special property of a '-2' representing
10256 /// a zero-ed lane of a vector.
10257 static bool canWidenShuffleElements(ArrayRef<int> Mask,
10258 SmallVectorImpl<int> &WidenedMask) {
10259 for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
10260 // If both elements are undef, its trivial.
10261 if (Mask[i] == SM_SentinelUndef && Mask[i + 1] == SM_SentinelUndef) {
10262 WidenedMask.push_back(SM_SentinelUndef);
10266 // Check for an undef mask and a mask value properly aligned to fit with
10267 // a pair of values. If we find such a case, use the non-undef mask's value.
10268 if (Mask[i] == SM_SentinelUndef && Mask[i + 1] >= 0 && Mask[i + 1] % 2 == 1) {
10269 WidenedMask.push_back(Mask[i + 1] / 2);
10272 if (Mask[i + 1] == SM_SentinelUndef && Mask[i] >= 0 && Mask[i] % 2 == 0) {
10273 WidenedMask.push_back(Mask[i] / 2);
10277 // When zeroing, we need to spread the zeroing across both lanes to widen.
10278 if (Mask[i] == SM_SentinelZero || Mask[i + 1] == SM_SentinelZero) {
10279 if ((Mask[i] == SM_SentinelZero || Mask[i] == SM_SentinelUndef) &&
10280 (Mask[i + 1] == SM_SentinelZero || Mask[i + 1] == SM_SentinelUndef)) {
10281 WidenedMask.push_back(SM_SentinelZero);
10287 // Finally check if the two mask values are adjacent and aligned with
10289 if (Mask[i] != SM_SentinelUndef && Mask[i] % 2 == 0 && Mask[i] + 1 == Mask[i + 1]) {
10290 WidenedMask.push_back(Mask[i] / 2);
10294 // Otherwise we can't safely widen the elements used in this shuffle.
10297 assert(WidenedMask.size() == Mask.size() / 2 &&
10298 "Incorrect size of mask after widening the elements!");
10303 /// \brief Generic routine to split vector shuffle into half-sized shuffles.
10305 /// This routine just extracts two subvectors, shuffles them independently, and
10306 /// then concatenates them back together. This should work effectively with all
10307 /// AVX vector shuffle types.
10308 static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1,
10309 SDValue V2, ArrayRef<int> Mask,
10310 SelectionDAG &DAG) {
10311 assert(VT.getSizeInBits() >= 256 &&
10312 "Only for 256-bit or wider vector shuffles!");
10313 assert(V1.getSimpleValueType() == VT && "Bad operand type!");
10314 assert(V2.getSimpleValueType() == VT && "Bad operand type!");
10316 ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
10317 ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
10319 int NumElements = VT.getVectorNumElements();
10320 int SplitNumElements = NumElements / 2;
10321 MVT ScalarVT = VT.getScalarType();
10322 MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2);
10324 // Rather than splitting build-vectors, just build two narrower build
10325 // vectors. This helps shuffling with splats and zeros.
10326 auto SplitVector = [&](SDValue V) {
10327 while (V.getOpcode() == ISD::BITCAST)
10328 V = V->getOperand(0);
10330 MVT OrigVT = V.getSimpleValueType();
10331 int OrigNumElements = OrigVT.getVectorNumElements();
10332 int OrigSplitNumElements = OrigNumElements / 2;
10333 MVT OrigScalarVT = OrigVT.getScalarType();
10334 MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2);
10338 auto *BV = dyn_cast<BuildVectorSDNode>(V);
10340 LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
10341 DAG.getIntPtrConstant(0));
10342 HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
10343 DAG.getIntPtrConstant(OrigSplitNumElements));
10346 SmallVector<SDValue, 16> LoOps, HiOps;
10347 for (int i = 0; i < OrigSplitNumElements; ++i) {
10348 LoOps.push_back(BV->getOperand(i));
10349 HiOps.push_back(BV->getOperand(i + OrigSplitNumElements));
10351 LoV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, LoOps);
10352 HiV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, HiOps);
10354 return std::make_pair(DAG.getNode(ISD::BITCAST, DL, SplitVT, LoV),
10355 DAG.getNode(ISD::BITCAST, DL, SplitVT, HiV));
10358 SDValue LoV1, HiV1, LoV2, HiV2;
10359 std::tie(LoV1, HiV1) = SplitVector(V1);
10360 std::tie(LoV2, HiV2) = SplitVector(V2);
10362 // Now create two 4-way blends of these half-width vectors.
10363 auto HalfBlend = [&](ArrayRef<int> HalfMask) {
10364 bool UseLoV1 = false, UseHiV1 = false, UseLoV2 = false, UseHiV2 = false;
10365 SmallVector<int, 32> V1BlendMask, V2BlendMask, BlendMask;
10366 for (int i = 0; i < SplitNumElements; ++i) {
10367 int M = HalfMask[i];
10368 if (M >= NumElements) {
10369 if (M >= NumElements + SplitNumElements)
10373 V2BlendMask.push_back(M - NumElements);
10374 V1BlendMask.push_back(-1);
10375 BlendMask.push_back(SplitNumElements + i);
10376 } else if (M >= 0) {
10377 if (M >= SplitNumElements)
10381 V2BlendMask.push_back(-1);
10382 V1BlendMask.push_back(M);
10383 BlendMask.push_back(i);
10385 V2BlendMask.push_back(-1);
10386 V1BlendMask.push_back(-1);
10387 BlendMask.push_back(-1);
10391 // Because the lowering happens after all combining takes place, we need to
10392 // manually combine these blend masks as much as possible so that we create
10393 // a minimal number of high-level vector shuffle nodes.
10395 // First try just blending the halves of V1 or V2.
10396 if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2)
10397 return DAG.getUNDEF(SplitVT);
10398 if (!UseLoV2 && !UseHiV2)
10399 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
10400 if (!UseLoV1 && !UseHiV1)
10401 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
10403 SDValue V1Blend, V2Blend;
10404 if (UseLoV1 && UseHiV1) {
10406 DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
10408 // We only use half of V1 so map the usage down into the final blend mask.
10409 V1Blend = UseLoV1 ? LoV1 : HiV1;
10410 for (int i = 0; i < SplitNumElements; ++i)
10411 if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements)
10412 BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements);
10414 if (UseLoV2 && UseHiV2) {
10416 DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
10418 // We only use half of V2 so map the usage down into the final blend mask.
10419 V2Blend = UseLoV2 ? LoV2 : HiV2;
10420 for (int i = 0; i < SplitNumElements; ++i)
10421 if (BlendMask[i] >= SplitNumElements)
10422 BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0);
10424 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
10426 SDValue Lo = HalfBlend(LoMask);
10427 SDValue Hi = HalfBlend(HiMask);
10428 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
10431 /// \brief Either split a vector in halves or decompose the shuffles and the
10434 /// This is provided as a good fallback for many lowerings of non-single-input
10435 /// shuffles with more than one 128-bit lane. In those cases, we want to select
10436 /// between splitting the shuffle into 128-bit components and stitching those
10437 /// back together vs. extracting the single-input shuffles and blending those
10439 static SDValue lowerVectorShuffleAsSplitOrBlend(SDLoc DL, MVT VT, SDValue V1,
10440 SDValue V2, ArrayRef<int> Mask,
10441 SelectionDAG &DAG) {
10442 assert(!isSingleInputShuffleMask(Mask) && "This routine must not be used to "
10443 "lower single-input shuffles as it "
10444 "could then recurse on itself.");
10445 int Size = Mask.size();
10447 // If this can be modeled as a broadcast of two elements followed by a blend,
10448 // prefer that lowering. This is especially important because broadcasts can
10449 // often fold with memory operands.
10450 auto DoBothBroadcast = [&] {
10451 int V1BroadcastIdx = -1, V2BroadcastIdx = -1;
10454 if (V2BroadcastIdx == -1)
10455 V2BroadcastIdx = M - Size;
10456 else if (M - Size != V2BroadcastIdx)
10458 } else if (M >= 0) {
10459 if (V1BroadcastIdx == -1)
10460 V1BroadcastIdx = M;
10461 else if (M != V1BroadcastIdx)
10466 if (DoBothBroadcast())
10467 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
10470 // If the inputs all stem from a single 128-bit lane of each input, then we
10471 // split them rather than blending because the split will decompose to
10472 // unusually few instructions.
10473 int LaneCount = VT.getSizeInBits() / 128;
10474 int LaneSize = Size / LaneCount;
10475 SmallBitVector LaneInputs[2];
10476 LaneInputs[0].resize(LaneCount, false);
10477 LaneInputs[1].resize(LaneCount, false);
10478 for (int i = 0; i < Size; ++i)
10480 LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
10481 if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
10482 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
10484 // Otherwise, just fall back to decomposed shuffles and a blend. This requires
10485 // that the decomposed single-input shuffles don't end up here.
10486 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG);
10489 /// \brief Lower a vector shuffle crossing multiple 128-bit lanes as
10490 /// a permutation and blend of those lanes.
10492 /// This essentially blends the out-of-lane inputs to each lane into the lane
10493 /// from a permuted copy of the vector. This lowering strategy results in four
10494 /// instructions in the worst case for a single-input cross lane shuffle which
10495 /// is lower than any other fully general cross-lane shuffle strategy I'm aware
10496 /// of. Special cases for each particular shuffle pattern should be handled
10497 /// prior to trying this lowering.
10498 static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT,
10499 SDValue V1, SDValue V2,
10500 ArrayRef<int> Mask,
10501 SelectionDAG &DAG) {
10502 // FIXME: This should probably be generalized for 512-bit vectors as well.
10503 assert(VT.getSizeInBits() == 256 && "Only for 256-bit vector shuffles!");
10504 int LaneSize = Mask.size() / 2;
10506 // If there are only inputs from one 128-bit lane, splitting will in fact be
10507 // less expensive. The flags track wether the given lane contains an element
10508 // that crosses to another lane.
10509 bool LaneCrossing[2] = {false, false};
10510 for (int i = 0, Size = Mask.size(); i < Size; ++i)
10511 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
10512 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
10513 if (!LaneCrossing[0] || !LaneCrossing[1])
10514 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
10516 if (isSingleInputShuffleMask(Mask)) {
10517 SmallVector<int, 32> FlippedBlendMask;
10518 for (int i = 0, Size = Mask.size(); i < Size; ++i)
10519 FlippedBlendMask.push_back(
10520 Mask[i] < 0 ? -1 : (((Mask[i] % Size) / LaneSize == i / LaneSize)
10522 : Mask[i] % LaneSize +
10523 (i / LaneSize) * LaneSize + Size));
10525 // Flip the vector, and blend the results which should now be in-lane. The
10526 // VPERM2X128 mask uses the low 2 bits for the low source and bits 4 and
10527 // 5 for the high source. The value 3 selects the high half of source 2 and
10528 // the value 2 selects the low half of source 2. We only use source 2 to
10529 // allow folding it into a memory operand.
10530 unsigned PERMMask = 3 | 2 << 4;
10531 SDValue Flipped = DAG.getNode(X86ISD::VPERM2X128, DL, VT, DAG.getUNDEF(VT),
10532 V1, DAG.getConstant(PERMMask, MVT::i8));
10533 return DAG.getVectorShuffle(VT, DL, V1, Flipped, FlippedBlendMask);
10536 // This now reduces to two single-input shuffles of V1 and V2 which at worst
10537 // will be handled by the above logic and a blend of the results, much like
10538 // other patterns in AVX.
10539 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG);
10542 /// \brief Handle lowering 2-lane 128-bit shuffles.
10543 static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1,
10544 SDValue V2, ArrayRef<int> Mask,
10545 const X86Subtarget *Subtarget,
10546 SelectionDAG &DAG) {
10547 // Blends are faster and handle all the non-lane-crossing cases.
10548 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask,
10552 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),
10553 VT.getVectorNumElements() / 2);
10554 // Check for patterns which can be matched with a single insert of a 128-bit
10556 if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 0, 1) ||
10557 isShuffleEquivalent(V1, V2, Mask, 0, 1, 4, 5)) {
10558 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
10559 DAG.getIntPtrConstant(0));
10560 SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
10561 Mask[2] < 4 ? V1 : V2, DAG.getIntPtrConstant(0));
10562 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV);
10564 if (isShuffleEquivalent(V1, V2, Mask, 0, 1, 6, 7)) {
10565 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
10566 DAG.getIntPtrConstant(0));
10567 SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
10568 DAG.getIntPtrConstant(2));
10569 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV);
10572 // Otherwise form a 128-bit permutation.
10573 // FIXME: Detect zero-vector inputs and use the VPERM2X128 to zero that half.
10574 unsigned PermMask = Mask[0] / 2 | (Mask[2] / 2) << 4;
10575 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
10576 DAG.getConstant(PermMask, MVT::i8));
10579 /// \brief Lower a vector shuffle by first fixing the 128-bit lanes and then
10580 /// shuffling each lane.
10582 /// This will only succeed when the result of fixing the 128-bit lanes results
10583 /// in a single-input non-lane-crossing shuffle with a repeating shuffle mask in
10584 /// each 128-bit lanes. This handles many cases where we can quickly blend away
10585 /// the lane crosses early and then use simpler shuffles within each lane.
10587 /// FIXME: It might be worthwhile at some point to support this without
10588 /// requiring the 128-bit lane-relative shuffles to be repeating, but currently
10589 /// in x86 only floating point has interesting non-repeating shuffles, and even
10590 /// those are still *marginally* more expensive.
10591 static SDValue lowerVectorShuffleByMerging128BitLanes(
10592 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
10593 const X86Subtarget *Subtarget, SelectionDAG &DAG) {
10594 assert(!isSingleInputShuffleMask(Mask) &&
10595 "This is only useful with multiple inputs.");
10597 int Size = Mask.size();
10598 int LaneSize = 128 / VT.getScalarSizeInBits();
10599 int NumLanes = Size / LaneSize;
10600 assert(NumLanes > 1 && "Only handles 256-bit and wider shuffles.");
10602 // See if we can build a hypothetical 128-bit lane-fixing shuffle mask. Also
10603 // check whether the in-128-bit lane shuffles share a repeating pattern.
10604 SmallVector<int, 4> Lanes;
10605 Lanes.resize(NumLanes, -1);
10606 SmallVector<int, 4> InLaneMask;
10607 InLaneMask.resize(LaneSize, -1);
10608 for (int i = 0; i < Size; ++i) {
10612 int j = i / LaneSize;
10614 if (Lanes[j] < 0) {
10615 // First entry we've seen for this lane.
10616 Lanes[j] = Mask[i] / LaneSize;
10617 } else if (Lanes[j] != Mask[i] / LaneSize) {
10618 // This doesn't match the lane selected previously!
10622 // Check that within each lane we have a consistent shuffle mask.
10623 int k = i % LaneSize;
10624 if (InLaneMask[k] < 0) {
10625 InLaneMask[k] = Mask[i] % LaneSize;
10626 } else if (InLaneMask[k] != Mask[i] % LaneSize) {
10627 // This doesn't fit a repeating in-lane mask.
10632 // First shuffle the lanes into place.
10633 MVT LaneVT = MVT::getVectorVT(VT.isFloatingPoint() ? MVT::f64 : MVT::i64,
10634 VT.getSizeInBits() / 64);
10635 SmallVector<int, 8> LaneMask;
10636 LaneMask.resize(NumLanes * 2, -1);
10637 for (int i = 0; i < NumLanes; ++i)
10638 if (Lanes[i] >= 0) {
10639 LaneMask[2 * i + 0] = 2*Lanes[i] + 0;
10640 LaneMask[2 * i + 1] = 2*Lanes[i] + 1;
10643 V1 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V1);
10644 V2 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V2);
10645 SDValue LaneShuffle = DAG.getVectorShuffle(LaneVT, DL, V1, V2, LaneMask);
10647 // Cast it back to the type we actually want.
10648 LaneShuffle = DAG.getNode(ISD::BITCAST, DL, VT, LaneShuffle);
10650 // Now do a simple shuffle that isn't lane crossing.
10651 SmallVector<int, 8> NewMask;
10652 NewMask.resize(Size, -1);
10653 for (int i = 0; i < Size; ++i)
10655 NewMask[i] = (i / LaneSize) * LaneSize + Mask[i] % LaneSize;
10656 assert(!is128BitLaneCrossingShuffleMask(VT, NewMask) &&
10657 "Must not introduce lane crosses at this point!");
10659 return DAG.getVectorShuffle(VT, DL, LaneShuffle, DAG.getUNDEF(VT), NewMask);
10662 /// \brief Test whether the specified input (0 or 1) is in-place blended by the
10665 /// This returns true if the elements from a particular input are already in the
10666 /// slot required by the given mask and require no permutation.
10667 static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
10668 assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.");
10669 int Size = Mask.size();
10670 for (int i = 0; i < Size; ++i)
10671 if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i)
10677 /// \brief Handle lowering of 4-lane 64-bit floating point shuffles.
10679 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
10680 /// isn't available.
10681 static SDValue lowerV4F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10682 const X86Subtarget *Subtarget,
10683 SelectionDAG &DAG) {
10685 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
10686 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
10687 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10688 ArrayRef<int> Mask = SVOp->getMask();
10689 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
10691 SmallVector<int, 4> WidenedMask;
10692 if (canWidenShuffleElements(Mask, WidenedMask))
10693 return lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, Subtarget,
10696 if (isSingleInputShuffleMask(Mask)) {
10697 // Check for being able to broadcast a single element.
10698 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4f64, DL, V1,
10699 Mask, Subtarget, DAG))
10702 // Use low duplicate instructions for masks that match their pattern.
10703 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2))
10704 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
10706 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {
10707 // Non-half-crossing single input shuffles can be lowerid with an
10708 // interleaved permutation.
10709 unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
10710 ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
10711 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
10712 DAG.getConstant(VPERMILPMask, MVT::i8));
10715 // With AVX2 we have direct support for this permutation.
10716 if (Subtarget->hasAVX2())
10717 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
10718 getV4X86ShuffleImm8ForMask(Mask, DAG));
10720 // Otherwise, fall back.
10721 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask,
10725 // X86 has dedicated unpack instructions that can handle specific blend
10726 // operations: UNPCKH and UNPCKL.
10727 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6))
10728 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V1, V2);
10729 if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7))
10730 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V1, V2);
10731 if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 6, 2))
10732 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4f64, V2, V1);
10733 if (isShuffleEquivalent(V1, V2, Mask, 5, 1, 7, 3))
10734 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4f64, V2, V1);
10736 // If we have a single input to the zero element, insert that into V1 if we
10737 // can do so cheaply.
10738 int NumV2Elements =
10739 std::count_if(Mask.begin(), Mask.end(), [](int M) { return M >= 4; });
10740 if (NumV2Elements == 1 && Mask[0] >= 4)
10741 if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
10742 MVT::v4f64, DL, V1, V2, Mask, Subtarget, DAG))
10745 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
10749 // Check if the blend happens to exactly fit that of SHUFPD.
10750 if ((Mask[0] == -1 || Mask[0] < 2) &&
10751 (Mask[1] == -1 || (Mask[1] >= 4 && Mask[1] < 6)) &&
10752 (Mask[2] == -1 || (Mask[2] >= 2 && Mask[2] < 4)) &&
10753 (Mask[3] == -1 || Mask[3] >= 6)) {
10754 unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 5) << 1) |
10755 ((Mask[2] == 3) << 2) | ((Mask[3] == 7) << 3);
10756 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V1, V2,
10757 DAG.getConstant(SHUFPDMask, MVT::i8));
10759 if ((Mask[0] == -1 || (Mask[0] >= 4 && Mask[0] < 6)) &&
10760 (Mask[1] == -1 || Mask[1] < 2) &&
10761 (Mask[2] == -1 || Mask[2] >= 6) &&
10762 (Mask[3] == -1 || (Mask[3] >= 2 && Mask[3] < 4))) {
10763 unsigned SHUFPDMask = (Mask[0] == 5) | ((Mask[1] == 1) << 1) |
10764 ((Mask[2] == 7) << 2) | ((Mask[3] == 3) << 3);
10765 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f64, V2, V1,
10766 DAG.getConstant(SHUFPDMask, MVT::i8));
10769 // Try to simplify this by merging 128-bit lanes to enable a lane-based
10770 // shuffle. However, if we have AVX2 and either inputs are already in place,
10771 // we will be able to shuffle even across lanes the other input in a single
10772 // instruction so skip this pattern.
10773 if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) ||
10774 isShuffleMaskInputInPlace(1, Mask))))
10775 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
10776 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
10779 // If we have AVX2 then we always want to lower with a blend because an v4 we
10780 // can fully permute the elements.
10781 if (Subtarget->hasAVX2())
10782 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2,
10785 // Otherwise fall back on generic lowering.
10786 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG);
10789 /// \brief Handle lowering of 4-lane 64-bit integer shuffles.
10791 /// This routine is only called when we have AVX2 and thus a reasonable
10792 /// instruction set for v4i64 shuffling..
10793 static SDValue lowerV4I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10794 const X86Subtarget *Subtarget,
10795 SelectionDAG &DAG) {
10797 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
10798 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
10799 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10800 ArrayRef<int> Mask = SVOp->getMask();
10801 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
10802 assert(Subtarget->hasAVX2() && "We can only lower v4i64 with AVX2!");
10804 SmallVector<int, 4> WidenedMask;
10805 if (canWidenShuffleElements(Mask, WidenedMask))
10806 return lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, Subtarget,
10809 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
10813 // Check for being able to broadcast a single element.
10814 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v4i64, DL, V1,
10815 Mask, Subtarget, DAG))
10818 // When the shuffle is mirrored between the 128-bit lanes of the unit, we can
10819 // use lower latency instructions that will operate on both 128-bit lanes.
10820 SmallVector<int, 2> RepeatedMask;
10821 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
10822 if (isSingleInputShuffleMask(Mask)) {
10823 int PSHUFDMask[] = {-1, -1, -1, -1};
10824 for (int i = 0; i < 2; ++i)
10825 if (RepeatedMask[i] >= 0) {
10826 PSHUFDMask[2 * i] = 2 * RepeatedMask[i];
10827 PSHUFDMask[2 * i + 1] = 2 * RepeatedMask[i] + 1;
10829 return DAG.getNode(
10830 ISD::BITCAST, DL, MVT::v4i64,
10831 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
10832 DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1),
10833 getV4X86ShuffleImm8ForMask(PSHUFDMask, DAG)));
10837 // AVX2 provides a direct instruction for permuting a single input across
10839 if (isSingleInputShuffleMask(Mask))
10840 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
10841 getV4X86ShuffleImm8ForMask(Mask, DAG));
10843 // Try to use byte shift instructions.
10844 if (SDValue Shift = lowerVectorShuffleAsByteShift(
10845 DL, MVT::v4i64, V1, V2, Mask, DAG))
10848 // Use dedicated unpack instructions for masks that match their pattern.
10849 if (isShuffleEquivalent(V1, V2, Mask, 0, 4, 2, 6))
10850 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V1, V2);
10851 if (isShuffleEquivalent(V1, V2, Mask, 1, 5, 3, 7))
10852 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V1, V2);
10853 if (isShuffleEquivalent(V1, V2, Mask, 4, 0, 6, 2))
10854 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v4i64, V2, V1);
10855 if (isShuffleEquivalent(V1, V2, Mask, 5, 1, 7, 3))
10856 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v4i64, V2, V1);
10858 // Try to simplify this by merging 128-bit lanes to enable a lane-based
10859 // shuffle. However, if we have AVX2 and either inputs are already in place,
10860 // we will be able to shuffle even across lanes the other input in a single
10861 // instruction so skip this pattern.
10862 if (!(Subtarget->hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) ||
10863 isShuffleMaskInputInPlace(1, Mask))))
10864 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
10865 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
10868 // Otherwise fall back on generic blend lowering.
10869 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2,
10873 /// \brief Handle lowering of 8-lane 32-bit floating point shuffles.
10875 /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
10876 /// isn't available.
10877 static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10878 const X86Subtarget *Subtarget,
10879 SelectionDAG &DAG) {
10881 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
10882 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
10883 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10884 ArrayRef<int> Mask = SVOp->getMask();
10885 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
10887 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
10891 // Check for being able to broadcast a single element.
10892 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8f32, DL, V1,
10893 Mask, Subtarget, DAG))
10896 // If the shuffle mask is repeated in each 128-bit lane, we have many more
10897 // options to efficiently lower the shuffle.
10898 SmallVector<int, 4> RepeatedMask;
10899 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {
10900 assert(RepeatedMask.size() == 4 &&
10901 "Repeated masks must be half the mask width!");
10903 // Use even/odd duplicate instructions for masks that match their pattern.
10904 if (isShuffleEquivalent(V1, V2, Mask, 0, 0, 2, 2, 4, 4, 6, 6))
10905 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
10906 if (isShuffleEquivalent(V1, V2, Mask, 1, 1, 3, 3, 5, 5, 7, 7))
10907 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
10909 if (isSingleInputShuffleMask(Mask))
10910 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
10911 getV4X86ShuffleImm8ForMask(RepeatedMask, DAG));
10913 // Use dedicated unpack instructions for masks that match their pattern.
10914 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13))
10915 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V1, V2);
10916 if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15))
10917 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V1, V2);
10918 if (isShuffleEquivalent(V1, V2, Mask, 8, 0, 9, 1, 12, 4, 13, 5))
10919 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f32, V2, V1);
10920 if (isShuffleEquivalent(V1, V2, Mask, 10, 2, 11, 3, 14, 6, 15, 7))
10921 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f32, V2, V1);
10923 // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
10924 // have already handled any direct blends. We also need to squash the
10925 // repeated mask into a simulated v4f32 mask.
10926 for (int i = 0; i < 4; ++i)
10927 if (RepeatedMask[i] >= 8)
10928 RepeatedMask[i] -= 4;
10929 return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
10932 // If we have a single input shuffle with different shuffle patterns in the
10933 // two 128-bit lanes use the variable mask to VPERMILPS.
10934 if (isSingleInputShuffleMask(Mask)) {
10935 SDValue VPermMask[8];
10936 for (int i = 0; i < 8; ++i)
10937 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
10938 : DAG.getConstant(Mask[i], MVT::i32);
10939 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask))
10940 return DAG.getNode(
10941 X86ISD::VPERMILPV, DL, MVT::v8f32, V1,
10942 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask));
10944 if (Subtarget->hasAVX2())
10945 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32,
10946 DAG.getNode(ISD::BITCAST, DL, MVT::v8f32,
10947 DAG.getNode(ISD::BUILD_VECTOR, DL,
10948 MVT::v8i32, VPermMask)),
10951 // Otherwise, fall back.
10952 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask,
10956 // Try to simplify this by merging 128-bit lanes to enable a lane-based
10958 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
10959 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
10962 // If we have AVX2 then we always want to lower with a blend because at v8 we
10963 // can fully permute the elements.
10964 if (Subtarget->hasAVX2())
10965 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2,
10968 // Otherwise fall back on generic lowering.
10969 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG);
10972 /// \brief Handle lowering of 8-lane 32-bit integer shuffles.
10974 /// This routine is only called when we have AVX2 and thus a reasonable
10975 /// instruction set for v8i32 shuffling..
10976 static SDValue lowerV8I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
10977 const X86Subtarget *Subtarget,
10978 SelectionDAG &DAG) {
10980 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
10981 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
10982 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10983 ArrayRef<int> Mask = SVOp->getMask();
10984 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
10985 assert(Subtarget->hasAVX2() && "We can only lower v8i32 with AVX2!");
10987 // Whenever we can lower this as a zext, that instruction is strictly faster
10988 // than any alternative. It also allows us to fold memory operands into the
10989 // shuffle in many cases.
10990 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2,
10991 Mask, Subtarget, DAG))
10994 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
10998 // Check for being able to broadcast a single element.
10999 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v8i32, DL, V1,
11000 Mask, Subtarget, DAG))
11003 // If the shuffle mask is repeated in each 128-bit lane we can use more
11004 // efficient instructions that mirror the shuffles across the two 128-bit
11006 SmallVector<int, 4> RepeatedMask;
11007 if (is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask)) {
11008 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!");
11009 if (isSingleInputShuffleMask(Mask))
11010 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
11011 getV4X86ShuffleImm8ForMask(RepeatedMask, DAG));
11013 // Use dedicated unpack instructions for masks that match their pattern.
11014 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 1, 9, 4, 12, 5, 13))
11015 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V1, V2);
11016 if (isShuffleEquivalent(V1, V2, Mask, 2, 10, 3, 11, 6, 14, 7, 15))
11017 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V1, V2);
11018 if (isShuffleEquivalent(V1, V2, Mask, 8, 0, 9, 1, 12, 4, 13, 5))
11019 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i32, V2, V1);
11020 if (isShuffleEquivalent(V1, V2, Mask, 10, 2, 11, 3, 14, 6, 15, 7))
11021 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i32, V2, V1);
11024 // Try to use bit shift instructions.
11025 if (SDValue Shift = lowerVectorShuffleAsBitShift(
11026 DL, MVT::v8i32, V1, V2, Mask, DAG))
11029 // Try to use byte shift instructions.
11030 if (SDValue Shift = lowerVectorShuffleAsByteShift(
11031 DL, MVT::v8i32, V1, V2, Mask, DAG))
11034 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
11035 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11038 // If the shuffle patterns aren't repeated but it is a single input, directly
11039 // generate a cross-lane VPERMD instruction.
11040 if (isSingleInputShuffleMask(Mask)) {
11041 SDValue VPermMask[8];
11042 for (int i = 0; i < 8; ++i)
11043 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
11044 : DAG.getConstant(Mask[i], MVT::i32);
11045 return DAG.getNode(
11046 X86ISD::VPERMV, DL, MVT::v8i32,
11047 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1);
11050 // Try to simplify this by merging 128-bit lanes to enable a lane-based
11052 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
11053 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11056 // Otherwise fall back on generic blend lowering.
11057 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2,
11061 /// \brief Handle lowering of 16-lane 16-bit integer shuffles.
11063 /// This routine is only called when we have AVX2 and thus a reasonable
11064 /// instruction set for v16i16 shuffling..
11065 static SDValue lowerV16I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11066 const X86Subtarget *Subtarget,
11067 SelectionDAG &DAG) {
11069 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
11070 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
11071 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11072 ArrayRef<int> Mask = SVOp->getMask();
11073 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
11074 assert(Subtarget->hasAVX2() && "We can only lower v16i16 with AVX2!");
11076 // Whenever we can lower this as a zext, that instruction is strictly faster
11077 // than any alternative. It also allows us to fold memory operands into the
11078 // shuffle in many cases.
11079 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2,
11080 Mask, Subtarget, DAG))
11083 // Check for being able to broadcast a single element.
11084 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v16i16, DL, V1,
11085 Mask, Subtarget, DAG))
11088 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
11092 // Use dedicated unpack instructions for masks that match their pattern.
11093 if (isShuffleEquivalent(V1, V2, Mask,
11094 // First 128-bit lane:
11095 0, 16, 1, 17, 2, 18, 3, 19,
11096 // Second 128-bit lane:
11097 8, 24, 9, 25, 10, 26, 11, 27))
11098 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i16, V1, V2);
11099 if (isShuffleEquivalent(V1, V2, Mask,
11100 // First 128-bit lane:
11101 4, 20, 5, 21, 6, 22, 7, 23,
11102 // Second 128-bit lane:
11103 12, 28, 13, 29, 14, 30, 15, 31))
11104 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i16, V1, V2);
11106 // Try to use bit shift instructions.
11107 if (SDValue Shift = lowerVectorShuffleAsBitShift(
11108 DL, MVT::v16i16, V1, V2, Mask, DAG))
11111 // Try to use byte shift instructions.
11112 if (SDValue Shift = lowerVectorShuffleAsByteShift(
11113 DL, MVT::v16i16, V1, V2, Mask, DAG))
11116 // Try to use byte rotation instructions.
11117 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
11118 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11121 if (isSingleInputShuffleMask(Mask)) {
11122 // There are no generalized cross-lane shuffle operations available on i16
11124 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask))
11125 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2,
11128 SDValue PSHUFBMask[32];
11129 for (int i = 0; i < 16; ++i) {
11130 if (Mask[i] == -1) {
11131 PSHUFBMask[2 * i] = PSHUFBMask[2 * i + 1] = DAG.getUNDEF(MVT::i8);
11135 int M = i < 8 ? Mask[i] : Mask[i] - 8;
11136 assert(M >= 0 && M < 8 && "Invalid single-input mask!");
11137 PSHUFBMask[2 * i] = DAG.getConstant(2 * M, MVT::i8);
11138 PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, MVT::i8);
11140 return DAG.getNode(
11141 ISD::BITCAST, DL, MVT::v16i16,
11143 X86ISD::PSHUFB, DL, MVT::v32i8,
11144 DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1),
11145 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)));
11148 // Try to simplify this by merging 128-bit lanes to enable a lane-based
11150 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
11151 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11154 // Otherwise fall back on generic lowering.
11155 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG);
11158 /// \brief Handle lowering of 32-lane 8-bit integer shuffles.
11160 /// This routine is only called when we have AVX2 and thus a reasonable
11161 /// instruction set for v32i8 shuffling..
11162 static SDValue lowerV32I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11163 const X86Subtarget *Subtarget,
11164 SelectionDAG &DAG) {
11166 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
11167 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
11168 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11169 ArrayRef<int> Mask = SVOp->getMask();
11170 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
11171 assert(Subtarget->hasAVX2() && "We can only lower v32i8 with AVX2!");
11173 // Whenever we can lower this as a zext, that instruction is strictly faster
11174 // than any alternative. It also allows us to fold memory operands into the
11175 // shuffle in many cases.
11176 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2,
11177 Mask, Subtarget, DAG))
11180 // Check for being able to broadcast a single element.
11181 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(MVT::v32i8, DL, V1,
11182 Mask, Subtarget, DAG))
11185 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
11189 // Use dedicated unpack instructions for masks that match their pattern.
11190 // Note that these are repeated 128-bit lane unpacks, not unpacks across all
11192 if (isShuffleEquivalent(
11194 // First 128-bit lane:
11195 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39,
11196 // Second 128-bit lane:
11197 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55))
11198 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v32i8, V1, V2);
11199 if (isShuffleEquivalent(
11201 // First 128-bit lane:
11202 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
11203 // Second 128-bit lane:
11204 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63))
11205 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v32i8, V1, V2);
11207 // Try to use bit shift instructions.
11208 if (SDValue Shift = lowerVectorShuffleAsBitShift(
11209 DL, MVT::v32i8, V1, V2, Mask, DAG))
11212 // Try to use byte shift instructions.
11213 if (SDValue Shift = lowerVectorShuffleAsByteShift(
11214 DL, MVT::v32i8, V1, V2, Mask, DAG))
11217 // Try to use byte rotation instructions.
11218 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
11219 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11222 if (isSingleInputShuffleMask(Mask)) {
11223 // There are no generalized cross-lane shuffle operations available on i8
11225 if (is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask))
11226 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2,
11229 SDValue PSHUFBMask[32];
11230 for (int i = 0; i < 32; ++i)
11233 ? DAG.getUNDEF(MVT::i8)
11234 : DAG.getConstant(Mask[i] < 16 ? Mask[i] : Mask[i] - 16, MVT::i8);
11236 return DAG.getNode(
11237 X86ISD::PSHUFB, DL, MVT::v32i8, V1,
11238 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask));
11241 // Try to simplify this by merging 128-bit lanes to enable a lane-based
11243 if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
11244 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11247 // Otherwise fall back on generic lowering.
11248 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG);
11251 /// \brief High-level routine to lower various 256-bit x86 vector shuffles.
11253 /// This routine either breaks down the specific type of a 256-bit x86 vector
11254 /// shuffle or splits it into two 128-bit shuffles and fuses the results back
11255 /// together based on the available instructions.
11256 static SDValue lower256BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11257 MVT VT, const X86Subtarget *Subtarget,
11258 SelectionDAG &DAG) {
11260 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11261 ArrayRef<int> Mask = SVOp->getMask();
11263 // There is a really nice hard cut-over between AVX1 and AVX2 that means we can
11264 // check for those subtargets here and avoid much of the subtarget querying in
11265 // the per-vector-type lowering routines. With AVX1 we have essentially *zero*
11266 // ability to manipulate a 256-bit vector with integer types. Since we'll use
11267 // floating point types there eventually, just immediately cast everything to
11268 // a float and operate entirely in that domain.
11269 if (VT.isInteger() && !Subtarget->hasAVX2()) {
11270 int ElementBits = VT.getScalarSizeInBits();
11271 if (ElementBits < 32)
11272 // No floating point type available, decompose into 128-bit vectors.
11273 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
11275 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
11276 VT.getVectorNumElements());
11277 V1 = DAG.getNode(ISD::BITCAST, DL, FpVT, V1);
11278 V2 = DAG.getNode(ISD::BITCAST, DL, FpVT, V2);
11279 return DAG.getNode(ISD::BITCAST, DL, VT,
11280 DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
11283 switch (VT.SimpleTy) {
11285 return lowerV4F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11287 return lowerV4I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11289 return lowerV8F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11291 return lowerV8I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11293 return lowerV16I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
11295 return lowerV32I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
11298 llvm_unreachable("Not a valid 256-bit x86 vector type!");
11302 /// \brief Handle lowering of 8-lane 64-bit floating point shuffles.
11303 static SDValue lowerV8F64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11304 const X86Subtarget *Subtarget,
11305 SelectionDAG &DAG) {
11307 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
11308 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
11309 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11310 ArrayRef<int> Mask = SVOp->getMask();
11311 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
11313 // X86 has dedicated unpack instructions that can handle specific blend
11314 // operations: UNPCKH and UNPCKL.
11315 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14))
11316 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8f64, V1, V2);
11317 if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15))
11318 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8f64, V1, V2);
11320 // FIXME: Implement direct support for this type!
11321 return splitAndLowerVectorShuffle(DL, MVT::v8f64, V1, V2, Mask, DAG);
11324 /// \brief Handle lowering of 16-lane 32-bit floating point shuffles.
11325 static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11326 const X86Subtarget *Subtarget,
11327 SelectionDAG &DAG) {
11329 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
11330 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
11331 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11332 ArrayRef<int> Mask = SVOp->getMask();
11333 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
11335 // Use dedicated unpack instructions for masks that match their pattern.
11336 if (isShuffleEquivalent(V1, V2, Mask,
11337 0, 16, 1, 17, 4, 20, 5, 21,
11338 8, 24, 9, 25, 12, 28, 13, 29))
11339 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16f32, V1, V2);
11340 if (isShuffleEquivalent(V1, V2, Mask,
11341 2, 18, 3, 19, 6, 22, 7, 23,
11342 10, 26, 11, 27, 14, 30, 15, 31))
11343 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16f32, V1, V2);
11345 // FIXME: Implement direct support for this type!
11346 return splitAndLowerVectorShuffle(DL, MVT::v16f32, V1, V2, Mask, DAG);
11349 /// \brief Handle lowering of 8-lane 64-bit integer shuffles.
11350 static SDValue lowerV8I64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11351 const X86Subtarget *Subtarget,
11352 SelectionDAG &DAG) {
11354 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
11355 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
11356 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11357 ArrayRef<int> Mask = SVOp->getMask();
11358 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
11360 // X86 has dedicated unpack instructions that can handle specific blend
11361 // operations: UNPCKH and UNPCKL.
11362 if (isShuffleEquivalent(V1, V2, Mask, 0, 8, 2, 10, 4, 12, 6, 14))
11363 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i64, V1, V2);
11364 if (isShuffleEquivalent(V1, V2, Mask, 1, 9, 3, 11, 5, 13, 7, 15))
11365 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v8i64, V1, V2);
11367 // FIXME: Implement direct support for this type!
11368 return splitAndLowerVectorShuffle(DL, MVT::v8i64, V1, V2, Mask, DAG);
11371 /// \brief Handle lowering of 16-lane 32-bit integer shuffles.
11372 static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11373 const X86Subtarget *Subtarget,
11374 SelectionDAG &DAG) {
11376 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
11377 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
11378 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11379 ArrayRef<int> Mask = SVOp->getMask();
11380 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
11382 // Use dedicated unpack instructions for masks that match their pattern.
11383 if (isShuffleEquivalent(V1, V2, Mask,
11384 0, 16, 1, 17, 4, 20, 5, 21,
11385 8, 24, 9, 25, 12, 28, 13, 29))
11386 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i32, V1, V2);
11387 if (isShuffleEquivalent(V1, V2, Mask,
11388 2, 18, 3, 19, 6, 22, 7, 23,
11389 10, 26, 11, 27, 14, 30, 15, 31))
11390 return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i32, V1, V2);
11392 // FIXME: Implement direct support for this type!
11393 return splitAndLowerVectorShuffle(DL, MVT::v16i32, V1, V2, Mask, DAG);
11396 /// \brief Handle lowering of 32-lane 16-bit integer shuffles.
11397 static SDValue lowerV32I16VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11398 const X86Subtarget *Subtarget,
11399 SelectionDAG &DAG) {
11401 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
11402 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
11403 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11404 ArrayRef<int> Mask = SVOp->getMask();
11405 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
11406 assert(Subtarget->hasBWI() && "We can only lower v32i16 with AVX-512-BWI!");
11408 // FIXME: Implement direct support for this type!
11409 return splitAndLowerVectorShuffle(DL, MVT::v32i16, V1, V2, Mask, DAG);
11412 /// \brief Handle lowering of 64-lane 8-bit integer shuffles.
11413 static SDValue lowerV64I8VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11414 const X86Subtarget *Subtarget,
11415 SelectionDAG &DAG) {
11417 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
11418 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
11419 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11420 ArrayRef<int> Mask = SVOp->getMask();
11421 assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!");
11422 assert(Subtarget->hasBWI() && "We can only lower v64i8 with AVX-512-BWI!");
11424 // FIXME: Implement direct support for this type!
11425 return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
11428 /// \brief High-level routine to lower various 512-bit x86 vector shuffles.
11430 /// This routine either breaks down the specific type of a 512-bit x86 vector
11431 /// shuffle or splits it into two 256-bit shuffles and fuses the results back
11432 /// together based on the available instructions.
11433 static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
11434 MVT VT, const X86Subtarget *Subtarget,
11435 SelectionDAG &DAG) {
11437 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11438 ArrayRef<int> Mask = SVOp->getMask();
11439 assert(Subtarget->hasAVX512() &&
11440 "Cannot lower 512-bit vectors w/ basic ISA!");
11442 // Check for being able to broadcast a single element.
11443 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(VT.SimpleTy, DL, V1,
11444 Mask, Subtarget, DAG))
11447 // Dispatch to each element type for lowering. If we don't have supprot for
11448 // specific element type shuffles at 512 bits, immediately split them and
11449 // lower them. Each lowering routine of a given type is allowed to assume that
11450 // the requisite ISA extensions for that element type are available.
11451 switch (VT.SimpleTy) {
11453 return lowerV8F64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11455 return lowerV16F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11457 return lowerV8I64VectorShuffle(Op, V1, V2, Subtarget, DAG);
11459 return lowerV16I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
11461 if (Subtarget->hasBWI())
11462 return lowerV32I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
11465 if (Subtarget->hasBWI())
11466 return lowerV64I8VectorShuffle(Op, V1, V2, Subtarget, DAG);
11470 llvm_unreachable("Not a valid 512-bit x86 vector type!");
11473 // Otherwise fall back on splitting.
11474 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
11477 /// \brief Top-level lowering for x86 vector shuffles.
11479 /// This handles decomposition, canonicalization, and lowering of all x86
11480 /// vector shuffles. Most of the specific lowering strategies are encapsulated
11481 /// above in helper routines. The canonicalization attempts to widen shuffles
11482 /// to involve fewer lanes of wider elements, consolidate symmetric patterns
11483 /// s.t. only one of the two inputs needs to be tested, etc.
11484 static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget *Subtarget,
11485 SelectionDAG &DAG) {
11486 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11487 ArrayRef<int> Mask = SVOp->getMask();
11488 SDValue V1 = Op.getOperand(0);
11489 SDValue V2 = Op.getOperand(1);
11490 MVT VT = Op.getSimpleValueType();
11491 int NumElements = VT.getVectorNumElements();
11494 assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
11496 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
11497 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
11498 if (V1IsUndef && V2IsUndef)
11499 return DAG.getUNDEF(VT);
11501 // When we create a shuffle node we put the UNDEF node to second operand,
11502 // but in some cases the first operand may be transformed to UNDEF.
11503 // In this case we should just commute the node.
11505 return DAG.getCommutedVectorShuffle(*SVOp);
11507 // Check for non-undef masks pointing at an undef vector and make the masks
11508 // undef as well. This makes it easier to match the shuffle based solely on
11512 if (M >= NumElements) {
11513 SmallVector<int, 8> NewMask(Mask.begin(), Mask.end());
11514 for (int &M : NewMask)
11515 if (M >= NumElements)
11517 return DAG.getVectorShuffle(VT, dl, V1, V2, NewMask);
11520 // We actually see shuffles that are entirely re-arrangements of a set of
11521 // zero inputs. This mostly happens while decomposing complex shuffles into
11522 // simple ones. Directly lower these as a buildvector of zeros.
11523 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
11524 if (Zeroable.all())
11525 return getZeroVector(VT, Subtarget, DAG, dl);
11527 // Try to collapse shuffles into using a vector type with fewer elements but
11528 // wider element types. We cap this to not form integers or floating point
11529 // elements wider than 64 bits, but it might be interesting to form i128
11530 // integers to handle flipping the low and high halves of AVX 256-bit vectors.
11531 SmallVector<int, 16> WidenedMask;
11532 if (VT.getScalarSizeInBits() < 64 &&
11533 canWidenShuffleElements(Mask, WidenedMask)) {
11534 MVT NewEltVT = VT.isFloatingPoint()
11535 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
11536 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
11537 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
11538 // Make sure that the new vector type is legal. For example, v2f64 isn't
11540 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
11541 V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1);
11542 V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2);
11543 return DAG.getNode(ISD::BITCAST, dl, VT,
11544 DAG.getVectorShuffle(NewVT, dl, V1, V2, WidenedMask));
11548 int NumV1Elements = 0, NumUndefElements = 0, NumV2Elements = 0;
11549 for (int M : SVOp->getMask())
11551 ++NumUndefElements;
11552 else if (M < NumElements)
11557 // Commute the shuffle as needed such that more elements come from V1 than
11558 // V2. This allows us to match the shuffle pattern strictly on how many
11559 // elements come from V1 without handling the symmetric cases.
11560 if (NumV2Elements > NumV1Elements)
11561 return DAG.getCommutedVectorShuffle(*SVOp);
11563 // When the number of V1 and V2 elements are the same, try to minimize the
11564 // number of uses of V2 in the low half of the vector. When that is tied,
11565 // ensure that the sum of indices for V1 is equal to or lower than the sum
11566 // indices for V2. When those are equal, try to ensure that the number of odd
11567 // indices for V1 is lower than the number of odd indices for V2.
11568 if (NumV1Elements == NumV2Elements) {
11569 int LowV1Elements = 0, LowV2Elements = 0;
11570 for (int M : SVOp->getMask().slice(0, NumElements / 2))
11571 if (M >= NumElements)
11575 if (LowV2Elements > LowV1Elements) {
11576 return DAG.getCommutedVectorShuffle(*SVOp);
11577 } else if (LowV2Elements == LowV1Elements) {
11578 int SumV1Indices = 0, SumV2Indices = 0;
11579 for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i)
11580 if (SVOp->getMask()[i] >= NumElements)
11582 else if (SVOp->getMask()[i] >= 0)
11584 if (SumV2Indices < SumV1Indices) {
11585 return DAG.getCommutedVectorShuffle(*SVOp);
11586 } else if (SumV2Indices == SumV1Indices) {
11587 int NumV1OddIndices = 0, NumV2OddIndices = 0;
11588 for (int i = 0, Size = SVOp->getMask().size(); i < Size; ++i)
11589 if (SVOp->getMask()[i] >= NumElements)
11590 NumV2OddIndices += i % 2;
11591 else if (SVOp->getMask()[i] >= 0)
11592 NumV1OddIndices += i % 2;
11593 if (NumV2OddIndices < NumV1OddIndices)
11594 return DAG.getCommutedVectorShuffle(*SVOp);
11599 // For each vector width, delegate to a specialized lowering routine.
11600 if (VT.getSizeInBits() == 128)
11601 return lower128BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
11603 if (VT.getSizeInBits() == 256)
11604 return lower256BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
11606 // Force AVX-512 vectors to be scalarized for now.
11607 // FIXME: Implement AVX-512 support!
11608 if (VT.getSizeInBits() == 512)
11609 return lower512BitVectorShuffle(Op, V1, V2, VT, Subtarget, DAG);
11611 llvm_unreachable("Unimplemented!");
11615 //===----------------------------------------------------------------------===//
11616 // Legacy vector shuffle lowering
11618 // This code is the legacy code handling vector shuffles until the above
11619 // replaces its functionality and performance.
11620 //===----------------------------------------------------------------------===//
11622 static bool isBlendMask(ArrayRef<int> MaskVals, MVT VT, bool hasSSE41,
11623 bool hasInt256, unsigned *MaskOut = nullptr) {
11624 MVT EltVT = VT.getVectorElementType();
11626 // There is no blend with immediate in AVX-512.
11627 if (VT.is512BitVector())
11630 if (!hasSSE41 || EltVT == MVT::i8)
11632 if (!hasInt256 && VT == MVT::v16i16)
11635 unsigned MaskValue = 0;
11636 unsigned NumElems = VT.getVectorNumElements();
11637 // There are 2 lanes if (NumElems > 8), and 1 lane otherwise.
11638 unsigned NumLanes = (NumElems - 1) / 8 + 1;
11639 unsigned NumElemsInLane = NumElems / NumLanes;
11641 // Blend for v16i16 should be symmetric for both lanes.
11642 for (unsigned i = 0; i < NumElemsInLane; ++i) {
11644 int SndLaneEltIdx = (NumLanes == 2) ? MaskVals[i + NumElemsInLane] : -1;
11645 int EltIdx = MaskVals[i];
11647 if ((EltIdx < 0 || EltIdx == (int)i) &&
11648 (SndLaneEltIdx < 0 || SndLaneEltIdx == (int)(i + NumElemsInLane)))
11651 if (((unsigned)EltIdx == (i + NumElems)) &&
11652 (SndLaneEltIdx < 0 ||
11653 (unsigned)SndLaneEltIdx == i + NumElems + NumElemsInLane))
11654 MaskValue |= (1 << i);
11660 *MaskOut = MaskValue;
11664 // Try to lower a shuffle node into a simple blend instruction.
11665 // This function assumes isBlendMask returns true for this
11666 // SuffleVectorSDNode
11667 static SDValue LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp,
11668 unsigned MaskValue,
11669 const X86Subtarget *Subtarget,
11670 SelectionDAG &DAG) {
11671 MVT VT = SVOp->getSimpleValueType(0);
11672 MVT EltVT = VT.getVectorElementType();
11673 assert(isBlendMask(SVOp->getMask(), VT, Subtarget->hasSSE41(),
11674 Subtarget->hasInt256() && "Trying to lower a "
11675 "VECTOR_SHUFFLE to a Blend but "
11676 "with the wrong mask"));
11677 SDValue V1 = SVOp->getOperand(0);
11678 SDValue V2 = SVOp->getOperand(1);
11680 unsigned NumElems = VT.getVectorNumElements();
11682 // Convert i32 vectors to floating point if it is not AVX2.
11683 // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors.
11685 if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) {
11686 BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()),
11688 V1 = DAG.getNode(ISD::BITCAST, dl, VT, V1);
11689 V2 = DAG.getNode(ISD::BITCAST, dl, VT, V2);
11692 SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, V1, V2,
11693 DAG.getConstant(MaskValue, MVT::i32));
11694 return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
11697 /// In vector type \p VT, return true if the element at index \p InputIdx
11698 /// falls on a different 128-bit lane than \p OutputIdx.
11699 static bool ShuffleCrosses128bitLane(MVT VT, unsigned InputIdx,
11700 unsigned OutputIdx) {
11701 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
11702 return InputIdx * EltSize / 128 != OutputIdx * EltSize / 128;
11705 /// Generate a PSHUFB if possible. Selects elements from \p V1 according to
11706 /// \p MaskVals. MaskVals[OutputIdx] = InputIdx specifies that we want to
11707 /// shuffle the element at InputIdx in V1 to OutputIdx in the result. If \p
11708 /// MaskVals refers to elements outside of \p V1 or is undef (-1), insert a
11710 static SDValue getPSHUFB(ArrayRef<int> MaskVals, SDValue V1, SDLoc &dl,
11711 SelectionDAG &DAG) {
11712 MVT VT = V1.getSimpleValueType();
11713 assert(VT.is128BitVector() || VT.is256BitVector());
11715 MVT EltVT = VT.getVectorElementType();
11716 unsigned EltSizeInBytes = EltVT.getSizeInBits() / 8;
11717 unsigned NumElts = VT.getVectorNumElements();
11719 SmallVector<SDValue, 32> PshufbMask;
11720 for (unsigned OutputIdx = 0; OutputIdx < NumElts; ++OutputIdx) {
11721 int InputIdx = MaskVals[OutputIdx];
11722 unsigned InputByteIdx;
11724 if (InputIdx < 0 || NumElts <= (unsigned)InputIdx)
11725 InputByteIdx = 0x80;
11727 // Cross lane is not allowed.
11728 if (ShuffleCrosses128bitLane(VT, InputIdx, OutputIdx))
11730 InputByteIdx = InputIdx * EltSizeInBytes;
11731 // Index is an byte offset within the 128-bit lane.
11732 InputByteIdx &= 0xf;
11735 for (unsigned j = 0; j < EltSizeInBytes; ++j) {
11736 PshufbMask.push_back(DAG.getConstant(InputByteIdx, MVT::i8));
11737 if (InputByteIdx != 0x80)
11742 MVT ShufVT = MVT::getVectorVT(MVT::i8, PshufbMask.size());
11744 V1 = DAG.getNode(ISD::BITCAST, dl, ShufVT, V1);
11745 return DAG.getNode(X86ISD::PSHUFB, dl, ShufVT, V1,
11746 DAG.getNode(ISD::BUILD_VECTOR, dl, ShufVT, PshufbMask));
11749 // v8i16 shuffles - Prefer shuffles in the following order:
11750 // 1. [all] pshuflw, pshufhw, optional move
11751 // 2. [ssse3] 1 x pshufb
11752 // 3. [ssse3] 2 x pshufb + 1 x por
11753 // 4. [all] mov + pshuflw + pshufhw + N x (pextrw + pinsrw)
11755 LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget,
11756 SelectionDAG &DAG) {
11757 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11758 SDValue V1 = SVOp->getOperand(0);
11759 SDValue V2 = SVOp->getOperand(1);
11761 SmallVector<int, 8> MaskVals;
11763 // Determine if more than 1 of the words in each of the low and high quadwords
11764 // of the result come from the same quadword of one of the two inputs. Undef
11765 // mask values count as coming from any quadword, for better codegen.
11767 // Lo/HiQuad[i] = j indicates how many words from the ith quad of the input
11768 // feeds this quad. For i, 0 and 1 refer to V1, 2 and 3 refer to V2.
11769 unsigned LoQuad[] = { 0, 0, 0, 0 };
11770 unsigned HiQuad[] = { 0, 0, 0, 0 };
11771 // Indices of quads used.
11772 std::bitset<4> InputQuads;
11773 for (unsigned i = 0; i < 8; ++i) {
11774 unsigned *Quad = i < 4 ? LoQuad : HiQuad;
11775 int EltIdx = SVOp->getMaskElt(i);
11776 MaskVals.push_back(EltIdx);
11784 ++Quad[EltIdx / 4];
11785 InputQuads.set(EltIdx / 4);
11788 int BestLoQuad = -1;
11789 unsigned MaxQuad = 1;
11790 for (unsigned i = 0; i < 4; ++i) {
11791 if (LoQuad[i] > MaxQuad) {
11793 MaxQuad = LoQuad[i];
11797 int BestHiQuad = -1;
11799 for (unsigned i = 0; i < 4; ++i) {
11800 if (HiQuad[i] > MaxQuad) {
11802 MaxQuad = HiQuad[i];
11806 // For SSSE3, If all 8 words of the result come from only 1 quadword of each
11807 // of the two input vectors, shuffle them into one input vector so only a
11808 // single pshufb instruction is necessary. If there are more than 2 input
11809 // quads, disable the next transformation since it does not help SSSE3.
11810 bool V1Used = InputQuads[0] || InputQuads[1];
11811 bool V2Used = InputQuads[2] || InputQuads[3];
11812 if (Subtarget->hasSSSE3()) {
11813 if (InputQuads.count() == 2 && V1Used && V2Used) {
11814 BestLoQuad = InputQuads[0] ? 0 : 1;
11815 BestHiQuad = InputQuads[2] ? 2 : 3;
11817 if (InputQuads.count() > 2) {
11823 // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update
11824 // the shuffle mask. If a quad is scored as -1, that means that it contains
11825 // words from all 4 input quadwords.
11827 if (BestLoQuad >= 0 || BestHiQuad >= 0) {
11829 BestLoQuad < 0 ? 0 : BestLoQuad,
11830 BestHiQuad < 0 ? 1 : BestHiQuad
11832 NewV = DAG.getVectorShuffle(MVT::v2i64, dl,
11833 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1),
11834 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2), &MaskV[0]);
11835 NewV = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, NewV);
11837 // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the
11838 // source words for the shuffle, to aid later transformations.
11839 bool AllWordsInNewV = true;
11840 bool InOrder[2] = { true, true };
11841 for (unsigned i = 0; i != 8; ++i) {
11842 int idx = MaskVals[i];
11844 InOrder[i/4] = false;
11845 if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad)
11847 AllWordsInNewV = false;
11851 bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV;
11852 if (AllWordsInNewV) {
11853 for (int i = 0; i != 8; ++i) {
11854 int idx = MaskVals[i];
11857 idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4;
11858 if ((idx != i) && idx < 4)
11860 if ((idx != i) && idx > 3)
11869 // If we've eliminated the use of V2, and the new mask is a pshuflw or
11870 // pshufhw, that's as cheap as it gets. Return the new shuffle.
11871 if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) {
11872 unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW;
11873 unsigned TargetMask = 0;
11874 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV,
11875 DAG.getUNDEF(MVT::v8i16), &MaskVals[0]);
11876 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
11877 TargetMask = pshufhw ? getShufflePSHUFHWImmediate(SVOp):
11878 getShufflePSHUFLWImmediate(SVOp);
11879 V1 = NewV.getOperand(0);
11880 return getTargetShuffleNode(Opc, dl, MVT::v8i16, V1, TargetMask, DAG);
11884 // Promote splats to a larger type which usually leads to more efficient code.
11885 // FIXME: Is this true if pshufb is available?
11886 if (SVOp->isSplat())
11887 return PromoteSplat(SVOp, DAG);
11889 // If we have SSSE3, and all words of the result are from 1 input vector,
11890 // case 2 is generated, otherwise case 3 is generated. If no SSSE3
11891 // is present, fall back to case 4.
11892 if (Subtarget->hasSSSE3()) {
11893 SmallVector<SDValue,16> pshufbMask;
11895 // If we have elements from both input vectors, set the high bit of the
11896 // shuffle mask element to zero out elements that come from V2 in the V1
11897 // mask, and elements that come from V1 in the V2 mask, so that the two
11898 // results can be OR'd together.
11899 bool TwoInputs = V1Used && V2Used;
11900 V1 = getPSHUFB(MaskVals, V1, dl, DAG);
11902 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
11904 // Calculate the shuffle mask for the second input, shuffle it, and
11905 // OR it with the first shuffled input.
11906 CommuteVectorShuffleMask(MaskVals, 8);
11907 V2 = getPSHUFB(MaskVals, V2, dl, DAG);
11908 V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
11909 return DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
11912 // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order,
11913 // and update MaskVals with new element order.
11914 std::bitset<8> InOrder;
11915 if (BestLoQuad >= 0) {
11916 int MaskV[] = { -1, -1, -1, -1, 4, 5, 6, 7 };
11917 for (int i = 0; i != 4; ++i) {
11918 int idx = MaskVals[i];
11921 } else if ((idx / 4) == BestLoQuad) {
11922 MaskV[i] = idx & 3;
11926 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
11929 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) {
11930 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
11931 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16,
11932 NewV.getOperand(0),
11933 getShufflePSHUFLWImmediate(SVOp), DAG);
11937 // If BestHi >= 0, generate a pshufhw to put the high elements in order,
11938 // and update MaskVals with the new element order.
11939 if (BestHiQuad >= 0) {
11940 int MaskV[] = { 0, 1, 2, 3, -1, -1, -1, -1 };
11941 for (unsigned i = 4; i != 8; ++i) {
11942 int idx = MaskVals[i];
11945 } else if ((idx / 4) == BestHiQuad) {
11946 MaskV[i] = (idx & 3) + 4;
11950 NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
11953 if (NewV.getOpcode() == ISD::VECTOR_SHUFFLE && Subtarget->hasSSE2()) {
11954 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(NewV.getNode());
11955 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16,
11956 NewV.getOperand(0),
11957 getShufflePSHUFHWImmediate(SVOp), DAG);
11961 // In case BestHi & BestLo were both -1, which means each quadword has a word
11962 // from each of the four input quadwords, calculate the InOrder bitvector now
11963 // before falling through to the insert/extract cleanup.
11964 if (BestLoQuad == -1 && BestHiQuad == -1) {
11966 for (int i = 0; i != 8; ++i)
11967 if (MaskVals[i] < 0 || MaskVals[i] == i)
11971 // The other elements are put in the right place using pextrw and pinsrw.
11972 for (unsigned i = 0; i != 8; ++i) {
11975 int EltIdx = MaskVals[i];
11978 SDValue ExtOp = (EltIdx < 8) ?
11979 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1,
11980 DAG.getIntPtrConstant(EltIdx)) :
11981 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2,
11982 DAG.getIntPtrConstant(EltIdx - 8));
11983 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp,
11984 DAG.getIntPtrConstant(i));
11989 /// \brief v16i16 shuffles
11991 /// FIXME: We only support generation of a single pshufb currently. We can
11992 /// generalize the other applicable cases from LowerVECTOR_SHUFFLEv8i16 as
11993 /// well (e.g 2 x pshufb + 1 x por).
11995 LowerVECTOR_SHUFFLEv16i16(SDValue Op, SelectionDAG &DAG) {
11996 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
11997 SDValue V1 = SVOp->getOperand(0);
11998 SDValue V2 = SVOp->getOperand(1);
12001 if (V2.getOpcode() != ISD::UNDEF)
12004 SmallVector<int, 16> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
12005 return getPSHUFB(MaskVals, V1, dl, DAG);
12008 // v16i8 shuffles - Prefer shuffles in the following order:
12009 // 1. [ssse3] 1 x pshufb
12010 // 2. [ssse3] 2 x pshufb + 1 x por
12011 // 3. [all] v8i16 shuffle + N x pextrw + rotate + pinsrw
12012 static SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
12013 const X86Subtarget* Subtarget,
12014 SelectionDAG &DAG) {
12015 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12016 SDValue V1 = SVOp->getOperand(0);
12017 SDValue V2 = SVOp->getOperand(1);
12019 ArrayRef<int> MaskVals = SVOp->getMask();
12021 // Promote splats to a larger type which usually leads to more efficient code.
12022 // FIXME: Is this true if pshufb is available?
12023 if (SVOp->isSplat())
12024 return PromoteSplat(SVOp, DAG);
12026 // If we have SSSE3, case 1 is generated when all result bytes come from
12027 // one of the inputs. Otherwise, case 2 is generated. If no SSSE3 is
12028 // present, fall back to case 3.
12030 // If SSSE3, use 1 pshufb instruction per vector with elements in the result.
12031 if (Subtarget->hasSSSE3()) {
12032 SmallVector<SDValue,16> pshufbMask;
12034 // If all result elements are from one input vector, then only translate
12035 // undef mask values to 0x80 (zero out result) in the pshufb mask.
12037 // Otherwise, we have elements from both input vectors, and must zero out
12038 // elements that come from V2 in the first mask, and V1 in the second mask
12039 // so that we can OR them together.
12040 for (unsigned i = 0; i != 16; ++i) {
12041 int EltIdx = MaskVals[i];
12042 if (EltIdx < 0 || EltIdx >= 16)
12044 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
12046 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
12047 DAG.getNode(ISD::BUILD_VECTOR, dl,
12048 MVT::v16i8, pshufbMask));
12050 // As PSHUFB will zero elements with negative indices, it's safe to ignore
12051 // the 2nd operand if it's undefined or zero.
12052 if (V2.getOpcode() == ISD::UNDEF ||
12053 ISD::isBuildVectorAllZeros(V2.getNode()))
12056 // Calculate the shuffle mask for the second input, shuffle it, and
12057 // OR it with the first shuffled input.
12058 pshufbMask.clear();
12059 for (unsigned i = 0; i != 16; ++i) {
12060 int EltIdx = MaskVals[i];
12061 EltIdx = (EltIdx < 16) ? 0x80 : EltIdx - 16;
12062 pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
12064 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
12065 DAG.getNode(ISD::BUILD_VECTOR, dl,
12066 MVT::v16i8, pshufbMask));
12067 return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
12070 // No SSSE3 - Calculate in place words and then fix all out of place words
12071 // With 0-16 extracts & inserts. Worst case is 16 bytes out of order from
12072 // the 16 different words that comprise the two doublequadword input vectors.
12073 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1);
12074 V2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2);
12076 for (int i = 0; i != 8; ++i) {
12077 int Elt0 = MaskVals[i*2];
12078 int Elt1 = MaskVals[i*2+1];
12080 // This word of the result is all undef, skip it.
12081 if (Elt0 < 0 && Elt1 < 0)
12084 // This word of the result is already in the correct place, skip it.
12085 if ((Elt0 == i*2) && (Elt1 == i*2+1))
12088 SDValue Elt0Src = Elt0 < 16 ? V1 : V2;
12089 SDValue Elt1Src = Elt1 < 16 ? V1 : V2;
12092 // If Elt0 and Elt1 are defined, are consecutive, and can be load
12093 // using a single extract together, load it and store it.
12094 if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) {
12095 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
12096 DAG.getIntPtrConstant(Elt1 / 2));
12097 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
12098 DAG.getIntPtrConstant(i));
12102 // If Elt1 is defined, extract it from the appropriate source. If the
12103 // source byte is not also odd, shift the extracted word left 8 bits
12104 // otherwise clear the bottom 8 bits if we need to do an or.
12106 InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
12107 DAG.getIntPtrConstant(Elt1 / 2));
12108 if ((Elt1 & 1) == 0)
12109 InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt,
12111 TLI.getShiftAmountTy(InsElt.getValueType())));
12112 else if (Elt0 >= 0)
12113 InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt,
12114 DAG.getConstant(0xFF00, MVT::i16));
12116 // If Elt0 is defined, extract it from the appropriate source. If the
12117 // source byte is not also even, shift the extracted word right 8 bits. If
12118 // Elt1 was also defined, OR the extracted values together before
12119 // inserting them in the result.
12121 SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
12122 Elt0Src, DAG.getIntPtrConstant(Elt0 / 2));
12123 if ((Elt0 & 1) != 0)
12124 InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0,
12126 TLI.getShiftAmountTy(InsElt0.getValueType())));
12127 else if (Elt1 >= 0)
12128 InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0,
12129 DAG.getConstant(0x00FF, MVT::i16));
12130 InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0)
12133 NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
12134 DAG.getIntPtrConstant(i));
12136 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV);
12139 // v32i8 shuffles - Translate to VPSHUFB if possible.
12141 SDValue LowerVECTOR_SHUFFLEv32i8(ShuffleVectorSDNode *SVOp,
12142 const X86Subtarget *Subtarget,
12143 SelectionDAG &DAG) {
12144 MVT VT = SVOp->getSimpleValueType(0);
12145 SDValue V1 = SVOp->getOperand(0);
12146 SDValue V2 = SVOp->getOperand(1);
12148 SmallVector<int, 32> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
12150 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
12151 bool V1IsAllZero = ISD::isBuildVectorAllZeros(V1.getNode());
12152 bool V2IsAllZero = ISD::isBuildVectorAllZeros(V2.getNode());
12154 // VPSHUFB may be generated if
12155 // (1) one of input vector is undefined or zeroinitializer.
12156 // The mask value 0x80 puts 0 in the corresponding slot of the vector.
12157 // And (2) the mask indexes don't cross the 128-bit lane.
12158 if (VT != MVT::v32i8 || !Subtarget->hasInt256() ||
12159 (!V2IsUndef && !V2IsAllZero && !V1IsAllZero))
12162 if (V1IsAllZero && !V2IsAllZero) {
12163 CommuteVectorShuffleMask(MaskVals, 32);
12166 return getPSHUFB(MaskVals, V1, dl, DAG);
12169 /// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide
12170 /// ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be
12171 /// done when every pair / quad of shuffle mask elements point to elements in
12172 /// the right sequence. e.g.
12173 /// vector_shuffle X, Y, <2, 3, | 10, 11, | 0, 1, | 14, 15>
12175 SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp,
12176 SelectionDAG &DAG) {
12177 MVT VT = SVOp->getSimpleValueType(0);
12179 unsigned NumElems = VT.getVectorNumElements();
12182 switch (VT.SimpleTy) {
12183 default: llvm_unreachable("Unexpected!");
12186 return SDValue(SVOp, 0);
12187 case MVT::v4f32: NewVT = MVT::v2f64; Scale = 2; break;
12188 case MVT::v4i32: NewVT = MVT::v2i64; Scale = 2; break;
12189 case MVT::v8i16: NewVT = MVT::v4i32; Scale = 2; break;
12190 case MVT::v16i8: NewVT = MVT::v4i32; Scale = 4; break;
12191 case MVT::v16i16: NewVT = MVT::v8i32; Scale = 2; break;
12192 case MVT::v32i8: NewVT = MVT::v8i32; Scale = 4; break;
12195 SmallVector<int, 8> MaskVec;
12196 for (unsigned i = 0; i != NumElems; i += Scale) {
12198 for (unsigned j = 0; j != Scale; ++j) {
12199 int EltIdx = SVOp->getMaskElt(i+j);
12203 StartIdx = (EltIdx / Scale);
12204 if (EltIdx != (int)(StartIdx*Scale + j))
12207 MaskVec.push_back(StartIdx);
12210 SDValue V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(0));
12211 SDValue V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, SVOp->getOperand(1));
12212 return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]);
12215 /// getVZextMovL - Return a zero-extending vector move low node.
12217 static SDValue getVZextMovL(MVT VT, MVT OpVT,
12218 SDValue SrcOp, SelectionDAG &DAG,
12219 const X86Subtarget *Subtarget, SDLoc dl) {
12220 if (VT == MVT::v2f64 || VT == MVT::v4f32) {
12221 LoadSDNode *LD = nullptr;
12222 if (!isScalarLoadToVector(SrcOp.getNode(), &LD))
12223 LD = dyn_cast<LoadSDNode>(SrcOp);
12225 // movssrr and movsdrr do not clear top bits. Try to use movd, movq
12227 MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32;
12228 if ((ExtVT != MVT::i64 || Subtarget->is64Bit()) &&
12229 SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR &&
12230 SrcOp.getOperand(0).getOpcode() == ISD::BITCAST &&
12231 SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) {
12233 OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32;
12234 return DAG.getNode(ISD::BITCAST, dl, VT,
12235 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
12236 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
12238 SrcOp.getOperand(0)
12244 return DAG.getNode(ISD::BITCAST, dl, VT,
12245 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
12246 DAG.getNode(ISD::BITCAST, dl,
12250 /// LowerVECTOR_SHUFFLE_256 - Handle all 256-bit wide vectors shuffles
12251 /// which could not be matched by any known target speficic shuffle
12253 LowerVECTOR_SHUFFLE_256(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
12255 SDValue NewOp = Compact8x32ShuffleNode(SVOp, DAG);
12256 if (NewOp.getNode())
12259 MVT VT = SVOp->getSimpleValueType(0);
12261 unsigned NumElems = VT.getVectorNumElements();
12262 unsigned NumLaneElems = NumElems / 2;
12265 MVT EltVT = VT.getVectorElementType();
12266 MVT NVT = MVT::getVectorVT(EltVT, NumLaneElems);
12269 SmallVector<int, 16> Mask;
12270 for (unsigned l = 0; l < 2; ++l) {
12271 // Build a shuffle mask for the output, discovering on the fly which
12272 // input vectors to use as shuffle operands (recorded in InputUsed).
12273 // If building a suitable shuffle vector proves too hard, then bail
12274 // out with UseBuildVector set.
12275 bool UseBuildVector = false;
12276 int InputUsed[2] = { -1, -1 }; // Not yet discovered.
12277 unsigned LaneStart = l * NumLaneElems;
12278 for (unsigned i = 0; i != NumLaneElems; ++i) {
12279 // The mask element. This indexes into the input.
12280 int Idx = SVOp->getMaskElt(i+LaneStart);
12282 // the mask element does not index into any input vector.
12283 Mask.push_back(-1);
12287 // The input vector this mask element indexes into.
12288 int Input = Idx / NumLaneElems;
12290 // Turn the index into an offset from the start of the input vector.
12291 Idx -= Input * NumLaneElems;
12293 // Find or create a shuffle vector operand to hold this input.
12295 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
12296 if (InputUsed[OpNo] == Input)
12297 // This input vector is already an operand.
12299 if (InputUsed[OpNo] < 0) {
12300 // Create a new operand for this input vector.
12301 InputUsed[OpNo] = Input;
12306 if (OpNo >= array_lengthof(InputUsed)) {
12307 // More than two input vectors used! Give up on trying to create a
12308 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
12309 UseBuildVector = true;
12313 // Add the mask index for the new shuffle vector.
12314 Mask.push_back(Idx + OpNo * NumLaneElems);
12317 if (UseBuildVector) {
12318 SmallVector<SDValue, 16> SVOps;
12319 for (unsigned i = 0; i != NumLaneElems; ++i) {
12320 // The mask element. This indexes into the input.
12321 int Idx = SVOp->getMaskElt(i+LaneStart);
12323 SVOps.push_back(DAG.getUNDEF(EltVT));
12327 // The input vector this mask element indexes into.
12328 int Input = Idx / NumElems;
12330 // Turn the index into an offset from the start of the input vector.
12331 Idx -= Input * NumElems;
12333 // Extract the vector element by hand.
12334 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
12335 SVOp->getOperand(Input),
12336 DAG.getIntPtrConstant(Idx)));
12339 // Construct the output using a BUILD_VECTOR.
12340 Output[l] = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, SVOps);
12341 } else if (InputUsed[0] < 0) {
12342 // No input vectors were used! The result is undefined.
12343 Output[l] = DAG.getUNDEF(NVT);
12345 SDValue Op0 = Extract128BitVector(SVOp->getOperand(InputUsed[0] / 2),
12346 (InputUsed[0] % 2) * NumLaneElems,
12348 // If only one input was used, use an undefined vector for the other.
12349 SDValue Op1 = (InputUsed[1] < 0) ? DAG.getUNDEF(NVT) :
12350 Extract128BitVector(SVOp->getOperand(InputUsed[1] / 2),
12351 (InputUsed[1] % 2) * NumLaneElems, DAG, dl);
12352 // At least one input vector was used. Create a new shuffle vector.
12353 Output[l] = DAG.getVectorShuffle(NVT, dl, Op0, Op1, &Mask[0]);
12359 // Concatenate the result back
12360 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Output[0], Output[1]);
12363 /// LowerVECTOR_SHUFFLE_128v4 - Handle all 128-bit wide vectors with
12364 /// 4 elements, and match them with several different shuffle types.
12366 LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
12367 SDValue V1 = SVOp->getOperand(0);
12368 SDValue V2 = SVOp->getOperand(1);
12370 MVT VT = SVOp->getSimpleValueType(0);
12372 assert(VT.is128BitVector() && "Unsupported vector size");
12374 std::pair<int, int> Locs[4];
12375 int Mask1[] = { -1, -1, -1, -1 };
12376 SmallVector<int, 8> PermMask(SVOp->getMask().begin(), SVOp->getMask().end());
12378 unsigned NumHi = 0;
12379 unsigned NumLo = 0;
12380 for (unsigned i = 0; i != 4; ++i) {
12381 int Idx = PermMask[i];
12383 Locs[i] = std::make_pair(-1, -1);
12385 assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!");
12387 Locs[i] = std::make_pair(0, NumLo);
12388 Mask1[NumLo] = Idx;
12391 Locs[i] = std::make_pair(1, NumHi);
12393 Mask1[2+NumHi] = Idx;
12399 if (NumLo <= 2 && NumHi <= 2) {
12400 // If no more than two elements come from either vector. This can be
12401 // implemented with two shuffles. First shuffle gather the elements.
12402 // The second shuffle, which takes the first shuffle as both of its
12403 // vector operands, put the elements into the right order.
12404 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
12406 int Mask2[] = { -1, -1, -1, -1 };
12408 for (unsigned i = 0; i != 4; ++i)
12409 if (Locs[i].first != -1) {
12410 unsigned Idx = (i < 2) ? 0 : 4;
12411 Idx += Locs[i].first * 2 + Locs[i].second;
12415 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]);
12418 if (NumLo == 3 || NumHi == 3) {
12419 // Otherwise, we must have three elements from one vector, call it X, and
12420 // one element from the other, call it Y. First, use a shufps to build an
12421 // intermediate vector with the one element from Y and the element from X
12422 // that will be in the same half in the final destination (the indexes don't
12423 // matter). Then, use a shufps to build the final vector, taking the half
12424 // containing the element from Y from the intermediate, and the other half
12427 // Normalize it so the 3 elements come from V1.
12428 CommuteVectorShuffleMask(PermMask, 4);
12432 // Find the element from V2.
12434 for (HiIndex = 0; HiIndex < 3; ++HiIndex) {
12435 int Val = PermMask[HiIndex];
12442 Mask1[0] = PermMask[HiIndex];
12444 Mask1[2] = PermMask[HiIndex^1];
12446 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
12448 if (HiIndex >= 2) {
12449 Mask1[0] = PermMask[0];
12450 Mask1[1] = PermMask[1];
12451 Mask1[2] = HiIndex & 1 ? 6 : 4;
12452 Mask1[3] = HiIndex & 1 ? 4 : 6;
12453 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
12456 Mask1[0] = HiIndex & 1 ? 2 : 0;
12457 Mask1[1] = HiIndex & 1 ? 0 : 2;
12458 Mask1[2] = PermMask[2];
12459 Mask1[3] = PermMask[3];
12464 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]);
12467 // Break it into (shuffle shuffle_hi, shuffle_lo).
12468 int LoMask[] = { -1, -1, -1, -1 };
12469 int HiMask[] = { -1, -1, -1, -1 };
12471 int *MaskPtr = LoMask;
12472 unsigned MaskIdx = 0;
12473 unsigned LoIdx = 0;
12474 unsigned HiIdx = 2;
12475 for (unsigned i = 0; i != 4; ++i) {
12482 int Idx = PermMask[i];
12484 Locs[i] = std::make_pair(-1, -1);
12485 } else if (Idx < 4) {
12486 Locs[i] = std::make_pair(MaskIdx, LoIdx);
12487 MaskPtr[LoIdx] = Idx;
12490 Locs[i] = std::make_pair(MaskIdx, HiIdx);
12491 MaskPtr[HiIdx] = Idx;
12496 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]);
12497 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]);
12498 int MaskOps[] = { -1, -1, -1, -1 };
12499 for (unsigned i = 0; i != 4; ++i)
12500 if (Locs[i].first != -1)
12501 MaskOps[i] = Locs[i].first * 4 + Locs[i].second;
12502 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]);
12505 static bool MayFoldVectorLoad(SDValue V) {
12506 while (V.hasOneUse() && V.getOpcode() == ISD::BITCAST)
12507 V = V.getOperand(0);
12509 if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR)
12510 V = V.getOperand(0);
12511 if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR &&
12512 V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF)
12513 // BUILD_VECTOR (load), undef
12514 V = V.getOperand(0);
12516 return MayFoldLoad(V);
12520 SDValue getMOVDDup(SDValue &Op, SDLoc &dl, SDValue V1, SelectionDAG &DAG) {
12521 MVT VT = Op.getSimpleValueType();
12523 // Canonicalize to v2f64.
12524 V1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1);
12525 return DAG.getNode(ISD::BITCAST, dl, VT,
12526 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64,
12531 SDValue getMOVLowToHigh(SDValue &Op, SDLoc &dl, SelectionDAG &DAG,
12533 SDValue V1 = Op.getOperand(0);
12534 SDValue V2 = Op.getOperand(1);
12535 MVT VT = Op.getSimpleValueType();
12537 assert(VT != MVT::v2i64 && "unsupported shuffle type");
12539 if (HasSSE2 && VT == MVT::v2f64)
12540 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
12542 // v4f32 or v4i32: canonicalize to v4f32 (which is legal for SSE1)
12543 return DAG.getNode(ISD::BITCAST, dl, VT,
12544 getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32,
12545 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V1),
12546 DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, V2), DAG));
12550 SDValue getMOVHighToLow(SDValue &Op, SDLoc &dl, SelectionDAG &DAG) {
12551 SDValue V1 = Op.getOperand(0);
12552 SDValue V2 = Op.getOperand(1);
12553 MVT VT = Op.getSimpleValueType();
12555 assert((VT == MVT::v4i32 || VT == MVT::v4f32) &&
12556 "unsupported shuffle type");
12558 if (V2.getOpcode() == ISD::UNDEF)
12562 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
12566 SDValue getMOVLP(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
12567 SDValue V1 = Op.getOperand(0);
12568 SDValue V2 = Op.getOperand(1);
12569 MVT VT = Op.getSimpleValueType();
12570 unsigned NumElems = VT.getVectorNumElements();
12572 // Use MOVLPS and MOVLPD in case V1 or V2 are loads. During isel, the second
12573 // operand of these instructions is only memory, so check if there's a
12574 // potencial load folding here, otherwise use SHUFPS or MOVSD to match the
12576 bool CanFoldLoad = false;
12578 // Trivial case, when V2 comes from a load.
12579 if (MayFoldVectorLoad(V2))
12580 CanFoldLoad = true;
12582 // When V1 is a load, it can be folded later into a store in isel, example:
12583 // (store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), addr:$src1)
12585 // (MOVLPSmr addr:$src1, VR128:$src2)
12586 // So, recognize this potential and also use MOVLPS or MOVLPD
12587 else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op))
12588 CanFoldLoad = true;
12590 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12592 if (HasSSE2 && NumElems == 2)
12593 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
12596 // If we don't care about the second element, proceed to use movss.
12597 if (SVOp->getMaskElt(1) != -1)
12598 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG);
12601 // movl and movlp will both match v2i64, but v2i64 is never matched by
12602 // movl earlier because we make it strict to avoid messing with the movlp load
12603 // folding logic (see the code above getMOVLP call). Match it here then,
12604 // this is horrible, but will stay like this until we move all shuffle
12605 // matching to x86 specific nodes. Note that for the 1st condition all
12606 // types are matched with movsd.
12608 // FIXME: isMOVLMask should be checked and matched before getMOVLP,
12609 // as to remove this logic from here, as much as possible
12610 if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT))
12611 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
12612 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
12615 assert(VT != MVT::v4i32 && "unsupported shuffle type");
12617 // Invert the operand order and use SHUFPS to match it.
12618 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1,
12619 getShuffleSHUFImmediate(SVOp), DAG);
12622 static SDValue NarrowVectorLoadToElement(LoadSDNode *Load, unsigned Index,
12623 SelectionDAG &DAG) {
12625 MVT VT = Load->getSimpleValueType(0);
12626 MVT EVT = VT.getVectorElementType();
12627 SDValue Addr = Load->getOperand(1);
12628 SDValue NewAddr = DAG.getNode(
12629 ISD::ADD, dl, Addr.getSimpleValueType(), Addr,
12630 DAG.getConstant(Index * EVT.getStoreSize(), Addr.getSimpleValueType()));
12633 DAG.getLoad(EVT, dl, Load->getChain(), NewAddr,
12634 DAG.getMachineFunction().getMachineMemOperand(
12635 Load->getMemOperand(), 0, EVT.getStoreSize()));
12639 // It is only safe to call this function if isINSERTPSMask is true for
12640 // this shufflevector mask.
12641 static SDValue getINSERTPS(ShuffleVectorSDNode *SVOp, SDLoc &dl,
12642 SelectionDAG &DAG) {
12643 // Generate an insertps instruction when inserting an f32 from memory onto a
12644 // v4f32 or when copying a member from one v4f32 to another.
12645 // We also use it for transferring i32 from one register to another,
12646 // since it simply copies the same bits.
12647 // If we're transferring an i32 from memory to a specific element in a
12648 // register, we output a generic DAG that will match the PINSRD
12650 MVT VT = SVOp->getSimpleValueType(0);
12651 MVT EVT = VT.getVectorElementType();
12652 SDValue V1 = SVOp->getOperand(0);
12653 SDValue V2 = SVOp->getOperand(1);
12654 auto Mask = SVOp->getMask();
12655 assert((VT == MVT::v4f32 || VT == MVT::v4i32) &&
12656 "unsupported vector type for insertps/pinsrd");
12658 auto FromV1Predicate = [](const int &i) { return i < 4 && i > -1; };
12659 auto FromV2Predicate = [](const int &i) { return i >= 4; };
12660 int FromV1 = std::count_if(Mask.begin(), Mask.end(), FromV1Predicate);
12664 unsigned DestIndex;
12668 DestIndex = std::find_if(Mask.begin(), Mask.end(), FromV1Predicate) -
12671 // If we have 1 element from each vector, we have to check if we're
12672 // changing V1's element's place. If so, we're done. Otherwise, we
12673 // should assume we're changing V2's element's place and behave
12675 int FromV2 = std::count_if(Mask.begin(), Mask.end(), FromV2Predicate);
12676 assert(DestIndex <= INT32_MAX && "truncated destination index");
12677 if (FromV1 == FromV2 &&
12678 static_cast<int>(DestIndex) == Mask[DestIndex] % 4) {
12682 std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin();
12685 assert(std::count_if(Mask.begin(), Mask.end(), FromV2Predicate) == 1 &&
12686 "More than one element from V1 and from V2, or no elements from one "
12687 "of the vectors. This case should not have returned true from "
12692 std::find_if(Mask.begin(), Mask.end(), FromV2Predicate) - Mask.begin();
12695 // Get an index into the source vector in the range [0,4) (the mask is
12696 // in the range [0,8) because it can address V1 and V2)
12697 unsigned SrcIndex = Mask[DestIndex] % 4;
12698 if (MayFoldLoad(From)) {
12699 // Trivial case, when From comes from a load and is only used by the
12700 // shuffle. Make it use insertps from the vector that we need from that
12703 NarrowVectorLoadToElement(cast<LoadSDNode>(From), SrcIndex, DAG);
12704 if (!NewLoad.getNode())
12707 if (EVT == MVT::f32) {
12708 // Create this as a scalar to vector to match the instruction pattern.
12709 SDValue LoadScalarToVector =
12710 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, NewLoad);
12711 SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4);
12712 return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, LoadScalarToVector,
12714 } else { // EVT == MVT::i32
12715 // If we're getting an i32 from memory, use an INSERT_VECTOR_ELT
12716 // instruction, to match the PINSRD instruction, which loads an i32 to a
12717 // certain vector element.
12718 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, To, NewLoad,
12719 DAG.getConstant(DestIndex, MVT::i32));
12723 // Vector-element-to-vector
12724 SDValue InsertpsMask = DAG.getIntPtrConstant(DestIndex << 4 | SrcIndex << 6);
12725 return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, From, InsertpsMask);
12728 // Reduce a vector shuffle to zext.
12729 static SDValue LowerVectorIntExtend(SDValue Op, const X86Subtarget *Subtarget,
12730 SelectionDAG &DAG) {
12731 // PMOVZX is only available from SSE41.
12732 if (!Subtarget->hasSSE41())
12735 MVT VT = Op.getSimpleValueType();
12737 // Only AVX2 support 256-bit vector integer extending.
12738 if (!Subtarget->hasInt256() && VT.is256BitVector())
12741 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12743 SDValue V1 = Op.getOperand(0);
12744 SDValue V2 = Op.getOperand(1);
12745 unsigned NumElems = VT.getVectorNumElements();
12747 // Extending is an unary operation and the element type of the source vector
12748 // won't be equal to or larger than i64.
12749 if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() ||
12750 VT.getVectorElementType() == MVT::i64)
12753 // Find the expansion ratio, e.g. expanding from i8 to i32 has a ratio of 4.
12754 unsigned Shift = 1; // Start from 2, i.e. 1 << 1.
12755 while ((1U << Shift) < NumElems) {
12756 if (SVOp->getMaskElt(1U << Shift) == 1)
12759 // The maximal ratio is 8, i.e. from i8 to i64.
12764 // Check the shuffle mask.
12765 unsigned Mask = (1U << Shift) - 1;
12766 for (unsigned i = 0; i != NumElems; ++i) {
12767 int EltIdx = SVOp->getMaskElt(i);
12768 if ((i & Mask) != 0 && EltIdx != -1)
12770 if ((i & Mask) == 0 && (unsigned)EltIdx != (i >> Shift))
12774 unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift;
12775 MVT NeVT = MVT::getIntegerVT(NBits);
12776 MVT NVT = MVT::getVectorVT(NeVT, NumElems >> Shift);
12778 if (!DAG.getTargetLoweringInfo().isTypeLegal(NVT))
12781 return DAG.getNode(ISD::BITCAST, DL, VT,
12782 DAG.getNode(X86ISD::VZEXT, DL, NVT, V1));
12785 static SDValue NormalizeVectorShuffle(SDValue Op, const X86Subtarget *Subtarget,
12786 SelectionDAG &DAG) {
12787 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12788 MVT VT = Op.getSimpleValueType();
12790 SDValue V1 = Op.getOperand(0);
12791 SDValue V2 = Op.getOperand(1);
12793 if (isZeroShuffle(SVOp))
12794 return getZeroVector(VT, Subtarget, DAG, dl);
12796 // Handle splat operations
12797 if (SVOp->isSplat()) {
12798 // Use vbroadcast whenever the splat comes from a foldable load
12799 SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG);
12800 if (Broadcast.getNode())
12804 // Check integer expanding shuffles.
12805 SDValue NewOp = LowerVectorIntExtend(Op, Subtarget, DAG);
12806 if (NewOp.getNode())
12809 // If the shuffle can be profitably rewritten as a narrower shuffle, then
12811 if (VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v16i16 ||
12812 VT == MVT::v32i8) {
12813 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
12814 if (NewOp.getNode())
12815 return DAG.getNode(ISD::BITCAST, dl, VT, NewOp);
12816 } else if (VT.is128BitVector() && Subtarget->hasSSE2()) {
12817 // FIXME: Figure out a cleaner way to do this.
12818 if (ISD::isBuildVectorAllZeros(V2.getNode())) {
12819 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
12820 if (NewOp.getNode()) {
12821 MVT NewVT = NewOp.getSimpleValueType();
12822 if (isCommutedMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(),
12823 NewVT, true, false))
12824 return getVZextMovL(VT, NewVT, NewOp.getOperand(0), DAG, Subtarget,
12827 } else if (ISD::isBuildVectorAllZeros(V1.getNode())) {
12828 SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG);
12829 if (NewOp.getNode()) {
12830 MVT NewVT = NewOp.getSimpleValueType();
12831 if (isMOVLMask(cast<ShuffleVectorSDNode>(NewOp)->getMask(), NewVT))
12832 return getVZextMovL(VT, NewVT, NewOp.getOperand(1), DAG, Subtarget,
12841 X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
12842 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12843 SDValue V1 = Op.getOperand(0);
12844 SDValue V2 = Op.getOperand(1);
12845 MVT VT = Op.getSimpleValueType();
12847 unsigned NumElems = VT.getVectorNumElements();
12848 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
12849 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
12850 bool V1IsSplat = false;
12851 bool V2IsSplat = false;
12852 bool HasSSE2 = Subtarget->hasSSE2();
12853 bool HasFp256 = Subtarget->hasFp256();
12854 bool HasInt256 = Subtarget->hasInt256();
12855 MachineFunction &MF = DAG.getMachineFunction();
12857 MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
12859 // Check if we should use the experimental vector shuffle lowering. If so,
12860 // delegate completely to that code path.
12861 if (ExperimentalVectorShuffleLowering)
12862 return lowerVectorShuffle(Op, Subtarget, DAG);
12864 assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
12866 if (V1IsUndef && V2IsUndef)
12867 return DAG.getUNDEF(VT);
12869 // When we create a shuffle node we put the UNDEF node to second operand,
12870 // but in some cases the first operand may be transformed to UNDEF.
12871 // In this case we should just commute the node.
12873 return DAG.getCommutedVectorShuffle(*SVOp);
12875 // Vector shuffle lowering takes 3 steps:
12877 // 1) Normalize the input vectors. Here splats, zeroed vectors, profitable
12878 // narrowing and commutation of operands should be handled.
12879 // 2) Matching of shuffles with known shuffle masks to x86 target specific
12881 // 3) Rewriting of unmatched masks into new generic shuffle operations,
12882 // so the shuffle can be broken into other shuffles and the legalizer can
12883 // try the lowering again.
12885 // The general idea is that no vector_shuffle operation should be left to
12886 // be matched during isel, all of them must be converted to a target specific
12889 // Normalize the input vectors. Here splats, zeroed vectors, profitable
12890 // narrowing and commutation of operands should be handled. The actual code
12891 // doesn't include all of those, work in progress...
12892 SDValue NewOp = NormalizeVectorShuffle(Op, Subtarget, DAG);
12893 if (NewOp.getNode())
12896 SmallVector<int, 8> M(SVOp->getMask().begin(), SVOp->getMask().end());
12898 // NOTE: isPSHUFDMask can also match both masks below (unpckl_undef and
12899 // unpckh_undef). Only use pshufd if speed is more important than size.
12900 if (OptForSize && isUNPCKL_v_undef_Mask(M, VT, HasInt256))
12901 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
12902 if (OptForSize && isUNPCKH_v_undef_Mask(M, VT, HasInt256))
12903 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
12905 if (isMOVDDUPMask(M, VT) && Subtarget->hasSSE3() &&
12906 V2IsUndef && MayFoldVectorLoad(V1))
12907 return getMOVDDup(Op, dl, V1, DAG);
12909 if (isMOVHLPS_v_undef_Mask(M, VT))
12910 return getMOVHighToLow(Op, dl, DAG);
12912 // Use to match splats
12913 if (HasSSE2 && isUNPCKHMask(M, VT, HasInt256) && V2IsUndef &&
12914 (VT == MVT::v2f64 || VT == MVT::v2i64))
12915 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
12917 if (isPSHUFDMask(M, VT)) {
12918 // The actual implementation will match the mask in the if above and then
12919 // during isel it can match several different instructions, not only pshufd
12920 // as its name says, sad but true, emulate the behavior for now...
12921 if (isMOVDDUPMask(M, VT) && ((VT == MVT::v4f32 || VT == MVT::v2i64)))
12922 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG);
12924 unsigned TargetMask = getShuffleSHUFImmediate(SVOp);
12926 if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32))
12927 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG);
12929 if (HasFp256 && (VT == MVT::v4f32 || VT == MVT::v2f64))
12930 return getTargetShuffleNode(X86ISD::VPERMILPI, dl, VT, V1, TargetMask,
12933 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V1,
12937 if (isPALIGNRMask(M, VT, Subtarget))
12938 return getTargetShuffleNode(X86ISD::PALIGNR, dl, VT, V1, V2,
12939 getShufflePALIGNRImmediate(SVOp),
12942 if (isVALIGNMask(M, VT, Subtarget))
12943 return getTargetShuffleNode(X86ISD::VALIGN, dl, VT, V1, V2,
12944 getShuffleVALIGNImmediate(SVOp),
12947 // Check if this can be converted into a logical shift.
12948 bool isLeft = false;
12949 unsigned ShAmt = 0;
12951 bool isShift = HasSSE2 && isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt);
12952 if (isShift && ShVal.hasOneUse()) {
12953 // If the shifted value has multiple uses, it may be cheaper to use
12954 // v_set0 + movlhps or movhlps, etc.
12955 MVT EltVT = VT.getVectorElementType();
12956 ShAmt *= EltVT.getSizeInBits();
12957 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
12960 if (isMOVLMask(M, VT)) {
12961 if (ISD::isBuildVectorAllZeros(V1.getNode()))
12962 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl);
12963 if (!isMOVLPMask(M, VT)) {
12964 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64))
12965 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
12967 if (VT == MVT::v4i32 || VT == MVT::v4f32)
12968 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
12972 // FIXME: fold these into legal mask.
12973 if (isMOVLHPSMask(M, VT) && !isUNPCKLMask(M, VT, HasInt256))
12974 return getMOVLowToHigh(Op, dl, DAG, HasSSE2);
12976 if (isMOVHLPSMask(M, VT))
12977 return getMOVHighToLow(Op, dl, DAG);
12979 if (V2IsUndef && isMOVSHDUPMask(M, VT, Subtarget))
12980 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG);
12982 if (V2IsUndef && isMOVSLDUPMask(M, VT, Subtarget))
12983 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG);
12985 if (isMOVLPMask(M, VT))
12986 return getMOVLP(Op, dl, DAG, HasSSE2);
12988 if (ShouldXformToMOVHLPS(M, VT) ||
12989 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), M, VT))
12990 return DAG.getCommutedVectorShuffle(*SVOp);
12993 // No better options. Use a vshldq / vsrldq.
12994 MVT EltVT = VT.getVectorElementType();
12995 ShAmt *= EltVT.getSizeInBits();
12996 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
12999 bool Commuted = false;
13000 // FIXME: This should also accept a bitcast of a splat? Be careful, not
13001 // 1,1,1,1 -> v8i16 though.
13002 BitVector UndefElements;
13003 if (auto *BVOp = dyn_cast<BuildVectorSDNode>(V1.getNode()))
13004 if (BVOp->getConstantSplatNode(&UndefElements) && UndefElements.none())
13006 if (auto *BVOp = dyn_cast<BuildVectorSDNode>(V2.getNode()))
13007 if (BVOp->getConstantSplatNode(&UndefElements) && UndefElements.none())
13010 // Canonicalize the splat or undef, if present, to be on the RHS.
13011 if (!V2IsUndef && V1IsSplat && !V2IsSplat) {
13012 CommuteVectorShuffleMask(M, NumElems);
13014 std::swap(V1IsSplat, V2IsSplat);
13018 if (isCommutedMOVLMask(M, VT, V2IsSplat, V2IsUndef)) {
13019 // Shuffling low element of v1 into undef, just return v1.
13022 // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which
13023 // the instruction selector will not match, so get a canonical MOVL with
13024 // swapped operands to undo the commute.
13025 return getMOVL(DAG, dl, VT, V2, V1);
13028 if (isUNPCKLMask(M, VT, HasInt256))
13029 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
13031 if (isUNPCKHMask(M, VT, HasInt256))
13032 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
13035 // Normalize mask so all entries that point to V2 points to its first
13036 // element then try to match unpck{h|l} again. If match, return a
13037 // new vector_shuffle with the corrected mask.p
13038 SmallVector<int, 8> NewMask(M.begin(), M.end());
13039 NormalizeMask(NewMask, NumElems);
13040 if (isUNPCKLMask(NewMask, VT, HasInt256, true))
13041 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
13042 if (isUNPCKHMask(NewMask, VT, HasInt256, true))
13043 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
13047 // Commute is back and try unpck* again.
13048 // FIXME: this seems wrong.
13049 CommuteVectorShuffleMask(M, NumElems);
13051 std::swap(V1IsSplat, V2IsSplat);
13053 if (isUNPCKLMask(M, VT, HasInt256))
13054 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
13056 if (isUNPCKHMask(M, VT, HasInt256))
13057 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
13060 // Normalize the node to match x86 shuffle ops if needed
13061 if (!V2IsUndef && (isSHUFPMask(M, VT, /* Commuted */ true)))
13062 return DAG.getCommutedVectorShuffle(*SVOp);
13064 // The checks below are all present in isShuffleMaskLegal, but they are
13065 // inlined here right now to enable us to directly emit target specific
13066 // nodes, and remove one by one until they don't return Op anymore.
13068 if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) &&
13069 SVOp->getSplatIndex() == 0 && V2IsUndef) {
13070 if (VT == MVT::v2f64 || VT == MVT::v2i64)
13071 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
13074 if (isPSHUFHWMask(M, VT, HasInt256))
13075 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1,
13076 getShufflePSHUFHWImmediate(SVOp),
13079 if (isPSHUFLWMask(M, VT, HasInt256))
13080 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1,
13081 getShufflePSHUFLWImmediate(SVOp),
13084 unsigned MaskValue;
13085 if (isBlendMask(M, VT, Subtarget->hasSSE41(), HasInt256, &MaskValue))
13086 return LowerVECTOR_SHUFFLEtoBlend(SVOp, MaskValue, Subtarget, DAG);
13088 if (isSHUFPMask(M, VT))
13089 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V2,
13090 getShuffleSHUFImmediate(SVOp), DAG);
13092 if (isUNPCKL_v_undef_Mask(M, VT, HasInt256))
13093 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
13094 if (isUNPCKH_v_undef_Mask(M, VT, HasInt256))
13095 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
13097 //===--------------------------------------------------------------------===//
13098 // Generate target specific nodes for 128 or 256-bit shuffles only
13099 // supported in the AVX instruction set.
13102 // Handle VMOVDDUPY permutations
13103 if (V2IsUndef && isMOVDDUPYMask(M, VT, HasFp256))
13104 return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG);
13106 // Handle VPERMILPS/D* permutations
13107 if (isVPERMILPMask(M, VT)) {
13108 if ((HasInt256 && VT == MVT::v8i32) || VT == MVT::v16i32)
13109 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1,
13110 getShuffleSHUFImmediate(SVOp), DAG);
13111 return getTargetShuffleNode(X86ISD::VPERMILPI, dl, VT, V1,
13112 getShuffleSHUFImmediate(SVOp), DAG);
13116 if (VT.is512BitVector() && isINSERT64x4Mask(M, VT, &Idx))
13117 return Insert256BitVector(V1, Extract256BitVector(V2, 0, DAG, dl),
13118 Idx*(NumElems/2), DAG, dl);
13120 // Handle VPERM2F128/VPERM2I128 permutations
13121 if (isVPERM2X128Mask(M, VT, HasFp256))
13122 return getTargetShuffleNode(X86ISD::VPERM2X128, dl, VT, V1,
13123 V2, getShuffleVPERM2X128Immediate(SVOp), DAG);
13125 if (Subtarget->hasSSE41() && isINSERTPSMask(M, VT))
13126 return getINSERTPS(SVOp, dl, DAG);
13129 if (V2IsUndef && HasInt256 && isPermImmMask(M, VT, Imm8))
13130 return getTargetShuffleNode(X86ISD::VPERMI, dl, VT, V1, Imm8, DAG);
13132 if ((V2IsUndef && HasInt256 && VT.is256BitVector() && NumElems == 8) ||
13133 VT.is512BitVector()) {
13134 MVT MaskEltVT = MVT::getIntegerVT(VT.getVectorElementType().getSizeInBits());
13135 MVT MaskVectorVT = MVT::getVectorVT(MaskEltVT, NumElems);
13136 SmallVector<SDValue, 16> permclMask;
13137 for (unsigned i = 0; i != NumElems; ++i) {
13138 permclMask.push_back(DAG.getConstant((M[i]>=0) ? M[i] : 0, MaskEltVT));
13141 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, MaskVectorVT, permclMask);
13143 // Bitcast is for VPERMPS since mask is v8i32 but node takes v8f32
13144 return DAG.getNode(X86ISD::VPERMV, dl, VT,
13145 DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1);
13146 return DAG.getNode(X86ISD::VPERMV3, dl, VT, V1,
13147 DAG.getNode(ISD::BITCAST, dl, VT, Mask), V2);
13150 //===--------------------------------------------------------------------===//
13151 // Since no target specific shuffle was selected for this generic one,
13152 // lower it into other known shuffles. FIXME: this isn't true yet, but
13153 // this is the plan.
13156 // Handle v8i16 specifically since SSE can do byte extraction and insertion.
13157 if (VT == MVT::v8i16) {
13158 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, Subtarget, DAG);
13159 if (NewOp.getNode())
13163 if (VT == MVT::v16i16 && HasInt256) {
13164 SDValue NewOp = LowerVECTOR_SHUFFLEv16i16(Op, DAG);
13165 if (NewOp.getNode())
13169 if (VT == MVT::v16i8) {
13170 SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, Subtarget, DAG);
13171 if (NewOp.getNode())
13175 if (VT == MVT::v32i8) {
13176 SDValue NewOp = LowerVECTOR_SHUFFLEv32i8(SVOp, Subtarget, DAG);
13177 if (NewOp.getNode())
13181 // Handle all 128-bit wide vectors with 4 elements, and match them with
13182 // several different shuffle types.
13183 if (NumElems == 4 && VT.is128BitVector())
13184 return LowerVECTOR_SHUFFLE_128v4(SVOp, DAG);
13186 // Handle general 256-bit shuffles
13187 if (VT.is256BitVector())
13188 return LowerVECTOR_SHUFFLE_256(SVOp, DAG);
13193 // This function assumes its argument is a BUILD_VECTOR of constants or
13194 // undef SDNodes. i.e: ISD::isBuildVectorOfConstantSDNodes(BuildVector) is
13196 static bool BUILD_VECTORtoBlendMask(BuildVectorSDNode *BuildVector,
13197 unsigned &MaskValue) {
13199 unsigned NumElems = BuildVector->getNumOperands();
13200 // There are 2 lanes if (NumElems > 8), and 1 lane otherwise.
13201 unsigned NumLanes = (NumElems - 1) / 8 + 1;
13202 unsigned NumElemsInLane = NumElems / NumLanes;
13204 // Blend for v16i16 should be symetric for the both lanes.
13205 for (unsigned i = 0; i < NumElemsInLane; ++i) {
13206 SDValue EltCond = BuildVector->getOperand(i);
13207 SDValue SndLaneEltCond =
13208 (NumLanes == 2) ? BuildVector->getOperand(i + NumElemsInLane) : EltCond;
13210 int Lane1Cond = -1, Lane2Cond = -1;
13211 if (isa<ConstantSDNode>(EltCond))
13212 Lane1Cond = !isZero(EltCond);
13213 if (isa<ConstantSDNode>(SndLaneEltCond))
13214 Lane2Cond = !isZero(SndLaneEltCond);
13216 if (Lane1Cond == Lane2Cond || Lane2Cond < 0)
13217 // Lane1Cond != 0, means we want the first argument.
13218 // Lane1Cond == 0, means we want the second argument.
13219 // The encoding of this argument is 0 for the first argument, 1
13220 // for the second. Therefore, invert the condition.
13221 MaskValue |= !Lane1Cond << i;
13222 else if (Lane1Cond < 0)
13223 MaskValue |= !Lane2Cond << i;
13230 /// \brief Try to lower a VSELECT instruction to an immediate-controlled blend
13232 static SDValue lowerVSELECTtoBLENDI(SDValue Op, const X86Subtarget *Subtarget,
13233 SelectionDAG &DAG) {
13234 SDValue Cond = Op.getOperand(0);
13235 SDValue LHS = Op.getOperand(1);
13236 SDValue RHS = Op.getOperand(2);
13238 MVT VT = Op.getSimpleValueType();
13239 MVT EltVT = VT.getVectorElementType();
13240 unsigned NumElems = VT.getVectorNumElements();
13242 // There is no blend with immediate in AVX-512.
13243 if (VT.is512BitVector())
13246 if (!Subtarget->hasSSE41() || EltVT == MVT::i8)
13248 if (!Subtarget->hasInt256() && VT == MVT::v16i16)
13251 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
13254 // Check the mask for BLEND and build the value.
13255 unsigned MaskValue = 0;
13256 if (!BUILD_VECTORtoBlendMask(cast<BuildVectorSDNode>(Cond), MaskValue))
13259 // Convert i32 vectors to floating point if it is not AVX2.
13260 // AVX2 introduced VPBLENDD instruction for 128 and 256-bit vectors.
13262 if (EltVT == MVT::i64 || (EltVT == MVT::i32 && !Subtarget->hasInt256())) {
13263 BlendVT = MVT::getVectorVT(MVT::getFloatingPointVT(EltVT.getSizeInBits()),
13265 LHS = DAG.getNode(ISD::BITCAST, dl, VT, LHS);
13266 RHS = DAG.getNode(ISD::BITCAST, dl, VT, RHS);
13269 SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, LHS, RHS,
13270 DAG.getConstant(MaskValue, MVT::i32));
13271 return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
13274 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
13275 // A vselect where all conditions and data are constants can be optimized into
13276 // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
13277 if (ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(0).getNode()) &&
13278 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(1).getNode()) &&
13279 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(2).getNode()))
13282 SDValue BlendOp = lowerVSELECTtoBLENDI(Op, Subtarget, DAG);
13283 if (BlendOp.getNode())
13286 // Some types for vselect were previously set to Expand, not Legal or
13287 // Custom. Return an empty SDValue so we fall-through to Expand, after
13288 // the Custom lowering phase.
13289 MVT VT = Op.getSimpleValueType();
13290 switch (VT.SimpleTy) {
13295 if (Subtarget->hasBWI() && Subtarget->hasVLX())
13300 // We couldn't create a "Blend with immediate" node.
13301 // This node should still be legal, but we'll have to emit a blendv*
13306 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
13307 MVT VT = Op.getSimpleValueType();
13310 if (!Op.getOperand(0).getSimpleValueType().is128BitVector())
13313 if (VT.getSizeInBits() == 8) {
13314 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
13315 Op.getOperand(0), Op.getOperand(1));
13316 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
13317 DAG.getValueType(VT));
13318 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
13321 if (VT.getSizeInBits() == 16) {
13322 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13323 // If Idx is 0, it's cheaper to do a move instead of a pextrw.
13325 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
13326 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
13327 DAG.getNode(ISD::BITCAST, dl,
13330 Op.getOperand(1)));
13331 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
13332 Op.getOperand(0), Op.getOperand(1));
13333 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
13334 DAG.getValueType(VT));
13335 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
13338 if (VT == MVT::f32) {
13339 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
13340 // the result back to FR32 register. It's only worth matching if the
13341 // result has a single use which is a store or a bitcast to i32. And in
13342 // the case of a store, it's not worth it if the index is a constant 0,
13343 // because a MOVSSmr can be used instead, which is smaller and faster.
13344 if (!Op.hasOneUse())
13346 SDNode *User = *Op.getNode()->use_begin();
13347 if ((User->getOpcode() != ISD::STORE ||
13348 (isa<ConstantSDNode>(Op.getOperand(1)) &&
13349 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) &&
13350 (User->getOpcode() != ISD::BITCAST ||
13351 User->getValueType(0) != MVT::i32))
13353 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
13354 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32,
13357 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract);
13360 if (VT == MVT::i32 || VT == MVT::i64) {
13361 // ExtractPS/pextrq works with constant index.
13362 if (isa<ConstantSDNode>(Op.getOperand(1)))
13368 /// Extract one bit from mask vector, like v16i1 or v8i1.
13369 /// AVX-512 feature.
13371 X86TargetLowering::ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG) const {
13372 SDValue Vec = Op.getOperand(0);
13374 MVT VecVT = Vec.getSimpleValueType();
13375 SDValue Idx = Op.getOperand(1);
13376 MVT EltVT = Op.getSimpleValueType();
13378 assert((EltVT == MVT::i1) && "Unexpected operands in ExtractBitFromMaskVector");
13379 assert((VecVT.getVectorNumElements() <= 16 || Subtarget->hasBWI()) &&
13380 "Unexpected vector type in ExtractBitFromMaskVector");
13382 // variable index can't be handled in mask registers,
13383 // extend vector to VR512
13384 if (!isa<ConstantSDNode>(Idx)) {
13385 MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32);
13386 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Vec);
13387 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
13388 ExtVT.getVectorElementType(), Ext, Idx);
13389 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
13392 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13393 const TargetRegisterClass* rc = getRegClassFor(VecVT);
13394 if (!Subtarget->hasDQI() && (VecVT.getVectorNumElements() <= 8))
13395 rc = getRegClassFor(MVT::v16i1);
13396 unsigned MaxSift = rc->getSize()*8 - 1;
13397 Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec,
13398 DAG.getConstant(MaxSift - IdxVal, MVT::i8));
13399 Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec,
13400 DAG.getConstant(MaxSift, MVT::i8));
13401 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec,
13402 DAG.getIntPtrConstant(0));
13406 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
13407 SelectionDAG &DAG) const {
13409 SDValue Vec = Op.getOperand(0);
13410 MVT VecVT = Vec.getSimpleValueType();
13411 SDValue Idx = Op.getOperand(1);
13413 if (Op.getSimpleValueType() == MVT::i1)
13414 return ExtractBitFromMaskVector(Op, DAG);
13416 if (!isa<ConstantSDNode>(Idx)) {
13417 if (VecVT.is512BitVector() ||
13418 (VecVT.is256BitVector() && Subtarget->hasInt256() &&
13419 VecVT.getVectorElementType().getSizeInBits() == 32)) {
13422 MVT::getIntegerVT(VecVT.getVectorElementType().getSizeInBits());
13423 MVT MaskVT = MVT::getVectorVT(MaskEltVT, VecVT.getSizeInBits() /
13424 MaskEltVT.getSizeInBits());
13426 Idx = DAG.getZExtOrTrunc(Idx, dl, MaskEltVT);
13427 SDValue Mask = DAG.getNode(X86ISD::VINSERT, dl, MaskVT,
13428 getZeroVector(MaskVT, Subtarget, DAG, dl),
13429 Idx, DAG.getConstant(0, getPointerTy()));
13430 SDValue Perm = DAG.getNode(X86ISD::VPERMV, dl, VecVT, Mask, Vec);
13431 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(),
13432 Perm, DAG.getConstant(0, getPointerTy()));
13437 // If this is a 256-bit vector result, first extract the 128-bit vector and
13438 // then extract the element from the 128-bit vector.
13439 if (VecVT.is256BitVector() || VecVT.is512BitVector()) {
13441 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13442 // Get the 128-bit vector.
13443 Vec = Extract128BitVector(Vec, IdxVal, DAG, dl);
13444 MVT EltVT = VecVT.getVectorElementType();
13446 unsigned ElemsPerChunk = 128 / EltVT.getSizeInBits();
13448 //if (IdxVal >= NumElems/2)
13449 // IdxVal -= NumElems/2;
13450 IdxVal -= (IdxVal/ElemsPerChunk)*ElemsPerChunk;
13451 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
13452 DAG.getConstant(IdxVal, MVT::i32));
13455 assert(VecVT.is128BitVector() && "Unexpected vector length");
13457 if (Subtarget->hasSSE41()) {
13458 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
13463 MVT VT = Op.getSimpleValueType();
13464 // TODO: handle v16i8.
13465 if (VT.getSizeInBits() == 16) {
13466 SDValue Vec = Op.getOperand(0);
13467 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13469 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
13470 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
13471 DAG.getNode(ISD::BITCAST, dl,
13473 Op.getOperand(1)));
13474 // Transform it so it match pextrw which produces a 32-bit result.
13475 MVT EltVT = MVT::i32;
13476 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT,
13477 Op.getOperand(0), Op.getOperand(1));
13478 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract,
13479 DAG.getValueType(VT));
13480 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
13483 if (VT.getSizeInBits() == 32) {
13484 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13488 // SHUFPS the element to the lowest double word, then movss.
13489 int Mask[4] = { static_cast<int>(Idx), -1, -1, -1 };
13490 MVT VVT = Op.getOperand(0).getSimpleValueType();
13491 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
13492 DAG.getUNDEF(VVT), Mask);
13493 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
13494 DAG.getIntPtrConstant(0));
13497 if (VT.getSizeInBits() == 64) {
13498 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
13499 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
13500 // to match extract_elt for f64.
13501 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
13505 // UNPCKHPD the element to the lowest double word, then movsd.
13506 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
13507 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
13508 int Mask[2] = { 1, -1 };
13509 MVT VVT = Op.getOperand(0).getSimpleValueType();
13510 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
13511 DAG.getUNDEF(VVT), Mask);
13512 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
13513 DAG.getIntPtrConstant(0));
13519 /// Insert one bit to mask vector, like v16i1 or v8i1.
13520 /// AVX-512 feature.
13522 X86TargetLowering::InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG) const {
13524 SDValue Vec = Op.getOperand(0);
13525 SDValue Elt = Op.getOperand(1);
13526 SDValue Idx = Op.getOperand(2);
13527 MVT VecVT = Vec.getSimpleValueType();
13529 if (!isa<ConstantSDNode>(Idx)) {
13530 // Non constant index. Extend source and destination,
13531 // insert element and then truncate the result.
13532 MVT ExtVecVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32);
13533 MVT ExtEltVT = (VecVT == MVT::v8i1 ? MVT::i64 : MVT::i32);
13534 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
13535 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVecVT, Vec),
13536 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtEltVT, Elt), Idx);
13537 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
13540 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13541 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Elt);
13542 if (Vec.getOpcode() == ISD::UNDEF)
13543 return DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
13544 DAG.getConstant(IdxVal, MVT::i8));
13545 const TargetRegisterClass* rc = getRegClassFor(VecVT);
13546 unsigned MaxSift = rc->getSize()*8 - 1;
13547 EltInVec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
13548 DAG.getConstant(MaxSift, MVT::i8));
13549 EltInVec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, EltInVec,
13550 DAG.getConstant(MaxSift - IdxVal, MVT::i8));
13551 return DAG.getNode(ISD::OR, dl, VecVT, Vec, EltInVec);
13554 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
13555 SelectionDAG &DAG) const {
13556 MVT VT = Op.getSimpleValueType();
13557 MVT EltVT = VT.getVectorElementType();
13559 if (EltVT == MVT::i1)
13560 return InsertBitToMaskVector(Op, DAG);
13563 SDValue N0 = Op.getOperand(0);
13564 SDValue N1 = Op.getOperand(1);
13565 SDValue N2 = Op.getOperand(2);
13566 if (!isa<ConstantSDNode>(N2))
13568 auto *N2C = cast<ConstantSDNode>(N2);
13569 unsigned IdxVal = N2C->getZExtValue();
13571 // If the vector is wider than 128 bits, extract the 128-bit subvector, insert
13572 // into that, and then insert the subvector back into the result.
13573 if (VT.is256BitVector() || VT.is512BitVector()) {
13574 // Get the desired 128-bit vector half.
13575 SDValue V = Extract128BitVector(N0, IdxVal, DAG, dl);
13577 // Insert the element into the desired half.
13578 unsigned NumEltsIn128 = 128 / EltVT.getSizeInBits();
13579 unsigned IdxIn128 = IdxVal - (IdxVal / NumEltsIn128) * NumEltsIn128;
13581 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
13582 DAG.getConstant(IdxIn128, MVT::i32));
13584 // Insert the changed part back to the 256-bit vector
13585 return Insert128BitVector(N0, V, IdxVal, DAG, dl);
13587 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!");
13589 if (Subtarget->hasSSE41()) {
13590 if (EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) {
13592 if (VT == MVT::v8i16) {
13593 Opc = X86ISD::PINSRW;
13595 assert(VT == MVT::v16i8);
13596 Opc = X86ISD::PINSRB;
13599 // Transform it so it match pinsr{b,w} which expects a GR32 as its second
13601 if (N1.getValueType() != MVT::i32)
13602 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
13603 if (N2.getValueType() != MVT::i32)
13604 N2 = DAG.getIntPtrConstant(IdxVal);
13605 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
13608 if (EltVT == MVT::f32) {
13609 // Bits [7:6] of the constant are the source select. This will always be
13610 // zero here. The DAG Combiner may combine an extract_elt index into
13612 // bits. For example (insert (extract, 3), 2) could be matched by
13614 // the '3' into bits [7:6] of X86ISD::INSERTPS.
13615 // Bits [5:4] of the constant are the destination select. This is the
13616 // value of the incoming immediate.
13617 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
13618 // combine either bitwise AND or insert of float 0.0 to set these bits.
13619 N2 = DAG.getIntPtrConstant(IdxVal << 4);
13620 // Create this as a scalar to vector..
13621 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
13622 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
13625 if (EltVT == MVT::i32 || EltVT == MVT::i64) {
13626 // PINSR* works with constant index.
13631 if (EltVT == MVT::i8)
13634 if (EltVT.getSizeInBits() == 16) {
13635 // Transform it so it match pinsrw which expects a 16-bit value in a GR32
13636 // as its second argument.
13637 if (N1.getValueType() != MVT::i32)
13638 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
13639 if (N2.getValueType() != MVT::i32)
13640 N2 = DAG.getIntPtrConstant(IdxVal);
13641 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
13646 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
13648 MVT OpVT = Op.getSimpleValueType();
13650 // If this is a 256-bit vector result, first insert into a 128-bit
13651 // vector and then insert into the 256-bit vector.
13652 if (!OpVT.is128BitVector()) {
13653 // Insert into a 128-bit vector.
13654 unsigned SizeFactor = OpVT.getSizeInBits()/128;
13655 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
13656 OpVT.getVectorNumElements() / SizeFactor);
13658 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
13660 // Insert the 128-bit vector.
13661 return Insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
13664 if (OpVT == MVT::v1i64 &&
13665 Op.getOperand(0).getValueType() == MVT::i64)
13666 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
13668 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
13669 assert(OpVT.is128BitVector() && "Expected an SSE type!");
13670 return DAG.getNode(ISD::BITCAST, dl, OpVT,
13671 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt));
13674 // Lower a node with an EXTRACT_SUBVECTOR opcode. This may result in
13675 // a simple subregister reference or explicit instructions to grab
13676 // upper bits of a vector.
13677 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
13678 SelectionDAG &DAG) {
13680 SDValue In = Op.getOperand(0);
13681 SDValue Idx = Op.getOperand(1);
13682 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13683 MVT ResVT = Op.getSimpleValueType();
13684 MVT InVT = In.getSimpleValueType();
13686 if (Subtarget->hasFp256()) {
13687 if (ResVT.is128BitVector() &&
13688 (InVT.is256BitVector() || InVT.is512BitVector()) &&
13689 isa<ConstantSDNode>(Idx)) {
13690 return Extract128BitVector(In, IdxVal, DAG, dl);
13692 if (ResVT.is256BitVector() && InVT.is512BitVector() &&
13693 isa<ConstantSDNode>(Idx)) {
13694 return Extract256BitVector(In, IdxVal, DAG, dl);
13700 // Lower a node with an INSERT_SUBVECTOR opcode. This may result in a
13701 // simple superregister reference or explicit instructions to insert
13702 // the upper bits of a vector.
13703 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
13704 SelectionDAG &DAG) {
13705 if (!Subtarget->hasAVX())
13709 SDValue Vec = Op.getOperand(0);
13710 SDValue SubVec = Op.getOperand(1);
13711 SDValue Idx = Op.getOperand(2);
13713 if (!isa<ConstantSDNode>(Idx))
13716 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
13717 MVT OpVT = Op.getSimpleValueType();
13718 MVT SubVecVT = SubVec.getSimpleValueType();
13720 // Fold two 16-byte subvector loads into one 32-byte load:
13721 // (insert_subvector (insert_subvector undef, (load addr), 0),
13722 // (load addr + 16), Elts/2)
13724 if ((IdxVal == OpVT.getVectorNumElements() / 2) &&
13725 Vec.getOpcode() == ISD::INSERT_SUBVECTOR &&
13726 OpVT.is256BitVector() && SubVecVT.is128BitVector() &&
13727 !Subtarget->isUnalignedMem32Slow()) {
13728 SDValue SubVec2 = Vec.getOperand(1);
13729 if (auto *Idx2 = dyn_cast<ConstantSDNode>(Vec.getOperand(2))) {
13730 if (Idx2->getZExtValue() == 0) {
13731 SDValue Ops[] = { SubVec2, SubVec };
13732 SDValue LD = EltsFromConsecutiveLoads(OpVT, Ops, dl, DAG, false);
13739 if ((OpVT.is256BitVector() || OpVT.is512BitVector()) &&
13740 SubVecVT.is128BitVector())
13741 return Insert128BitVector(Vec, SubVec, IdxVal, DAG, dl);
13743 if (OpVT.is512BitVector() && SubVecVT.is256BitVector())
13744 return Insert256BitVector(Vec, SubVec, IdxVal, DAG, dl);
13749 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
13750 // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
13751 // one of the above mentioned nodes. It has to be wrapped because otherwise
13752 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
13753 // be used to form addressing mode. These wrapped nodes will be selected
13756 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
13757 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
13759 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
13760 // global base reg.
13761 unsigned char OpFlag = 0;
13762 unsigned WrapperKind = X86ISD::Wrapper;
13763 CodeModel::Model M = DAG.getTarget().getCodeModel();
13765 if (Subtarget->isPICStyleRIPRel() &&
13766 (M == CodeModel::Small || M == CodeModel::Kernel))
13767 WrapperKind = X86ISD::WrapperRIP;
13768 else if (Subtarget->isPICStyleGOT())
13769 OpFlag = X86II::MO_GOTOFF;
13770 else if (Subtarget->isPICStyleStubPIC())
13771 OpFlag = X86II::MO_PIC_BASE_OFFSET;
13773 SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(),
13774 CP->getAlignment(),
13775 CP->getOffset(), OpFlag);
13777 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
13778 // With PIC, the address is actually $g + Offset.
13780 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
13781 DAG.getNode(X86ISD::GlobalBaseReg,
13782 SDLoc(), getPointerTy()),
13789 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
13790 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
13792 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
13793 // global base reg.
13794 unsigned char OpFlag = 0;
13795 unsigned WrapperKind = X86ISD::Wrapper;
13796 CodeModel::Model M = DAG.getTarget().getCodeModel();
13798 if (Subtarget->isPICStyleRIPRel() &&
13799 (M == CodeModel::Small || M == CodeModel::Kernel))
13800 WrapperKind = X86ISD::WrapperRIP;
13801 else if (Subtarget->isPICStyleGOT())
13802 OpFlag = X86II::MO_GOTOFF;
13803 else if (Subtarget->isPICStyleStubPIC())
13804 OpFlag = X86II::MO_PIC_BASE_OFFSET;
13806 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(),
13809 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
13811 // With PIC, the address is actually $g + Offset.
13813 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
13814 DAG.getNode(X86ISD::GlobalBaseReg,
13815 SDLoc(), getPointerTy()),
13822 X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
13823 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
13825 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
13826 // global base reg.
13827 unsigned char OpFlag = 0;
13828 unsigned WrapperKind = X86ISD::Wrapper;
13829 CodeModel::Model M = DAG.getTarget().getCodeModel();
13831 if (Subtarget->isPICStyleRIPRel() &&
13832 (M == CodeModel::Small || M == CodeModel::Kernel)) {
13833 if (Subtarget->isTargetDarwin() || Subtarget->isTargetELF())
13834 OpFlag = X86II::MO_GOTPCREL;
13835 WrapperKind = X86ISD::WrapperRIP;
13836 } else if (Subtarget->isPICStyleGOT()) {
13837 OpFlag = X86II::MO_GOT;
13838 } else if (Subtarget->isPICStyleStubPIC()) {
13839 OpFlag = X86II::MO_DARWIN_NONLAZY_PIC_BASE;
13840 } else if (Subtarget->isPICStyleStubNoDynamic()) {
13841 OpFlag = X86II::MO_DARWIN_NONLAZY;
13844 SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag);
13847 Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
13849 // With PIC, the address is actually $g + Offset.
13850 if (DAG.getTarget().getRelocationModel() == Reloc::PIC_ &&
13851 !Subtarget->is64Bit()) {
13852 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
13853 DAG.getNode(X86ISD::GlobalBaseReg,
13854 SDLoc(), getPointerTy()),
13858 // For symbols that require a load from a stub to get the address, emit the
13860 if (isGlobalStubReference(OpFlag))
13861 Result = DAG.getLoad(getPointerTy(), DL, DAG.getEntryNode(), Result,
13862 MachinePointerInfo::getGOT(), false, false, false, 0);
13868 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
13869 // Create the TargetBlockAddressAddress node.
13870 unsigned char OpFlags =
13871 Subtarget->ClassifyBlockAddressReference();
13872 CodeModel::Model M = DAG.getTarget().getCodeModel();
13873 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
13874 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
13876 SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy(), Offset,
13879 if (Subtarget->isPICStyleRIPRel() &&
13880 (M == CodeModel::Small || M == CodeModel::Kernel))
13881 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
13883 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
13885 // With PIC, the address is actually $g + Offset.
13886 if (isGlobalRelativeToPICBase(OpFlags)) {
13887 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
13888 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
13896 X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, SDLoc dl,
13897 int64_t Offset, SelectionDAG &DAG) const {
13898 // Create the TargetGlobalAddress node, folding in the constant
13899 // offset if it is legal.
13900 unsigned char OpFlags =
13901 Subtarget->ClassifyGlobalReference(GV, DAG.getTarget());
13902 CodeModel::Model M = DAG.getTarget().getCodeModel();
13904 if (OpFlags == X86II::MO_NO_FLAG &&
13905 X86::isOffsetSuitableForCodeModel(Offset, M)) {
13906 // A direct static reference to a global.
13907 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), Offset);
13910 Result = DAG.getTargetGlobalAddress(GV, dl, getPointerTy(), 0, OpFlags);
13913 if (Subtarget->isPICStyleRIPRel() &&
13914 (M == CodeModel::Small || M == CodeModel::Kernel))
13915 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
13917 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
13919 // With PIC, the address is actually $g + Offset.
13920 if (isGlobalRelativeToPICBase(OpFlags)) {
13921 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
13922 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
13926 // For globals that require a load from a stub to get the address, emit the
13928 if (isGlobalStubReference(OpFlags))
13929 Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result,
13930 MachinePointerInfo::getGOT(), false, false, false, 0);
13932 // If there was a non-zero offset that we didn't fold, create an explicit
13933 // addition for it.
13935 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result,
13936 DAG.getConstant(Offset, getPointerTy()));
13942 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
13943 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
13944 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
13945 return LowerGlobalAddress(GV, SDLoc(Op), Offset, DAG);
13949 GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
13950 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
13951 unsigned char OperandFlags, bool LocalDynamic = false) {
13952 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
13953 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
13955 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
13956 GA->getValueType(0),
13960 X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
13964 SDValue Ops[] = { Chain, TGA, *InFlag };
13965 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
13967 SDValue Ops[] = { Chain, TGA };
13968 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
13971 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
13972 MFI->setAdjustsStack(true);
13973 MFI->setHasCalls(true);
13975 SDValue Flag = Chain.getValue(1);
13976 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag);
13979 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
13981 LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
13984 SDLoc dl(GA); // ? function entry point might be better
13985 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
13986 DAG.getNode(X86ISD::GlobalBaseReg,
13987 SDLoc(), PtrVT), InFlag);
13988 InFlag = Chain.getValue(1);
13990 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
13993 // Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit
13995 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
13997 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
13998 X86::RAX, X86II::MO_TLSGD);
14001 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
14007 // Get the start address of the TLS block for this module.
14008 X86MachineFunctionInfo* MFI = DAG.getMachineFunction()
14009 .getInfo<X86MachineFunctionInfo>();
14010 MFI->incNumLocalDynamicTLSAccesses();
14014 Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, X86::RAX,
14015 X86II::MO_TLSLD, /*LocalDynamic=*/true);
14018 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
14019 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
14020 InFlag = Chain.getValue(1);
14021 Base = GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX,
14022 X86II::MO_TLSLDM, /*LocalDynamic=*/true);
14025 // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
14029 unsigned char OperandFlags = X86II::MO_DTPOFF;
14030 unsigned WrapperKind = X86ISD::Wrapper;
14031 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
14032 GA->getValueType(0),
14033 GA->getOffset(), OperandFlags);
14034 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
14036 // Add x@dtpoff with the base.
14037 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
14040 // Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
14041 static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
14042 const EVT PtrVT, TLSModel::Model model,
14043 bool is64Bit, bool isPIC) {
14046 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
14047 Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(),
14048 is64Bit ? 257 : 256));
14050 SDValue ThreadPointer =
14051 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0),
14052 MachinePointerInfo(Ptr), false, false, false, 0);
14054 unsigned char OperandFlags = 0;
14055 // Most TLS accesses are not RIP relative, even on x86-64. One exception is
14057 unsigned WrapperKind = X86ISD::Wrapper;
14058 if (model == TLSModel::LocalExec) {
14059 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
14060 } else if (model == TLSModel::InitialExec) {
14062 OperandFlags = X86II::MO_GOTTPOFF;
14063 WrapperKind = X86ISD::WrapperRIP;
14065 OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
14068 llvm_unreachable("Unexpected model");
14071 // emit "addl x@ntpoff,%eax" (local exec)
14072 // or "addl x@indntpoff,%eax" (initial exec)
14073 // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
14075 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
14076 GA->getOffset(), OperandFlags);
14077 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
14079 if (model == TLSModel::InitialExec) {
14080 if (isPIC && !is64Bit) {
14081 Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
14082 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
14086 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
14087 MachinePointerInfo::getGOT(), false, false, false, 0);
14090 // The address of the thread local variable is the add of the thread
14091 // pointer with the offset of the variable.
14092 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
14096 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
14098 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
14099 const GlobalValue *GV = GA->getGlobal();
14101 if (Subtarget->isTargetELF()) {
14102 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
14105 case TLSModel::GeneralDynamic:
14106 if (Subtarget->is64Bit())
14107 return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy());
14108 return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy());
14109 case TLSModel::LocalDynamic:
14110 return LowerToTLSLocalDynamicModel(GA, DAG, getPointerTy(),
14111 Subtarget->is64Bit());
14112 case TLSModel::InitialExec:
14113 case TLSModel::LocalExec:
14114 return LowerToTLSExecModel(
14115 GA, DAG, getPointerTy(), model, Subtarget->is64Bit(),
14116 DAG.getTarget().getRelocationModel() == Reloc::PIC_);
14118 llvm_unreachable("Unknown TLS model.");
14121 if (Subtarget->isTargetDarwin()) {
14122 // Darwin only has one model of TLS. Lower to that.
14123 unsigned char OpFlag = 0;
14124 unsigned WrapperKind = Subtarget->isPICStyleRIPRel() ?
14125 X86ISD::WrapperRIP : X86ISD::Wrapper;
14127 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
14128 // global base reg.
14129 bool PIC32 = (DAG.getTarget().getRelocationModel() == Reloc::PIC_) &&
14130 !Subtarget->is64Bit();
14132 OpFlag = X86II::MO_TLVP_PIC_BASE;
14134 OpFlag = X86II::MO_TLVP;
14136 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
14137 GA->getValueType(0),
14138 GA->getOffset(), OpFlag);
14139 SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
14141 // With PIC32, the address is actually $g + Offset.
14143 Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(),
14144 DAG.getNode(X86ISD::GlobalBaseReg,
14145 SDLoc(), getPointerTy()),
14148 // Lowering the machine isd will make sure everything is in the right
14150 SDValue Chain = DAG.getEntryNode();
14151 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
14152 SDValue Args[] = { Chain, Offset };
14153 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
14155 // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
14156 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
14157 MFI->setAdjustsStack(true);
14159 // And our return value (tls address) is in the standard call return value
14161 unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
14162 return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy(),
14163 Chain.getValue(1));
14166 if (Subtarget->isTargetKnownWindowsMSVC() ||
14167 Subtarget->isTargetWindowsGNU()) {
14168 // Just use the implicit TLS architecture
14169 // Need to generate someting similar to:
14170 // mov rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage
14172 // mov ecx, dword [rel _tls_index]: Load index (from C runtime)
14173 // mov rcx, qword [rdx+rcx*8]
14174 // mov eax, .tls$:tlsvar
14175 // [rax+rcx] contains the address
14176 // Windows 64bit: gs:0x58
14177 // Windows 32bit: fs:__tls_array
14180 SDValue Chain = DAG.getEntryNode();
14182 // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or
14183 // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly
14184 // use its literal value of 0x2C.
14185 Value *Ptr = Constant::getNullValue(Subtarget->is64Bit()
14186 ? Type::getInt8PtrTy(*DAG.getContext(),
14188 : Type::getInt32PtrTy(*DAG.getContext(),
14192 Subtarget->is64Bit()
14193 ? DAG.getIntPtrConstant(0x58)
14194 : (Subtarget->isTargetWindowsGNU()
14195 ? DAG.getIntPtrConstant(0x2C)
14196 : DAG.getExternalSymbol("_tls_array", getPointerTy()));
14198 SDValue ThreadPointer =
14199 DAG.getLoad(getPointerTy(), dl, Chain, TlsArray,
14200 MachinePointerInfo(Ptr), false, false, false, 0);
14202 // Load the _tls_index variable
14203 SDValue IDX = DAG.getExternalSymbol("_tls_index", getPointerTy());
14204 if (Subtarget->is64Bit())
14205 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, getPointerTy(), Chain,
14206 IDX, MachinePointerInfo(), MVT::i32,
14207 false, false, false, 0);
14209 IDX = DAG.getLoad(getPointerTy(), dl, Chain, IDX, MachinePointerInfo(),
14210 false, false, false, 0);
14212 SDValue Scale = DAG.getConstant(Log2_64_Ceil(TD->getPointerSize()),
14214 IDX = DAG.getNode(ISD::SHL, dl, getPointerTy(), IDX, Scale);
14216 SDValue res = DAG.getNode(ISD::ADD, dl, getPointerTy(), ThreadPointer, IDX);
14217 res = DAG.getLoad(getPointerTy(), dl, Chain, res, MachinePointerInfo(),
14218 false, false, false, 0);
14220 // Get the offset of start of .tls section
14221 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
14222 GA->getValueType(0),
14223 GA->getOffset(), X86II::MO_SECREL);
14224 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), TGA);
14226 // The address of the thread local variable is the add of the thread
14227 // pointer with the offset of the variable.
14228 return DAG.getNode(ISD::ADD, dl, getPointerTy(), res, Offset);
14231 llvm_unreachable("TLS not implemented for this target.");
14234 /// LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values
14235 /// and take a 2 x i32 value to shift plus a shift amount.
14236 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
14237 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
14238 MVT VT = Op.getSimpleValueType();
14239 unsigned VTBits = VT.getSizeInBits();
14241 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
14242 SDValue ShOpLo = Op.getOperand(0);
14243 SDValue ShOpHi = Op.getOperand(1);
14244 SDValue ShAmt = Op.getOperand(2);
14245 // X86ISD::SHLD and X86ISD::SHRD have defined overflow behavior but the
14246 // generic ISD nodes haven't. Insert an AND to be safe, it's optimized away
14248 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
14249 DAG.getConstant(VTBits - 1, MVT::i8));
14250 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
14251 DAG.getConstant(VTBits - 1, MVT::i8))
14252 : DAG.getConstant(0, VT);
14254 SDValue Tmp2, Tmp3;
14255 if (Op.getOpcode() == ISD::SHL_PARTS) {
14256 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
14257 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
14259 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
14260 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
14263 // If the shift amount is larger or equal than the width of a part we can't
14264 // rely on the results of shld/shrd. Insert a test and select the appropriate
14265 // values for large shift amounts.
14266 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
14267 DAG.getConstant(VTBits, MVT::i8));
14268 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
14269 AndNode, DAG.getConstant(0, MVT::i8));
14272 SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8);
14273 SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond };
14274 SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond };
14276 if (Op.getOpcode() == ISD::SHL_PARTS) {
14277 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
14278 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
14280 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
14281 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
14284 SDValue Ops[2] = { Lo, Hi };
14285 return DAG.getMergeValues(Ops, dl);
14288 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
14289 SelectionDAG &DAG) const {
14290 MVT SrcVT = Op.getOperand(0).getSimpleValueType();
14293 if (SrcVT.isVector()) {
14294 if (SrcVT.getVectorElementType() == MVT::i1) {
14295 MVT IntegerVT = MVT::getVectorVT(MVT::i32, SrcVT.getVectorNumElements());
14296 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
14297 DAG.getNode(ISD::SIGN_EXTEND, dl, IntegerVT,
14298 Op.getOperand(0)));
14303 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
14304 "Unknown SINT_TO_FP to lower!");
14306 // These are really Legal; return the operand so the caller accepts it as
14308 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType()))
14310 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
14311 Subtarget->is64Bit()) {
14315 unsigned Size = SrcVT.getSizeInBits()/8;
14316 MachineFunction &MF = DAG.getMachineFunction();
14317 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size, false);
14318 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14319 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
14321 MachinePointerInfo::getFixedStack(SSFI),
14323 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
14326 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
14328 SelectionDAG &DAG) const {
14332 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
14334 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue);
14336 Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
14338 unsigned ByteSize = SrcVT.getSizeInBits()/8;
14340 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(StackSlot);
14341 MachineMemOperand *MMO;
14343 int SSFI = FI->getIndex();
14345 DAG.getMachineFunction()
14346 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14347 MachineMemOperand::MOLoad, ByteSize, ByteSize);
14349 MMO = cast<LoadSDNode>(StackSlot)->getMemOperand();
14350 StackSlot = StackSlot.getOperand(1);
14352 SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) };
14353 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
14355 Tys, Ops, SrcVT, MMO);
14358 Chain = Result.getValue(1);
14359 SDValue InFlag = Result.getValue(2);
14361 // FIXME: Currently the FST is flagged to the FILD_FLAG. This
14362 // shouldn't be necessary except that RFP cannot be live across
14363 // multiple blocks. When stackifier is fixed, they can be uncoupled.
14364 MachineFunction &MF = DAG.getMachineFunction();
14365 unsigned SSFISize = Op.getValueType().getSizeInBits()/8;
14366 int SSFI = MF.getFrameInfo()->CreateStackObject(SSFISize, SSFISize, false);
14367 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14368 Tys = DAG.getVTList(MVT::Other);
14370 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
14372 MachineMemOperand *MMO =
14373 DAG.getMachineFunction()
14374 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14375 MachineMemOperand::MOStore, SSFISize, SSFISize);
14377 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys,
14378 Ops, Op.getValueType(), MMO);
14379 Result = DAG.getLoad(Op.getValueType(), DL, Chain, StackSlot,
14380 MachinePointerInfo::getFixedStack(SSFI),
14381 false, false, false, 0);
14387 // LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion.
14388 SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
14389 SelectionDAG &DAG) const {
14390 // This algorithm is not obvious. Here it is what we're trying to output:
14393 punpckldq (c0), %xmm0 // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U }
14394 subpd (c1), %xmm0 // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 }
14396 haddpd %xmm0, %xmm0
14398 pshufd $0x4e, %xmm0, %xmm1
14404 LLVMContext *Context = DAG.getContext();
14406 // Build some magic constants.
14407 static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
14408 Constant *C0 = ConstantDataVector::get(*Context, CV0);
14409 SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16);
14411 SmallVector<Constant*,2> CV1;
14413 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
14414 APInt(64, 0x4330000000000000ULL))));
14416 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble,
14417 APInt(64, 0x4530000000000000ULL))));
14418 Constant *C1 = ConstantVector::get(CV1);
14419 SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16);
14421 // Load the 64-bit value into an XMM register.
14422 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
14424 SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
14425 MachinePointerInfo::getConstantPool(),
14426 false, false, false, 16);
14427 SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32,
14428 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1),
14431 SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
14432 MachinePointerInfo::getConstantPool(),
14433 false, false, false, 16);
14434 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1);
14435 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
14438 if (Subtarget->hasSSE3()) {
14439 // FIXME: The 'haddpd' instruction may be slower than 'movhlps + addsd'.
14440 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
14442 SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub);
14443 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32,
14445 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64,
14446 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle),
14450 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
14451 DAG.getIntPtrConstant(0));
14454 // LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion.
14455 SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
14456 SelectionDAG &DAG) const {
14458 // FP constant to bias correct the final result.
14459 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
14462 // Load the 32-bit value into an XMM register.
14463 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
14466 // Zero out the upper parts of the register.
14467 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
14469 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
14470 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load),
14471 DAG.getIntPtrConstant(0));
14473 // Or the load with the bias.
14474 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64,
14475 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
14476 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
14477 MVT::v2f64, Load)),
14478 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64,
14479 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
14480 MVT::v2f64, Bias)));
14481 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
14482 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or),
14483 DAG.getIntPtrConstant(0));
14485 // Subtract the bias.
14486 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
14488 // Handle final rounding.
14489 EVT DestVT = Op.getValueType();
14491 if (DestVT.bitsLT(MVT::f64))
14492 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
14493 DAG.getIntPtrConstant(0));
14494 if (DestVT.bitsGT(MVT::f64))
14495 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
14497 // Handle final rounding.
14501 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
14502 const X86Subtarget &Subtarget) {
14503 // The algorithm is the following:
14504 // #ifdef __SSE4_1__
14505 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
14506 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
14507 // (uint4) 0x53000000, 0xaa);
14509 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
14510 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
14512 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
14513 // return (float4) lo + fhi;
14516 SDValue V = Op->getOperand(0);
14517 EVT VecIntVT = V.getValueType();
14518 bool Is128 = VecIntVT == MVT::v4i32;
14519 EVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
14520 // If we convert to something else than the supported type, e.g., to v4f64,
14522 if (VecFloatVT != Op->getValueType(0))
14525 unsigned NumElts = VecIntVT.getVectorNumElements();
14526 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
14527 "Unsupported custom type");
14528 assert(NumElts <= 8 && "The size of the constant array must be fixed");
14530 // In the #idef/#else code, we have in common:
14531 // - The vector of constants:
14537 // Create the splat vector for 0x4b000000.
14538 SDValue CstLow = DAG.getConstant(0x4b000000, MVT::i32);
14539 SDValue CstLowArray[] = {CstLow, CstLow, CstLow, CstLow,
14540 CstLow, CstLow, CstLow, CstLow};
14541 SDValue VecCstLow = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
14542 makeArrayRef(&CstLowArray[0], NumElts));
14543 // Create the splat vector for 0x53000000.
14544 SDValue CstHigh = DAG.getConstant(0x53000000, MVT::i32);
14545 SDValue CstHighArray[] = {CstHigh, CstHigh, CstHigh, CstHigh,
14546 CstHigh, CstHigh, CstHigh, CstHigh};
14547 SDValue VecCstHigh = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
14548 makeArrayRef(&CstHighArray[0], NumElts));
14550 // Create the right shift.
14551 SDValue CstShift = DAG.getConstant(16, MVT::i32);
14552 SDValue CstShiftArray[] = {CstShift, CstShift, CstShift, CstShift,
14553 CstShift, CstShift, CstShift, CstShift};
14554 SDValue VecCstShift = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT,
14555 makeArrayRef(&CstShiftArray[0], NumElts));
14556 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
14559 if (Subtarget.hasSSE41()) {
14560 EVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
14561 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
14562 SDValue VecCstLowBitcast =
14563 DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstLow);
14564 SDValue VecBitcast = DAG.getNode(ISD::BITCAST, DL, VecI16VT, V);
14565 // Low will be bitcasted right away, so do not bother bitcasting back to its
14567 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
14568 VecCstLowBitcast, DAG.getConstant(0xaa, MVT::i32));
14569 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
14570 // (uint4) 0x53000000, 0xaa);
14571 SDValue VecCstHighBitcast =
14572 DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstHigh);
14573 SDValue VecShiftBitcast =
14574 DAG.getNode(ISD::BITCAST, DL, VecI16VT, HighShift);
14575 // High will be bitcasted right away, so do not bother bitcasting back to
14576 // its original type.
14577 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
14578 VecCstHighBitcast, DAG.getConstant(0xaa, MVT::i32));
14580 SDValue CstMask = DAG.getConstant(0xffff, MVT::i32);
14581 SDValue VecCstMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, CstMask,
14582 CstMask, CstMask, CstMask);
14583 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
14584 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
14585 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
14587 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
14588 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
14591 // Create the vector constant for -(0x1.0p39f + 0x1.0p23f).
14592 SDValue CstFAdd = DAG.getConstantFP(
14593 APFloat(APFloat::IEEEsingle, APInt(32, 0xD3000080)), MVT::f32);
14594 SDValue CstFAddArray[] = {CstFAdd, CstFAdd, CstFAdd, CstFAdd,
14595 CstFAdd, CstFAdd, CstFAdd, CstFAdd};
14596 SDValue VecCstFAdd = DAG.getNode(ISD::BUILD_VECTOR, DL, VecFloatVT,
14597 makeArrayRef(&CstFAddArray[0], NumElts));
14599 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
14600 SDValue HighBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, High);
14602 DAG.getNode(ISD::FADD, DL, VecFloatVT, HighBitcast, VecCstFAdd);
14603 // return (float4) lo + fhi;
14604 SDValue LowBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, Low);
14605 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
14608 SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op,
14609 SelectionDAG &DAG) const {
14610 SDValue N0 = Op.getOperand(0);
14611 MVT SVT = N0.getSimpleValueType();
14614 switch (SVT.SimpleTy) {
14616 llvm_unreachable("Custom UINT_TO_FP is not supported!");
14621 MVT NVT = MVT::getVectorVT(MVT::i32, SVT.getVectorNumElements());
14622 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
14623 DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N0));
14627 return lowerUINT_TO_FP_vXi32(Op, DAG, *Subtarget);
14629 llvm_unreachable(nullptr);
14632 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
14633 SelectionDAG &DAG) const {
14634 SDValue N0 = Op.getOperand(0);
14637 if (Op.getValueType().isVector())
14638 return lowerUINT_TO_FP_vec(Op, DAG);
14640 // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
14641 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
14642 // the optimization here.
14643 if (DAG.SignBitIsZero(N0))
14644 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0);
14646 MVT SrcVT = N0.getSimpleValueType();
14647 MVT DstVT = Op.getSimpleValueType();
14648 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64)
14649 return LowerUINT_TO_FP_i64(Op, DAG);
14650 if (SrcVT == MVT::i32 && X86ScalarSSEf64)
14651 return LowerUINT_TO_FP_i32(Op, DAG);
14652 if (Subtarget->is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32)
14655 // Make a 64-bit buffer, and use it to build an FILD.
14656 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
14657 if (SrcVT == MVT::i32) {
14658 SDValue WordOff = DAG.getConstant(4, getPointerTy());
14659 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl,
14660 getPointerTy(), StackSlot, WordOff);
14661 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
14662 StackSlot, MachinePointerInfo(),
14664 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32),
14665 OffsetSlot, MachinePointerInfo(),
14667 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
14671 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");
14672 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
14673 StackSlot, MachinePointerInfo(),
14675 // For i64 source, we need to add the appropriate power of 2 if the input
14676 // was negative. This is the same as the optimization in
14677 // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here,
14678 // we must be careful to do the computation in x87 extended precision, not
14679 // in SSE. (The generic code can't know it's OK to do this, or how to.)
14680 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex();
14681 MachineMemOperand *MMO =
14682 DAG.getMachineFunction()
14683 .getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14684 MachineMemOperand::MOLoad, 8, 8);
14686 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
14687 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) };
14688 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
14691 APInt FF(32, 0x5F800000ULL);
14693 // Check whether the sign bit is set.
14694 SDValue SignSet = DAG.getSetCC(dl,
14695 getSetCCResultType(*DAG.getContext(), MVT::i64),
14696 Op.getOperand(0), DAG.getConstant(0, MVT::i64),
14699 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
14700 SDValue FudgePtr = DAG.getConstantPool(
14701 ConstantInt::get(*DAG.getContext(), FF.zext(64)),
14704 // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
14705 SDValue Zero = DAG.getIntPtrConstant(0);
14706 SDValue Four = DAG.getIntPtrConstant(4);
14707 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet,
14709 FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset);
14711 // Load the value out, extending it from f32 to f80.
14712 // FIXME: Avoid the extend by constructing the right constant pool?
14713 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(),
14714 FudgePtr, MachinePointerInfo::getConstantPool(),
14715 MVT::f32, false, false, false, 4);
14716 // Extend everything to 80 bits to force it to be done on x87.
14717 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
14718 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0));
14721 std::pair<SDValue,SDValue>
14722 X86TargetLowering:: FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
14723 bool IsSigned, bool IsReplace) const {
14726 EVT DstTy = Op.getValueType();
14728 if (!IsSigned && !isIntegerTypeFTOL(DstTy)) {
14729 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
14733 assert(DstTy.getSimpleVT() <= MVT::i64 &&
14734 DstTy.getSimpleVT() >= MVT::i16 &&
14735 "Unknown FP_TO_INT to lower!");
14737 // These are really Legal.
14738 if (DstTy == MVT::i32 &&
14739 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
14740 return std::make_pair(SDValue(), SDValue());
14741 if (Subtarget->is64Bit() &&
14742 DstTy == MVT::i64 &&
14743 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
14744 return std::make_pair(SDValue(), SDValue());
14746 // We lower FP->int64 either into FISTP64 followed by a load from a temporary
14747 // stack slot, or into the FTOL runtime function.
14748 MachineFunction &MF = DAG.getMachineFunction();
14749 unsigned MemSize = DstTy.getSizeInBits()/8;
14750 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
14751 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14754 if (!IsSigned && isIntegerTypeFTOL(DstTy))
14755 Opc = X86ISD::WIN_FTOL;
14757 switch (DstTy.getSimpleVT().SimpleTy) {
14758 default: llvm_unreachable("Invalid FP_TO_SINT to lower!");
14759 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
14760 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
14761 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
14764 SDValue Chain = DAG.getEntryNode();
14765 SDValue Value = Op.getOperand(0);
14766 EVT TheVT = Op.getOperand(0).getValueType();
14767 // FIXME This causes a redundant load/store if the SSE-class value is already
14768 // in memory, such as if it is on the callstack.
14769 if (isScalarFPTypeInSSEReg(TheVT)) {
14770 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
14771 Chain = DAG.getStore(Chain, DL, Value, StackSlot,
14772 MachinePointerInfo::getFixedStack(SSFI),
14774 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
14776 Chain, StackSlot, DAG.getValueType(TheVT)
14779 MachineMemOperand *MMO =
14780 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14781 MachineMemOperand::MOLoad, MemSize, MemSize);
14782 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
14783 Chain = Value.getValue(1);
14784 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
14785 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
14788 MachineMemOperand *MMO =
14789 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
14790 MachineMemOperand::MOStore, MemSize, MemSize);
14792 if (Opc != X86ISD::WIN_FTOL) {
14793 // Build the FP_TO_INT*_IN_MEM
14794 SDValue Ops[] = { Chain, Value, StackSlot };
14795 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other),
14797 return std::make_pair(FIST, StackSlot);
14799 SDValue ftol = DAG.getNode(X86ISD::WIN_FTOL, DL,
14800 DAG.getVTList(MVT::Other, MVT::Glue),
14802 SDValue eax = DAG.getCopyFromReg(ftol, DL, X86::EAX,
14803 MVT::i32, ftol.getValue(1));
14804 SDValue edx = DAG.getCopyFromReg(eax.getValue(1), DL, X86::EDX,
14805 MVT::i32, eax.getValue(2));
14806 SDValue Ops[] = { eax, edx };
14807 SDValue pair = IsReplace
14808 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops)
14809 : DAG.getMergeValues(Ops, DL);
14810 return std::make_pair(pair, SDValue());
14814 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
14815 const X86Subtarget *Subtarget) {
14816 MVT VT = Op->getSimpleValueType(0);
14817 SDValue In = Op->getOperand(0);
14818 MVT InVT = In.getSimpleValueType();
14821 // Optimize vectors in AVX mode:
14824 // Use vpunpcklwd for 4 lower elements v8i16 -> v4i32.
14825 // Use vpunpckhwd for 4 upper elements v8i16 -> v4i32.
14826 // Concat upper and lower parts.
14829 // Use vpunpckldq for 4 lower elements v4i32 -> v2i64.
14830 // Use vpunpckhdq for 4 upper elements v4i32 -> v2i64.
14831 // Concat upper and lower parts.
14834 if (((VT != MVT::v16i16) || (InVT != MVT::v16i8)) &&
14835 ((VT != MVT::v8i32) || (InVT != MVT::v8i16)) &&
14836 ((VT != MVT::v4i64) || (InVT != MVT::v4i32)))
14839 if (Subtarget->hasInt256())
14840 return DAG.getNode(X86ISD::VZEXT, dl, VT, In);
14842 SDValue ZeroVec = getZeroVector(InVT, Subtarget, DAG, dl);
14843 SDValue Undef = DAG.getUNDEF(InVT);
14844 bool NeedZero = Op.getOpcode() == ISD::ZERO_EXTEND;
14845 SDValue OpLo = getUnpackl(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
14846 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
14848 MVT HVT = MVT::getVectorVT(VT.getVectorElementType(),
14849 VT.getVectorNumElements()/2);
14851 OpLo = DAG.getNode(ISD::BITCAST, dl, HVT, OpLo);
14852 OpHi = DAG.getNode(ISD::BITCAST, dl, HVT, OpHi);
14854 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
14857 static SDValue LowerZERO_EXTEND_AVX512(SDValue Op,
14858 SelectionDAG &DAG) {
14859 MVT VT = Op->getSimpleValueType(0);
14860 SDValue In = Op->getOperand(0);
14861 MVT InVT = In.getSimpleValueType();
14863 unsigned int NumElts = VT.getVectorNumElements();
14864 if (NumElts != 8 && NumElts != 16)
14867 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1)
14868 return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
14870 EVT ExtVT = (NumElts == 8)? MVT::v8i64 : MVT::v16i32;
14871 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14872 // Now we have only mask extension
14873 assert(InVT.getVectorElementType() == MVT::i1);
14874 SDValue Cst = DAG.getTargetConstant(1, ExtVT.getScalarType());
14875 const Constant *C = (dyn_cast<ConstantSDNode>(Cst))->getConstantIntValue();
14876 SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
14877 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
14878 SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
14879 MachinePointerInfo::getConstantPool(),
14880 false, false, false, Alignment);
14882 SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, DL, ExtVT, In, Ld);
14883 if (VT.is512BitVector())
14885 return DAG.getNode(X86ISD::VTRUNC, DL, VT, Brcst);
14888 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
14889 SelectionDAG &DAG) {
14890 if (Subtarget->hasFp256()) {
14891 SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
14899 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
14900 SelectionDAG &DAG) {
14902 MVT VT = Op.getSimpleValueType();
14903 SDValue In = Op.getOperand(0);
14904 MVT SVT = In.getSimpleValueType();
14906 if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1)
14907 return LowerZERO_EXTEND_AVX512(Op, DAG);
14909 if (Subtarget->hasFp256()) {
14910 SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
14915 assert(!VT.is256BitVector() || !SVT.is128BitVector() ||
14916 VT.getVectorNumElements() != SVT.getVectorNumElements());
14920 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
14922 MVT VT = Op.getSimpleValueType();
14923 SDValue In = Op.getOperand(0);
14924 MVT InVT = In.getSimpleValueType();
14926 if (VT == MVT::i1) {
14927 assert((InVT.isInteger() && (InVT.getSizeInBits() <= 64)) &&
14928 "Invalid scalar TRUNCATE operation");
14929 if (InVT.getSizeInBits() >= 32)
14931 In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In);
14932 return DAG.getNode(ISD::TRUNCATE, DL, VT, In);
14934 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&
14935 "Invalid TRUNCATE operation");
14937 if (InVT.is512BitVector() || VT.getVectorElementType() == MVT::i1) {
14938 if (VT.getVectorElementType().getSizeInBits() >=8)
14939 return DAG.getNode(X86ISD::VTRUNC, DL, VT, In);
14941 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
14942 unsigned NumElts = InVT.getVectorNumElements();
14943 assert ((NumElts == 8 || NumElts == 16) && "Unexpected vector type");
14944 if (InVT.getSizeInBits() < 512) {
14945 MVT ExtVT = (NumElts == 16)? MVT::v16i32 : MVT::v8i64;
14946 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
14950 SDValue Cst = DAG.getTargetConstant(1, InVT.getVectorElementType());
14951 const Constant *C = (dyn_cast<ConstantSDNode>(Cst))->getConstantIntValue();
14952 SDValue CP = DAG.getConstantPool(C, getPointerTy());
14953 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
14954 SDValue Ld = DAG.getLoad(Cst.getValueType(), DL, DAG.getEntryNode(), CP,
14955 MachinePointerInfo::getConstantPool(),
14956 false, false, false, Alignment);
14957 SDValue OneV = DAG.getNode(X86ISD::VBROADCAST, DL, InVT, Ld);
14958 SDValue And = DAG.getNode(ISD::AND, DL, InVT, OneV, In);
14959 return DAG.getNode(X86ISD::TESTM, DL, VT, And, And);
14962 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
14963 // On AVX2, v4i64 -> v4i32 becomes VPERMD.
14964 if (Subtarget->hasInt256()) {
14965 static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
14966 In = DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, In);
14967 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, DAG.getUNDEF(MVT::v8i32),
14969 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
14970 DAG.getIntPtrConstant(0));
14973 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
14974 DAG.getIntPtrConstant(0));
14975 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
14976 DAG.getIntPtrConstant(2));
14977 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo);
14978 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi);
14979 static const int ShufMask[] = {0, 2, 4, 6};
14980 return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);
14983 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
14984 // On AVX2, v8i32 -> v8i16 becomed PSHUFB.
14985 if (Subtarget->hasInt256()) {
14986 In = DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, In);
14988 SmallVector<SDValue,32> pshufbMask;
14989 for (unsigned i = 0; i < 2; ++i) {
14990 pshufbMask.push_back(DAG.getConstant(0x0, MVT::i8));
14991 pshufbMask.push_back(DAG.getConstant(0x1, MVT::i8));
14992 pshufbMask.push_back(DAG.getConstant(0x4, MVT::i8));
14993 pshufbMask.push_back(DAG.getConstant(0x5, MVT::i8));
14994 pshufbMask.push_back(DAG.getConstant(0x8, MVT::i8));
14995 pshufbMask.push_back(DAG.getConstant(0x9, MVT::i8));
14996 pshufbMask.push_back(DAG.getConstant(0xc, MVT::i8));
14997 pshufbMask.push_back(DAG.getConstant(0xd, MVT::i8));
14998 for (unsigned j = 0; j < 8; ++j)
14999 pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
15001 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask);
15002 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV);
15003 In = DAG.getNode(ISD::BITCAST, DL, MVT::v4i64, In);
15005 static const int ShufMask[] = {0, 2, -1, -1};
15006 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, DAG.getUNDEF(MVT::v4i64),
15008 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
15009 DAG.getIntPtrConstant(0));
15010 return DAG.getNode(ISD::BITCAST, DL, VT, In);
15013 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
15014 DAG.getIntPtrConstant(0));
15016 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
15017 DAG.getIntPtrConstant(4));
15019 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpLo);
15020 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpHi);
15022 // The PSHUFB mask:
15023 static const int ShufMask1[] = {0, 1, 4, 5, 8, 9, 12, 13,
15024 -1, -1, -1, -1, -1, -1, -1, -1};
15026 SDValue Undef = DAG.getUNDEF(MVT::v16i8);
15027 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, Undef, ShufMask1);
15028 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, Undef, ShufMask1);
15030 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo);
15031 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi);
15033 // The MOVLHPS Mask:
15034 static const int ShufMask2[] = {0, 1, 4, 5};
15035 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
15036 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, res);
15039 // Handle truncation of V256 to V128 using shuffles.
15040 if (!VT.is128BitVector() || !InVT.is256BitVector())
15043 assert(Subtarget->hasFp256() && "256-bit vector without AVX!");
15045 unsigned NumElems = VT.getVectorNumElements();
15046 MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2);
15048 SmallVector<int, 16> MaskVec(NumElems * 2, -1);
15049 // Prepare truncation shuffle mask
15050 for (unsigned i = 0; i != NumElems; ++i)
15051 MaskVec[i] = i * 2;
15052 SDValue V = DAG.getVectorShuffle(NVT, DL,
15053 DAG.getNode(ISD::BITCAST, DL, NVT, In),
15054 DAG.getUNDEF(NVT), &MaskVec[0]);
15055 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V,
15056 DAG.getIntPtrConstant(0));
15059 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
15060 SelectionDAG &DAG) const {
15061 assert(!Op.getSimpleValueType().isVector());
15063 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
15064 /*IsSigned=*/ true, /*IsReplace=*/ false);
15065 SDValue FIST = Vals.first, StackSlot = Vals.second;
15066 // If FP_TO_INTHelper failed, the node is actually supposed to be Legal.
15067 if (!FIST.getNode()) return Op;
15069 if (StackSlot.getNode())
15070 // Load the result.
15071 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
15072 FIST, StackSlot, MachinePointerInfo(),
15073 false, false, false, 0);
15075 // The node is the result.
15079 SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op,
15080 SelectionDAG &DAG) const {
15081 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
15082 /*IsSigned=*/ false, /*IsReplace=*/ false);
15083 SDValue FIST = Vals.first, StackSlot = Vals.second;
15084 assert(FIST.getNode() && "Unexpected failure");
15086 if (StackSlot.getNode())
15087 // Load the result.
15088 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
15089 FIST, StackSlot, MachinePointerInfo(),
15090 false, false, false, 0);
15092 // The node is the result.
15096 static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) {
15098 MVT VT = Op.getSimpleValueType();
15099 SDValue In = Op.getOperand(0);
15100 MVT SVT = In.getSimpleValueType();
15102 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
15104 return DAG.getNode(X86ISD::VFPEXT, DL, VT,
15105 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
15106 In, DAG.getUNDEF(SVT)));
15109 /// The only differences between FABS and FNEG are the mask and the logic op.
15110 /// FNEG also has a folding opportunity for FNEG(FABS(x)).
15111 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
15112 assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&
15113 "Wrong opcode for lowering FABS or FNEG.");
15115 bool IsFABS = (Op.getOpcode() == ISD::FABS);
15117 // If this is a FABS and it has an FNEG user, bail out to fold the combination
15118 // into an FNABS. We'll lower the FABS after that if it is still in use.
15120 for (SDNode *User : Op->uses())
15121 if (User->getOpcode() == ISD::FNEG)
15124 SDValue Op0 = Op.getOperand(0);
15125 bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
15128 MVT VT = Op.getSimpleValueType();
15129 // Assume scalar op for initialization; update for vector if needed.
15130 // Note that there are no scalar bitwise logical SSE/AVX instructions, so we
15131 // generate a 16-byte vector constant and logic op even for the scalar case.
15132 // Using a 16-byte mask allows folding the load of the mask with
15133 // the logic op, so it can save (~4 bytes) on code size.
15135 unsigned NumElts = VT == MVT::f64 ? 2 : 4;
15136 // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOpt::Level to
15137 // decide if we should generate a 16-byte constant mask when we only need 4 or
15138 // 8 bytes for the scalar case.
15139 if (VT.isVector()) {
15140 EltVT = VT.getVectorElementType();
15141 NumElts = VT.getVectorNumElements();
15144 unsigned EltBits = EltVT.getSizeInBits();
15145 LLVMContext *Context = DAG.getContext();
15146 // For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
15148 IsFABS ? APInt::getSignedMaxValue(EltBits) : APInt::getSignBit(EltBits);
15149 Constant *C = ConstantInt::get(*Context, MaskElt);
15150 C = ConstantVector::getSplat(NumElts, C);
15151 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15152 SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy());
15153 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
15154 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
15155 MachinePointerInfo::getConstantPool(),
15156 false, false, false, Alignment);
15158 if (VT.isVector()) {
15159 // For a vector, cast operands to a vector type, perform the logic op,
15160 // and cast the result back to the original value type.
15161 MVT VecVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64);
15162 SDValue MaskCasted = DAG.getNode(ISD::BITCAST, dl, VecVT, Mask);
15163 SDValue Operand = IsFNABS ?
15164 DAG.getNode(ISD::BITCAST, dl, VecVT, Op0.getOperand(0)) :
15165 DAG.getNode(ISD::BITCAST, dl, VecVT, Op0);
15166 unsigned BitOp = IsFABS ? ISD::AND : IsFNABS ? ISD::OR : ISD::XOR;
15167 return DAG.getNode(ISD::BITCAST, dl, VT,
15168 DAG.getNode(BitOp, dl, VecVT, Operand, MaskCasted));
15171 // If not vector, then scalar.
15172 unsigned BitOp = IsFABS ? X86ISD::FAND : IsFNABS ? X86ISD::FOR : X86ISD::FXOR;
15173 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
15174 return DAG.getNode(BitOp, dl, VT, Operand, Mask);
15177 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
15178 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15179 LLVMContext *Context = DAG.getContext();
15180 SDValue Op0 = Op.getOperand(0);
15181 SDValue Op1 = Op.getOperand(1);
15183 MVT VT = Op.getSimpleValueType();
15184 MVT SrcVT = Op1.getSimpleValueType();
15186 // If second operand is smaller, extend it first.
15187 if (SrcVT.bitsLT(VT)) {
15188 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1);
15191 // And if it is bigger, shrink it first.
15192 if (SrcVT.bitsGT(VT)) {
15193 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
15197 // At this point the operands and the result should have the same
15198 // type, and that won't be f80 since that is not custom lowered.
15200 const fltSemantics &Sem =
15201 VT == MVT::f64 ? APFloat::IEEEdouble : APFloat::IEEEsingle;
15202 const unsigned SizeInBits = VT.getSizeInBits();
15204 SmallVector<Constant *, 4> CV(
15205 VT == MVT::f64 ? 2 : 4,
15206 ConstantFP::get(*Context, APFloat(Sem, APInt(SizeInBits, 0))));
15208 // First, clear all bits but the sign bit from the second operand (sign).
15209 CV[0] = ConstantFP::get(*Context,
15210 APFloat(Sem, APInt::getHighBitsSet(SizeInBits, 1)));
15211 Constant *C = ConstantVector::get(CV);
15212 SDValue CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
15213 SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx,
15214 MachinePointerInfo::getConstantPool(),
15215 false, false, false, 16);
15216 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
15218 // Next, clear the sign bit from the first operand (magnitude).
15219 // If it's a constant, we can clear it here.
15220 if (ConstantFPSDNode *Op0CN = dyn_cast<ConstantFPSDNode>(Op0)) {
15221 APFloat APF = Op0CN->getValueAPF();
15222 // If the magnitude is a positive zero, the sign bit alone is enough.
15223 if (APF.isPosZero())
15226 CV[0] = ConstantFP::get(*Context, APF);
15228 CV[0] = ConstantFP::get(
15230 APFloat(Sem, APInt::getLowBitsSet(SizeInBits, SizeInBits - 1)));
15232 C = ConstantVector::get(CV);
15233 CPIdx = DAG.getConstantPool(C, TLI.getPointerTy(), 16);
15234 SDValue Val = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
15235 MachinePointerInfo::getConstantPool(),
15236 false, false, false, 16);
15237 // If the magnitude operand wasn't a constant, we need to AND out the sign.
15238 if (!isa<ConstantFPSDNode>(Op0))
15239 Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Val);
15241 // OR the magnitude value with the sign bit.
15242 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
15245 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
15246 SDValue N0 = Op.getOperand(0);
15248 MVT VT = Op.getSimpleValueType();
15250 // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1).
15251 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
15252 DAG.getConstant(1, VT));
15253 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT));
15256 // Check whether an OR'd tree is PTEST-able.
15257 static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget *Subtarget,
15258 SelectionDAG &DAG) {
15259 assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
15261 if (!Subtarget->hasSSE41())
15264 if (!Op->hasOneUse())
15267 SDNode *N = Op.getNode();
15270 SmallVector<SDValue, 8> Opnds;
15271 DenseMap<SDValue, unsigned> VecInMap;
15272 SmallVector<SDValue, 8> VecIns;
15273 EVT VT = MVT::Other;
15275 // Recognize a special case where a vector is casted into wide integer to
15277 Opnds.push_back(N->getOperand(0));
15278 Opnds.push_back(N->getOperand(1));
15280 for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
15281 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
15282 // BFS traverse all OR'd operands.
15283 if (I->getOpcode() == ISD::OR) {
15284 Opnds.push_back(I->getOperand(0));
15285 Opnds.push_back(I->getOperand(1));
15286 // Re-evaluate the number of nodes to be traversed.
15287 e += 2; // 2 more nodes (LHS and RHS) are pushed.
15291 // Quit if a non-EXTRACT_VECTOR_ELT
15292 if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
15295 // Quit if without a constant index.
15296 SDValue Idx = I->getOperand(1);
15297 if (!isa<ConstantSDNode>(Idx))
15300 SDValue ExtractedFromVec = I->getOperand(0);
15301 DenseMap<SDValue, unsigned>::iterator M = VecInMap.find(ExtractedFromVec);
15302 if (M == VecInMap.end()) {
15303 VT = ExtractedFromVec.getValueType();
15304 // Quit if not 128/256-bit vector.
15305 if (!VT.is128BitVector() && !VT.is256BitVector())
15307 // Quit if not the same type.
15308 if (VecInMap.begin() != VecInMap.end() &&
15309 VT != VecInMap.begin()->first.getValueType())
15311 M = VecInMap.insert(std::make_pair(ExtractedFromVec, 0)).first;
15312 VecIns.push_back(ExtractedFromVec);
15314 M->second |= 1U << cast<ConstantSDNode>(Idx)->getZExtValue();
15317 assert((VT.is128BitVector() || VT.is256BitVector()) &&
15318 "Not extracted from 128-/256-bit vector.");
15320 unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U;
15322 for (DenseMap<SDValue, unsigned>::const_iterator
15323 I = VecInMap.begin(), E = VecInMap.end(); I != E; ++I) {
15324 // Quit if not all elements are used.
15325 if (I->second != FullMask)
15329 EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
15331 // Cast all vectors into TestVT for PTEST.
15332 for (unsigned i = 0, e = VecIns.size(); i < e; ++i)
15333 VecIns[i] = DAG.getNode(ISD::BITCAST, DL, TestVT, VecIns[i]);
15335 // If more than one full vectors are evaluated, OR them first before PTEST.
15336 for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) {
15337 // Each iteration will OR 2 nodes and append the result until there is only
15338 // 1 node left, i.e. the final OR'd value of all vectors.
15339 SDValue LHS = VecIns[Slot];
15340 SDValue RHS = VecIns[Slot + 1];
15341 VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
15344 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
15345 VecIns.back(), VecIns.back());
15348 /// \brief return true if \c Op has a use that doesn't just read flags.
15349 static bool hasNonFlagsUse(SDValue Op) {
15350 for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
15352 SDNode *User = *UI;
15353 unsigned UOpNo = UI.getOperandNo();
15354 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
15355 // Look pass truncate.
15356 UOpNo = User->use_begin().getOperandNo();
15357 User = *User->use_begin();
15360 if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
15361 !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
15367 /// Emit nodes that will be selected as "test Op0,Op0", or something
15369 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, SDLoc dl,
15370 SelectionDAG &DAG) const {
15371 if (Op.getValueType() == MVT::i1) {
15372 SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op);
15373 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp,
15374 DAG.getConstant(0, MVT::i8));
15376 // CF and OF aren't always set the way we want. Determine which
15377 // of these we need.
15378 bool NeedCF = false;
15379 bool NeedOF = false;
15382 case X86::COND_A: case X86::COND_AE:
15383 case X86::COND_B: case X86::COND_BE:
15386 case X86::COND_G: case X86::COND_GE:
15387 case X86::COND_L: case X86::COND_LE:
15388 case X86::COND_O: case X86::COND_NO: {
15389 // Check if we really need to set the
15390 // Overflow flag. If NoSignedWrap is present
15391 // that is not actually needed.
15392 switch (Op->getOpcode()) {
15397 const BinaryWithFlagsSDNode *BinNode =
15398 cast<BinaryWithFlagsSDNode>(Op.getNode());
15399 if (BinNode->hasNoSignedWrap())
15409 // See if we can use the EFLAGS value from the operand instead of
15410 // doing a separate TEST. TEST always sets OF and CF to 0, so unless
15411 // we prove that the arithmetic won't overflow, we can't use OF or CF.
15412 if (Op.getResNo() != 0 || NeedOF || NeedCF) {
15413 // Emit a CMP with 0, which is the TEST pattern.
15414 //if (Op.getValueType() == MVT::i1)
15415 // return DAG.getNode(X86ISD::CMP, dl, MVT::i1, Op,
15416 // DAG.getConstant(0, MVT::i1));
15417 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
15418 DAG.getConstant(0, Op.getValueType()));
15420 unsigned Opcode = 0;
15421 unsigned NumOperands = 0;
15423 // Truncate operations may prevent the merge of the SETCC instruction
15424 // and the arithmetic instruction before it. Attempt to truncate the operands
15425 // of the arithmetic instruction and use a reduced bit-width instruction.
15426 bool NeedTruncation = false;
15427 SDValue ArithOp = Op;
15428 if (Op->getOpcode() == ISD::TRUNCATE && Op->hasOneUse()) {
15429 SDValue Arith = Op->getOperand(0);
15430 // Both the trunc and the arithmetic op need to have one user each.
15431 if (Arith->hasOneUse())
15432 switch (Arith.getOpcode()) {
15439 NeedTruncation = true;
15445 // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
15446 // which may be the result of a CAST. We use the variable 'Op', which is the
15447 // non-casted variable when we check for possible users.
15448 switch (ArithOp.getOpcode()) {
15450 // Due to an isel shortcoming, be conservative if this add is likely to be
15451 // selected as part of a load-modify-store instruction. When the root node
15452 // in a match is a store, isel doesn't know how to remap non-chain non-flag
15453 // uses of other nodes in the match, such as the ADD in this case. This
15454 // leads to the ADD being left around and reselected, with the result being
15455 // two adds in the output. Alas, even if none our users are stores, that
15456 // doesn't prove we're O.K. Ergo, if we have any parents that aren't
15457 // CopyToReg or SETCC, eschew INC/DEC. A better fix seems to require
15458 // climbing the DAG back to the root, and it doesn't seem to be worth the
15460 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
15461 UE = Op.getNode()->use_end(); UI != UE; ++UI)
15462 if (UI->getOpcode() != ISD::CopyToReg &&
15463 UI->getOpcode() != ISD::SETCC &&
15464 UI->getOpcode() != ISD::STORE)
15467 if (ConstantSDNode *C =
15468 dyn_cast<ConstantSDNode>(ArithOp.getNode()->getOperand(1))) {
15469 // An add of one will be selected as an INC.
15470 if (C->getAPIntValue() == 1 && !Subtarget->slowIncDec()) {
15471 Opcode = X86ISD::INC;
15476 // An add of negative one (subtract of one) will be selected as a DEC.
15477 if (C->getAPIntValue().isAllOnesValue() && !Subtarget->slowIncDec()) {
15478 Opcode = X86ISD::DEC;
15484 // Otherwise use a regular EFLAGS-setting add.
15485 Opcode = X86ISD::ADD;
15490 // If we have a constant logical shift that's only used in a comparison
15491 // against zero turn it into an equivalent AND. This allows turning it into
15492 // a TEST instruction later.
15493 if ((X86CC == X86::COND_E || X86CC == X86::COND_NE) && Op->hasOneUse() &&
15494 isa<ConstantSDNode>(Op->getOperand(1)) && !hasNonFlagsUse(Op)) {
15495 EVT VT = Op.getValueType();
15496 unsigned BitWidth = VT.getSizeInBits();
15497 unsigned ShAmt = Op->getConstantOperandVal(1);
15498 if (ShAmt >= BitWidth) // Avoid undefined shifts.
15500 APInt Mask = ArithOp.getOpcode() == ISD::SRL
15501 ? APInt::getHighBitsSet(BitWidth, BitWidth - ShAmt)
15502 : APInt::getLowBitsSet(BitWidth, BitWidth - ShAmt);
15503 if (!Mask.isSignedIntN(32)) // Avoid large immediates.
15505 SDValue New = DAG.getNode(ISD::AND, dl, VT, Op->getOperand(0),
15506 DAG.getConstant(Mask, VT));
15507 DAG.ReplaceAllUsesWith(Op, New);
15513 // If the primary and result isn't used, don't bother using X86ISD::AND,
15514 // because a TEST instruction will be better.
15515 if (!hasNonFlagsUse(Op))
15521 // Due to the ISEL shortcoming noted above, be conservative if this op is
15522 // likely to be selected as part of a load-modify-store instruction.
15523 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
15524 UE = Op.getNode()->use_end(); UI != UE; ++UI)
15525 if (UI->getOpcode() == ISD::STORE)
15528 // Otherwise use a regular EFLAGS-setting instruction.
15529 switch (ArithOp.getOpcode()) {
15530 default: llvm_unreachable("unexpected operator!");
15531 case ISD::SUB: Opcode = X86ISD::SUB; break;
15532 case ISD::XOR: Opcode = X86ISD::XOR; break;
15533 case ISD::AND: Opcode = X86ISD::AND; break;
15535 if (!NeedTruncation && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
15536 SDValue EFLAGS = LowerVectorAllZeroTest(Op, Subtarget, DAG);
15537 if (EFLAGS.getNode())
15540 Opcode = X86ISD::OR;
15554 return SDValue(Op.getNode(), 1);
15560 // If we found that truncation is beneficial, perform the truncation and
15562 if (NeedTruncation) {
15563 EVT VT = Op.getValueType();
15564 SDValue WideVal = Op->getOperand(0);
15565 EVT WideVT = WideVal.getValueType();
15566 unsigned ConvertedOp = 0;
15567 // Use a target machine opcode to prevent further DAGCombine
15568 // optimizations that may separate the arithmetic operations
15569 // from the setcc node.
15570 switch (WideVal.getOpcode()) {
15572 case ISD::ADD: ConvertedOp = X86ISD::ADD; break;
15573 case ISD::SUB: ConvertedOp = X86ISD::SUB; break;
15574 case ISD::AND: ConvertedOp = X86ISD::AND; break;
15575 case ISD::OR: ConvertedOp = X86ISD::OR; break;
15576 case ISD::XOR: ConvertedOp = X86ISD::XOR; break;
15580 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15581 if (TLI.isOperationLegal(WideVal.getOpcode(), WideVT)) {
15582 SDValue V0 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(0));
15583 SDValue V1 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(1));
15584 Op = DAG.getNode(ConvertedOp, dl, VT, V0, V1);
15590 // Emit a CMP with 0, which is the TEST pattern.
15591 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
15592 DAG.getConstant(0, Op.getValueType()));
15594 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
15595 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
15597 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
15598 DAG.ReplaceAllUsesWith(Op, New);
15599 return SDValue(New.getNode(), 1);
15602 /// Emit nodes that will be selected as "cmp Op0,Op1", or something
15604 SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
15605 SDLoc dl, SelectionDAG &DAG) const {
15606 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1)) {
15607 if (C->getAPIntValue() == 0)
15608 return EmitTest(Op0, X86CC, dl, DAG);
15610 if (Op0.getValueType() == MVT::i1)
15611 llvm_unreachable("Unexpected comparison operation for MVT::i1 operands");
15614 if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 ||
15615 Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) {
15616 // Do the comparison at i32 if it's smaller, besides the Atom case.
15617 // This avoids subregister aliasing issues. Keep the smaller reference
15618 // if we're optimizing for size, however, as that'll allow better folding
15619 // of memory operations.
15620 if (Op0.getValueType() != MVT::i32 && Op0.getValueType() != MVT::i64 &&
15621 !DAG.getMachineFunction().getFunction()->hasFnAttribute(
15622 Attribute::MinSize) &&
15623 !Subtarget->isAtom()) {
15624 unsigned ExtendOp =
15625 isX86CCUnsigned(X86CC) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
15626 Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0);
15627 Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1);
15629 // Use SUB instead of CMP to enable CSE between SUB and CMP.
15630 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
15631 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs,
15633 return SDValue(Sub.getNode(), 1);
15635 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
15638 /// Convert a comparison if required by the subtarget.
15639 SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
15640 SelectionDAG &DAG) const {
15641 // If the subtarget does not support the FUCOMI instruction, floating-point
15642 // comparisons have to be converted.
15643 if (Subtarget->hasCMov() ||
15644 Cmp.getOpcode() != X86ISD::CMP ||
15645 !Cmp.getOperand(0).getValueType().isFloatingPoint() ||
15646 !Cmp.getOperand(1).getValueType().isFloatingPoint())
15649 // The instruction selector will select an FUCOM instruction instead of
15650 // FUCOMI, which writes the comparison result to FPSW instead of EFLAGS. Hence
15651 // build an SDNode sequence that transfers the result from FPSW into EFLAGS:
15652 // (X86sahf (trunc (srl (X86fp_stsw (trunc (X86cmp ...)), 8))))
15654 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp);
15655 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
15656 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW,
15657 DAG.getConstant(8, MVT::i8));
15658 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl);
15659 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
15662 /// The minimum architected relative accuracy is 2^-12. We need one
15663 /// Newton-Raphson step to have a good float result (24 bits of precision).
15664 SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op,
15665 DAGCombinerInfo &DCI,
15666 unsigned &RefinementSteps,
15667 bool &UseOneConstNR) const {
15668 // FIXME: We should use instruction latency models to calculate the cost of
15669 // each potential sequence, but this is very hard to do reliably because
15670 // at least Intel's Core* chips have variable timing based on the number of
15671 // significant digits in the divisor and/or sqrt operand.
15672 if (!Subtarget->useSqrtEst())
15675 EVT VT = Op.getValueType();
15677 // SSE1 has rsqrtss and rsqrtps.
15678 // TODO: Add support for AVX512 (v16f32).
15679 // It is likely not profitable to do this for f64 because a double-precision
15680 // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
15681 // instructions: convert to single, rsqrtss, convert back to double, refine
15682 // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
15683 // along with FMA, this could be a throughput win.
15684 if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||
15685 (Subtarget->hasAVX() && VT == MVT::v8f32)) {
15686 RefinementSteps = 1;
15687 UseOneConstNR = false;
15688 return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
15693 /// The minimum architected relative accuracy is 2^-12. We need one
15694 /// Newton-Raphson step to have a good float result (24 bits of precision).
15695 SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
15696 DAGCombinerInfo &DCI,
15697 unsigned &RefinementSteps) const {
15698 // FIXME: We should use instruction latency models to calculate the cost of
15699 // each potential sequence, but this is very hard to do reliably because
15700 // at least Intel's Core* chips have variable timing based on the number of
15701 // significant digits in the divisor.
15702 if (!Subtarget->useReciprocalEst())
15705 EVT VT = Op.getValueType();
15707 // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
15708 // TODO: Add support for AVX512 (v16f32).
15709 // It is likely not profitable to do this for f64 because a double-precision
15710 // reciprocal estimate with refinement on x86 prior to FMA requires
15711 // 15 instructions: convert to single, rcpss, convert back to double, refine
15712 // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
15713 // along with FMA, this could be a throughput win.
15714 if ((Subtarget->hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||
15715 (Subtarget->hasAVX() && VT == MVT::v8f32)) {
15716 RefinementSteps = ReciprocalEstimateRefinementSteps;
15717 return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
15722 static bool isAllOnes(SDValue V) {
15723 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
15724 return C && C->isAllOnesValue();
15727 /// LowerToBT - Result of 'and' is compared against zero. Turn it into a BT node
15728 /// if it's possible.
15729 SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
15730 SDLoc dl, SelectionDAG &DAG) const {
15731 SDValue Op0 = And.getOperand(0);
15732 SDValue Op1 = And.getOperand(1);
15733 if (Op0.getOpcode() == ISD::TRUNCATE)
15734 Op0 = Op0.getOperand(0);
15735 if (Op1.getOpcode() == ISD::TRUNCATE)
15736 Op1 = Op1.getOperand(0);
15739 if (Op1.getOpcode() == ISD::SHL)
15740 std::swap(Op0, Op1);
15741 if (Op0.getOpcode() == ISD::SHL) {
15742 if (ConstantSDNode *And00C = dyn_cast<ConstantSDNode>(Op0.getOperand(0)))
15743 if (And00C->getZExtValue() == 1) {
15744 // If we looked past a truncate, check that it's only truncating away
15746 unsigned BitWidth = Op0.getValueSizeInBits();
15747 unsigned AndBitWidth = And.getValueSizeInBits();
15748 if (BitWidth > AndBitWidth) {
15750 DAG.computeKnownBits(Op0, Zeros, Ones);
15751 if (Zeros.countLeadingOnes() < BitWidth - AndBitWidth)
15755 RHS = Op0.getOperand(1);
15757 } else if (Op1.getOpcode() == ISD::Constant) {
15758 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
15759 uint64_t AndRHSVal = AndRHS->getZExtValue();
15760 SDValue AndLHS = Op0;
15762 if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
15763 LHS = AndLHS.getOperand(0);
15764 RHS = AndLHS.getOperand(1);
15767 // Use BT if the immediate can't be encoded in a TEST instruction.
15768 if (!isUInt<32>(AndRHSVal) && isPowerOf2_64(AndRHSVal)) {
15770 RHS = DAG.getConstant(Log2_64_Ceil(AndRHSVal), LHS.getValueType());
15774 if (LHS.getNode()) {
15775 // If LHS is i8, promote it to i32 with any_extend. There is no i8 BT
15776 // instruction. Since the shift amount is in-range-or-undefined, we know
15777 // that doing a bittest on the i32 value is ok. We extend to i32 because
15778 // the encoding for the i16 version is larger than the i32 version.
15779 // Also promote i16 to i32 for performance / code size reason.
15780 if (LHS.getValueType() == MVT::i8 ||
15781 LHS.getValueType() == MVT::i16)
15782 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
15784 // If the operand types disagree, extend the shift amount to match. Since
15785 // BT ignores high bits (like shifts) we can use anyextend.
15786 if (LHS.getValueType() != RHS.getValueType())
15787 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS);
15789 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
15790 X86::CondCode Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
15791 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
15792 DAG.getConstant(Cond, MVT::i8), BT);
15798 /// \brief - Turns an ISD::CondCode into a value suitable for SSE floating point
15800 static int translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
15805 // SSE Condition code mapping:
15814 switch (SetCCOpcode) {
15815 default: llvm_unreachable("Unexpected SETCC condition");
15817 case ISD::SETEQ: SSECC = 0; break;
15819 case ISD::SETGT: Swap = true; // Fallthrough
15821 case ISD::SETOLT: SSECC = 1; break;
15823 case ISD::SETGE: Swap = true; // Fallthrough
15825 case ISD::SETOLE: SSECC = 2; break;
15826 case ISD::SETUO: SSECC = 3; break;
15828 case ISD::SETNE: SSECC = 4; break;
15829 case ISD::SETULE: Swap = true; // Fallthrough
15830 case ISD::SETUGE: SSECC = 5; break;
15831 case ISD::SETULT: Swap = true; // Fallthrough
15832 case ISD::SETUGT: SSECC = 6; break;
15833 case ISD::SETO: SSECC = 7; break;
15835 case ISD::SETONE: SSECC = 8; break;
15838 std::swap(Op0, Op1);
15843 // Lower256IntVSETCC - Break a VSETCC 256-bit integer VSETCC into two new 128
15844 // ones, and then concatenate the result back.
15845 static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
15846 MVT VT = Op.getSimpleValueType();
15848 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&
15849 "Unsupported value type for operation");
15851 unsigned NumElems = VT.getVectorNumElements();
15853 SDValue CC = Op.getOperand(2);
15855 // Extract the LHS vectors
15856 SDValue LHS = Op.getOperand(0);
15857 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
15858 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
15860 // Extract the RHS vectors
15861 SDValue RHS = Op.getOperand(1);
15862 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl);
15863 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl);
15865 // Issue the operation on the smaller types and concatenate the result back
15866 MVT EltVT = VT.getVectorElementType();
15867 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
15868 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
15869 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
15870 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
15873 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG,
15874 const X86Subtarget *Subtarget) {
15875 SDValue Op0 = Op.getOperand(0);
15876 SDValue Op1 = Op.getOperand(1);
15877 SDValue CC = Op.getOperand(2);
15878 MVT VT = Op.getSimpleValueType();
15881 assert(Op0.getValueType().getVectorElementType().getSizeInBits() >= 8 &&
15882 Op.getValueType().getScalarType() == MVT::i1 &&
15883 "Cannot set masked compare for this operation");
15885 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
15887 bool Unsigned = false;
15890 switch (SetCCOpcode) {
15891 default: llvm_unreachable("Unexpected SETCC condition");
15892 case ISD::SETNE: SSECC = 4; break;
15893 case ISD::SETEQ: Opc = X86ISD::PCMPEQM; break;
15894 case ISD::SETUGT: SSECC = 6; Unsigned = true; break;
15895 case ISD::SETLT: Swap = true; //fall-through
15896 case ISD::SETGT: Opc = X86ISD::PCMPGTM; break;
15897 case ISD::SETULT: SSECC = 1; Unsigned = true; break;
15898 case ISD::SETUGE: SSECC = 5; Unsigned = true; break; //NLT
15899 case ISD::SETGE: Swap = true; SSECC = 2; break; // LE + swap
15900 case ISD::SETULE: Unsigned = true; //fall-through
15901 case ISD::SETLE: SSECC = 2; break;
15905 std::swap(Op0, Op1);
15907 return DAG.getNode(Opc, dl, VT, Op0, Op1);
15908 Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM;
15909 return DAG.getNode(Opc, dl, VT, Op0, Op1,
15910 DAG.getConstant(SSECC, MVT::i8));
15913 /// \brief Try to turn a VSETULT into a VSETULE by modifying its second
15914 /// operand \p Op1. If non-trivial (for example because it's not constant)
15915 /// return an empty value.
15916 static SDValue ChangeVSETULTtoVSETULE(SDLoc dl, SDValue Op1, SelectionDAG &DAG)
15918 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op1.getNode());
15922 MVT VT = Op1.getSimpleValueType();
15923 MVT EVT = VT.getVectorElementType();
15924 unsigned n = VT.getVectorNumElements();
15925 SmallVector<SDValue, 8> ULTOp1;
15927 for (unsigned i = 0; i < n; ++i) {
15928 ConstantSDNode *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
15929 if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT)
15932 // Avoid underflow.
15933 APInt Val = Elt->getAPIntValue();
15937 ULTOp1.push_back(DAG.getConstant(Val - 1, EVT));
15940 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, ULTOp1);
15943 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
15944 SelectionDAG &DAG) {
15945 SDValue Op0 = Op.getOperand(0);
15946 SDValue Op1 = Op.getOperand(1);
15947 SDValue CC = Op.getOperand(2);
15948 MVT VT = Op.getSimpleValueType();
15949 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
15950 bool isFP = Op.getOperand(1).getSimpleValueType().isFloatingPoint();
15955 MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
15956 assert(EltVT == MVT::f32 || EltVT == MVT::f64);
15959 unsigned SSECC = translateX86FSETCC(SetCCOpcode, Op0, Op1);
15960 unsigned Opc = X86ISD::CMPP;
15961 if (Subtarget->hasAVX512() && VT.getVectorElementType() == MVT::i1) {
15962 assert(VT.getVectorNumElements() <= 16);
15963 Opc = X86ISD::CMPM;
15965 // In the two special cases we can't handle, emit two comparisons.
15968 unsigned CombineOpc;
15969 if (SetCCOpcode == ISD::SETUEQ) {
15970 CC0 = 3; CC1 = 0; CombineOpc = ISD::OR;
15972 assert(SetCCOpcode == ISD::SETONE);
15973 CC0 = 7; CC1 = 4; CombineOpc = ISD::AND;
15976 SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
15977 DAG.getConstant(CC0, MVT::i8));
15978 SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
15979 DAG.getConstant(CC1, MVT::i8));
15980 return DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
15982 // Handle all other FP comparisons here.
15983 return DAG.getNode(Opc, dl, VT, Op0, Op1,
15984 DAG.getConstant(SSECC, MVT::i8));
15987 // Break 256-bit integer vector compare into smaller ones.
15988 if (VT.is256BitVector() && !Subtarget->hasInt256())
15989 return Lower256IntVSETCC(Op, DAG);
15991 bool MaskResult = (VT.getVectorElementType() == MVT::i1);
15992 EVT OpVT = Op1.getValueType();
15993 if (Subtarget->hasAVX512()) {
15994 if (Op1.getValueType().is512BitVector() ||
15995 (Subtarget->hasBWI() && Subtarget->hasVLX()) ||
15996 (MaskResult && OpVT.getVectorElementType().getSizeInBits() >= 32))
15997 return LowerIntVSETCC_AVX512(Op, DAG, Subtarget);
15999 // In AVX-512 architecture setcc returns mask with i1 elements,
16000 // But there is no compare instruction for i8 and i16 elements in KNL.
16001 // We are not talking about 512-bit operands in this case, these
16002 // types are illegal.
16004 (OpVT.getVectorElementType().getSizeInBits() < 32 &&
16005 OpVT.getVectorElementType().getSizeInBits() >= 8))
16006 return DAG.getNode(ISD::TRUNCATE, dl, VT,
16007 DAG.getNode(ISD::SETCC, dl, OpVT, Op0, Op1, CC));
16010 // We are handling one of the integer comparisons here. Since SSE only has
16011 // GT and EQ comparisons for integer, swapping operands and multiple
16012 // operations may be required for some comparisons.
16014 bool Swap = false, Invert = false, FlipSigns = false, MinMax = false;
16015 bool Subus = false;
16017 switch (SetCCOpcode) {
16018 default: llvm_unreachable("Unexpected SETCC condition");
16019 case ISD::SETNE: Invert = true;
16020 case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break;
16021 case ISD::SETLT: Swap = true;
16022 case ISD::SETGT: Opc = X86ISD::PCMPGT; break;
16023 case ISD::SETGE: Swap = true;
16024 case ISD::SETLE: Opc = X86ISD::PCMPGT;
16025 Invert = true; break;
16026 case ISD::SETULT: Swap = true;
16027 case ISD::SETUGT: Opc = X86ISD::PCMPGT;
16028 FlipSigns = true; break;
16029 case ISD::SETUGE: Swap = true;
16030 case ISD::SETULE: Opc = X86ISD::PCMPGT;
16031 FlipSigns = true; Invert = true; break;
16034 // Special case: Use min/max operations for SETULE/SETUGE
16035 MVT VET = VT.getVectorElementType();
16037 (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32))
16038 || (Subtarget->hasSSE2() && (VET == MVT::i8));
16041 switch (SetCCOpcode) {
16043 case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break;
16044 case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break;
16047 if (MinMax) { Swap = false; Invert = false; FlipSigns = false; }
16050 bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16);
16051 if (!MinMax && hasSubus) {
16052 // As another special case, use PSUBUS[BW] when it's profitable. E.g. for
16054 // t = psubus Op0, Op1
16055 // pcmpeq t, <0..0>
16056 switch (SetCCOpcode) {
16058 case ISD::SETULT: {
16059 // If the comparison is against a constant we can turn this into a
16060 // setule. With psubus, setule does not require a swap. This is
16061 // beneficial because the constant in the register is no longer
16062 // destructed as the destination so it can be hoisted out of a loop.
16063 // Only do this pre-AVX since vpcmp* is no longer destructive.
16064 if (Subtarget->hasAVX())
16066 SDValue ULEOp1 = ChangeVSETULTtoVSETULE(dl, Op1, DAG);
16067 if (ULEOp1.getNode()) {
16069 Subus = true; Invert = false; Swap = false;
16073 // Psubus is better than flip-sign because it requires no inversion.
16074 case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break;
16075 case ISD::SETULE: Subus = true; Invert = false; Swap = false; break;
16079 Opc = X86ISD::SUBUS;
16085 std::swap(Op0, Op1);
16087 // Check that the operation in question is available (most are plain SSE2,
16088 // but PCMPGTQ and PCMPEQQ have different requirements).
16089 if (VT == MVT::v2i64) {
16090 if (Opc == X86ISD::PCMPGT && !Subtarget->hasSSE42()) {
16091 assert(Subtarget->hasSSE2() && "Don't know how to lower!");
16093 // First cast everything to the right type.
16094 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
16095 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
16097 // Since SSE has no unsigned integer comparisons, we need to flip the sign
16098 // bits of the inputs before performing those operations. The lower
16099 // compare is always unsigned.
16102 SB = DAG.getConstant(0x80000000U, MVT::v4i32);
16104 SDValue Sign = DAG.getConstant(0x80000000U, MVT::i32);
16105 SDValue Zero = DAG.getConstant(0x00000000U, MVT::i32);
16106 SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32,
16107 Sign, Zero, Sign, Zero);
16109 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB);
16110 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB);
16112 // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
16113 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
16114 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
16116 // Create masks for only the low parts/high parts of the 64 bit integers.
16117 static const int MaskHi[] = { 1, 1, 3, 3 };
16118 static const int MaskLo[] = { 0, 0, 2, 2 };
16119 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
16120 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
16121 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
16123 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
16124 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
16127 Result = DAG.getNOT(dl, Result, MVT::v4i32);
16129 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
16132 if (Opc == X86ISD::PCMPEQ && !Subtarget->hasSSE41()) {
16133 // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
16134 // pcmpeqd + pshufd + pand.
16135 assert(Subtarget->hasSSE2() && !FlipSigns && "Don't know how to lower!");
16137 // First cast everything to the right type.
16138 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0);
16139 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1);
16142 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
16144 // Make sure the lower and upper halves are both all-ones.
16145 static const int Mask[] = { 1, 0, 3, 2 };
16146 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
16147 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
16150 Result = DAG.getNOT(dl, Result, MVT::v4i32);
16152 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
16156 // Since SSE has no unsigned integer comparisons, we need to flip the sign
16157 // bits of the inputs before performing those operations.
16159 EVT EltVT = VT.getVectorElementType();
16160 SDValue SB = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()), VT);
16161 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SB);
16162 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SB);
16165 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
16167 // If the logical-not of the result is required, perform that now.
16169 Result = DAG.getNOT(dl, Result, VT);
16172 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
16175 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
16176 getZeroVector(VT, Subtarget, DAG, dl));
16181 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
16183 MVT VT = Op.getSimpleValueType();
16185 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
16187 assert(((!Subtarget->hasAVX512() && VT == MVT::i8) || (VT == MVT::i1))
16188 && "SetCC type must be 8-bit or 1-bit integer");
16189 SDValue Op0 = Op.getOperand(0);
16190 SDValue Op1 = Op.getOperand(1);
16192 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
16194 // Optimize to BT if possible.
16195 // Lower (X & (1 << N)) == 0 to BT(X, N).
16196 // Lower ((X >>u N) & 1) != 0 to BT(X, N).
16197 // Lower ((X >>s N) & 1) != 0 to BT(X, N).
16198 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() &&
16199 Op1.getOpcode() == ISD::Constant &&
16200 cast<ConstantSDNode>(Op1)->isNullValue() &&
16201 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
16202 SDValue NewSetCC = LowerToBT(Op0, CC, dl, DAG);
16203 if (NewSetCC.getNode()) {
16205 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewSetCC);
16210 // Look for X == 0, X == 1, X != 0, or X != 1. We can simplify some forms of
16212 if (Op1.getOpcode() == ISD::Constant &&
16213 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1 ||
16214 cast<ConstantSDNode>(Op1)->isNullValue()) &&
16215 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
16217 // If the input is a setcc, then reuse the input setcc or use a new one with
16218 // the inverted condition.
16219 if (Op0.getOpcode() == X86ISD::SETCC) {
16220 X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
16221 bool Invert = (CC == ISD::SETNE) ^
16222 cast<ConstantSDNode>(Op1)->isNullValue();
16226 CCode = X86::GetOppositeBranchCondition(CCode);
16227 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
16228 DAG.getConstant(CCode, MVT::i8),
16229 Op0.getOperand(1));
16231 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC);
16235 if ((Op0.getValueType() == MVT::i1) && (Op1.getOpcode() == ISD::Constant) &&
16236 (cast<ConstantSDNode>(Op1)->getZExtValue() == 1) &&
16237 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
16239 ISD::CondCode NewCC = ISD::getSetCCInverse(CC, true);
16240 return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, MVT::i1), NewCC);
16243 bool isFP = Op1.getSimpleValueType().isFloatingPoint();
16244 unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG);
16245 if (X86CC == X86::COND_INVALID)
16248 SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, dl, DAG);
16249 EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
16250 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
16251 DAG.getConstant(X86CC, MVT::i8), EFLAGS);
16253 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC);
16257 // isX86LogicalCmp - Return true if opcode is a X86 logical comparison.
16258 static bool isX86LogicalCmp(SDValue Op) {
16259 unsigned Opc = Op.getNode()->getOpcode();
16260 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
16261 Opc == X86ISD::SAHF)
16263 if (Op.getResNo() == 1 &&
16264 (Opc == X86ISD::ADD ||
16265 Opc == X86ISD::SUB ||
16266 Opc == X86ISD::ADC ||
16267 Opc == X86ISD::SBB ||
16268 Opc == X86ISD::SMUL ||
16269 Opc == X86ISD::UMUL ||
16270 Opc == X86ISD::INC ||
16271 Opc == X86ISD::DEC ||
16272 Opc == X86ISD::OR ||
16273 Opc == X86ISD::XOR ||
16274 Opc == X86ISD::AND))
16277 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
16283 static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
16284 if (V.getOpcode() != ISD::TRUNCATE)
16287 SDValue VOp0 = V.getOperand(0);
16288 unsigned InBits = VOp0.getValueSizeInBits();
16289 unsigned Bits = V.getValueSizeInBits();
16290 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
16293 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
16294 bool addTest = true;
16295 SDValue Cond = Op.getOperand(0);
16296 SDValue Op1 = Op.getOperand(1);
16297 SDValue Op2 = Op.getOperand(2);
16299 EVT VT = Op1.getValueType();
16302 // Lower fp selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
16303 // are available. Otherwise fp cmovs get lowered into a less efficient branch
16304 // sequence later on.
16305 if (Cond.getOpcode() == ISD::SETCC &&
16306 ((Subtarget->hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) ||
16307 (Subtarget->hasSSE1() && VT == MVT::f32)) &&
16308 VT == Cond.getOperand(0).getValueType() && Cond->hasOneUse()) {
16309 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
16310 int SSECC = translateX86FSETCC(
16311 cast<CondCodeSDNode>(Cond.getOperand(2))->get(), CondOp0, CondOp1);
16314 if (Subtarget->hasAVX512()) {
16315 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1,
16316 DAG.getConstant(SSECC, MVT::i8));
16317 return DAG.getNode(X86ISD::SELECT, DL, VT, Cmp, Op1, Op2);
16319 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
16320 DAG.getConstant(SSECC, MVT::i8));
16321 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
16322 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
16323 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
16327 if (Cond.getOpcode() == ISD::SETCC) {
16328 SDValue NewCond = LowerSETCC(Cond, DAG);
16329 if (NewCond.getNode())
16333 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y
16334 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y
16335 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y
16336 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y
16337 if (Cond.getOpcode() == X86ISD::SETCC &&
16338 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
16339 isZero(Cond.getOperand(1).getOperand(1))) {
16340 SDValue Cmp = Cond.getOperand(1);
16342 unsigned CondCode =cast<ConstantSDNode>(Cond.getOperand(0))->getZExtValue();
16344 if ((isAllOnes(Op1) || isAllOnes(Op2)) &&
16345 (CondCode == X86::COND_E || CondCode == X86::COND_NE)) {
16346 SDValue Y = isAllOnes(Op2) ? Op1 : Op2;
16348 SDValue CmpOp0 = Cmp.getOperand(0);
16349 // Apply further optimizations for special cases
16350 // (select (x != 0), -1, 0) -> neg & sbb
16351 // (select (x == 0), 0, -1) -> neg & sbb
16352 if (ConstantSDNode *YC = dyn_cast<ConstantSDNode>(Y))
16353 if (YC->isNullValue() &&
16354 (isAllOnes(Op1) == (CondCode == X86::COND_NE))) {
16355 SDVTList VTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);
16356 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, VTs,
16357 DAG.getConstant(0, CmpOp0.getValueType()),
16359 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
16360 DAG.getConstant(X86::COND_B, MVT::i8),
16361 SDValue(Neg.getNode(), 1));
16365 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
16366 CmpOp0, DAG.getConstant(1, CmpOp0.getValueType()));
16367 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
16369 SDValue Res = // Res = 0 or -1.
16370 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
16371 DAG.getConstant(X86::COND_B, MVT::i8), Cmp);
16373 if (isAllOnes(Op1) != (CondCode == X86::COND_E))
16374 Res = DAG.getNOT(DL, Res, Res.getValueType());
16376 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Op2);
16377 if (!N2C || !N2C->isNullValue())
16378 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y);
16383 // Look past (and (setcc_carry (cmp ...)), 1).
16384 if (Cond.getOpcode() == ISD::AND &&
16385 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
16386 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
16387 if (C && C->getAPIntValue() == 1)
16388 Cond = Cond.getOperand(0);
16391 // If condition flag is set by a X86ISD::CMP, then use it as the condition
16392 // setting operand in place of the X86ISD::SETCC.
16393 unsigned CondOpcode = Cond.getOpcode();
16394 if (CondOpcode == X86ISD::SETCC ||
16395 CondOpcode == X86ISD::SETCC_CARRY) {
16396 CC = Cond.getOperand(0);
16398 SDValue Cmp = Cond.getOperand(1);
16399 unsigned Opc = Cmp.getOpcode();
16400 MVT VT = Op.getSimpleValueType();
16402 bool IllegalFPCMov = false;
16403 if (VT.isFloatingPoint() && !VT.isVector() &&
16404 !isScalarFPTypeInSSEReg(VT)) // FPStack?
16405 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
16407 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
16408 Opc == X86ISD::BT) { // FIXME
16412 } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
16413 CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
16414 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) &&
16415 Cond.getOperand(0).getValueType() != MVT::i8)) {
16416 SDValue LHS = Cond.getOperand(0);
16417 SDValue RHS = Cond.getOperand(1);
16418 unsigned X86Opcode;
16421 switch (CondOpcode) {
16422 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
16423 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
16424 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
16425 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
16426 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
16427 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
16428 default: llvm_unreachable("unexpected overflowing operator");
16430 if (CondOpcode == ISD::UMULO)
16431 VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(),
16434 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
16436 SDValue X86Op = DAG.getNode(X86Opcode, DL, VTs, LHS, RHS);
16438 if (CondOpcode == ISD::UMULO)
16439 Cond = X86Op.getValue(2);
16441 Cond = X86Op.getValue(1);
16443 CC = DAG.getConstant(X86Cond, MVT::i8);
16448 // Look pass the truncate if the high bits are known zero.
16449 if (isTruncWithZeroHighBitsInput(Cond, DAG))
16450 Cond = Cond.getOperand(0);
16452 // We know the result of AND is compared against zero. Try to match
16454 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
16455 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, DL, DAG);
16456 if (NewSetCC.getNode()) {
16457 CC = NewSetCC.getOperand(0);
16458 Cond = NewSetCC.getOperand(1);
16465 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
16466 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG);
16469 // a < b ? -1 : 0 -> RES = ~setcc_carry
16470 // a < b ? 0 : -1 -> RES = setcc_carry
16471 // a >= b ? -1 : 0 -> RES = setcc_carry
16472 // a >= b ? 0 : -1 -> RES = ~setcc_carry
16473 if (Cond.getOpcode() == X86ISD::SUB) {
16474 Cond = ConvertCmpIfNecessary(Cond, DAG);
16475 unsigned CondCode = cast<ConstantSDNode>(CC)->getZExtValue();
16477 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
16478 (isAllOnes(Op1) || isAllOnes(Op2)) && (isZero(Op1) || isZero(Op2))) {
16479 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
16480 DAG.getConstant(X86::COND_B, MVT::i8), Cond);
16481 if (isAllOnes(Op1) != (CondCode == X86::COND_B))
16482 return DAG.getNOT(DL, Res, Res.getValueType());
16487 // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
16488 // widen the cmov and push the truncate through. This avoids introducing a new
16489 // branch during isel and doesn't add any extensions.
16490 if (Op.getValueType() == MVT::i8 &&
16491 Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
16492 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
16493 if (T1.getValueType() == T2.getValueType() &&
16494 // Blacklist CopyFromReg to avoid partial register stalls.
16495 T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
16496 SDVTList VTs = DAG.getVTList(T1.getValueType(), MVT::Glue);
16497 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VTs, T2, T1, CC, Cond);
16498 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
16502 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
16503 // condition is true.
16504 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
16505 SDValue Ops[] = { Op2, Op1, CC, Cond };
16506 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops);
16509 static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op, const X86Subtarget *Subtarget,
16510 SelectionDAG &DAG) {
16511 MVT VT = Op->getSimpleValueType(0);
16512 SDValue In = Op->getOperand(0);
16513 MVT InVT = In.getSimpleValueType();
16514 MVT VTElt = VT.getVectorElementType();
16515 MVT InVTElt = InVT.getVectorElementType();
16519 if ((InVTElt == MVT::i1) &&
16520 (((Subtarget->hasBWI() && Subtarget->hasVLX() &&
16521 VT.getSizeInBits() <= 256 && VTElt.getSizeInBits() <= 16)) ||
16523 ((Subtarget->hasBWI() && VT.is512BitVector() &&
16524 VTElt.getSizeInBits() <= 16)) ||
16526 ((Subtarget->hasDQI() && Subtarget->hasVLX() &&
16527 VT.getSizeInBits() <= 256 && VTElt.getSizeInBits() >= 32)) ||
16529 ((Subtarget->hasDQI() && VT.is512BitVector() &&
16530 VTElt.getSizeInBits() >= 32))))
16531 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16533 unsigned int NumElts = VT.getVectorNumElements();
16535 if (NumElts != 8 && NumElts != 16)
16538 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) {
16539 if (In.getOpcode() == X86ISD::VSEXT || In.getOpcode() == X86ISD::VZEXT)
16540 return DAG.getNode(In.getOpcode(), dl, VT, In.getOperand(0));
16541 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16544 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16545 assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type");
16547 MVT ExtVT = (NumElts == 8) ? MVT::v8i64 : MVT::v16i32;
16548 Constant *C = ConstantInt::get(*DAG.getContext(),
16549 APInt::getAllOnesValue(ExtVT.getScalarType().getSizeInBits()));
16551 SDValue CP = DAG.getConstantPool(C, TLI.getPointerTy());
16552 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
16553 SDValue Ld = DAG.getLoad(ExtVT.getScalarType(), dl, DAG.getEntryNode(), CP,
16554 MachinePointerInfo::getConstantPool(),
16555 false, false, false, Alignment);
16556 SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, dl, ExtVT, In, Ld);
16557 if (VT.is512BitVector())
16559 return DAG.getNode(X86ISD::VTRUNC, dl, VT, Brcst);
16562 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget *Subtarget,
16563 SelectionDAG &DAG) {
16564 MVT VT = Op->getSimpleValueType(0);
16565 SDValue In = Op->getOperand(0);
16566 MVT InVT = In.getSimpleValueType();
16569 if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1)
16570 return LowerSIGN_EXTEND_AVX512(Op, Subtarget, DAG);
16572 if ((VT != MVT::v4i64 || InVT != MVT::v4i32) &&
16573 (VT != MVT::v8i32 || InVT != MVT::v8i16) &&
16574 (VT != MVT::v16i16 || InVT != MVT::v16i8))
16577 if (Subtarget->hasInt256())
16578 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16580 // Optimize vectors in AVX mode
16581 // Sign extend v8i16 to v8i32 and
16584 // Divide input vector into two parts
16585 // for v4i32 the shuffle mask will be { 0, 1, -1, -1} {2, 3, -1, -1}
16586 // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
16587 // concat the vectors to original VT
16589 unsigned NumElems = InVT.getVectorNumElements();
16590 SDValue Undef = DAG.getUNDEF(InVT);
16592 SmallVector<int,8> ShufMask1(NumElems, -1);
16593 for (unsigned i = 0; i != NumElems/2; ++i)
16596 SDValue OpLo = DAG.getVectorShuffle(InVT, dl, In, Undef, &ShufMask1[0]);
16598 SmallVector<int,8> ShufMask2(NumElems, -1);
16599 for (unsigned i = 0; i != NumElems/2; ++i)
16600 ShufMask2[i] = i + NumElems/2;
16602 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, Undef, &ShufMask2[0]);
16604 MVT HalfVT = MVT::getVectorVT(VT.getScalarType(),
16605 VT.getVectorNumElements()/2);
16607 OpLo = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpLo);
16608 OpHi = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpHi);
16610 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
16613 // Lower vector extended loads using a shuffle. If SSSE3 is not available we
16614 // may emit an illegal shuffle but the expansion is still better than scalar
16615 // code. We generate X86ISD::VSEXT for SEXTLOADs if it's available, otherwise
16616 // we'll emit a shuffle and a arithmetic shift.
16617 // FIXME: Is the expansion actually better than scalar code? It doesn't seem so.
16618 // TODO: It is possible to support ZExt by zeroing the undef values during
16619 // the shuffle phase or after the shuffle.
16620 static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget *Subtarget,
16621 SelectionDAG &DAG) {
16622 MVT RegVT = Op.getSimpleValueType();
16623 assert(RegVT.isVector() && "We only custom lower vector sext loads.");
16624 assert(RegVT.isInteger() &&
16625 "We only custom lower integer vector sext loads.");
16627 // Nothing useful we can do without SSE2 shuffles.
16628 assert(Subtarget->hasSSE2() && "We only custom lower sext loads with SSE2.");
16630 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
16632 EVT MemVT = Ld->getMemoryVT();
16633 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16634 unsigned RegSz = RegVT.getSizeInBits();
16636 ISD::LoadExtType Ext = Ld->getExtensionType();
16638 assert((Ext == ISD::EXTLOAD || Ext == ISD::SEXTLOAD)
16639 && "Only anyext and sext are currently implemented.");
16640 assert(MemVT != RegVT && "Cannot extend to the same type");
16641 assert(MemVT.isVector() && "Must load a vector from memory");
16643 unsigned NumElems = RegVT.getVectorNumElements();
16644 unsigned MemSz = MemVT.getSizeInBits();
16645 assert(RegSz > MemSz && "Register size must be greater than the mem size");
16647 if (Ext == ISD::SEXTLOAD && RegSz == 256 && !Subtarget->hasInt256()) {
16648 // The only way in which we have a legal 256-bit vector result but not the
16649 // integer 256-bit operations needed to directly lower a sextload is if we
16650 // have AVX1 but not AVX2. In that case, we can always emit a sextload to
16651 // a 128-bit vector and a normal sign_extend to 256-bits that should get
16652 // correctly legalized. We do this late to allow the canonical form of
16653 // sextload to persist throughout the rest of the DAG combiner -- it wants
16654 // to fold together any extensions it can, and so will fuse a sign_extend
16655 // of an sextload into a sextload targeting a wider value.
16657 if (MemSz == 128) {
16658 // Just switch this to a normal load.
16659 assert(TLI.isTypeLegal(MemVT) && "If the memory type is a 128-bit type, "
16660 "it must be a legal 128-bit vector "
16662 Load = DAG.getLoad(MemVT, dl, Ld->getChain(), Ld->getBasePtr(),
16663 Ld->getPointerInfo(), Ld->isVolatile(), Ld->isNonTemporal(),
16664 Ld->isInvariant(), Ld->getAlignment());
16666 assert(MemSz < 128 &&
16667 "Can't extend a type wider than 128 bits to a 256 bit vector!");
16668 // Do an sext load to a 128-bit vector type. We want to use the same
16669 // number of elements, but elements half as wide. This will end up being
16670 // recursively lowered by this routine, but will succeed as we definitely
16671 // have all the necessary features if we're using AVX1.
16673 EVT::getIntegerVT(*DAG.getContext(), RegVT.getScalarSizeInBits() / 2);
16674 EVT HalfVecVT = EVT::getVectorVT(*DAG.getContext(), HalfEltVT, NumElems);
16676 DAG.getExtLoad(Ext, dl, HalfVecVT, Ld->getChain(), Ld->getBasePtr(),
16677 Ld->getPointerInfo(), MemVT, Ld->isVolatile(),
16678 Ld->isNonTemporal(), Ld->isInvariant(),
16679 Ld->getAlignment());
16682 // Replace chain users with the new chain.
16683 assert(Load->getNumValues() == 2 && "Loads must carry a chain!");
16684 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
16686 // Finally, do a normal sign-extend to the desired register.
16687 return DAG.getSExtOrTrunc(Load, dl, RegVT);
16690 // All sizes must be a power of two.
16691 assert(isPowerOf2_32(RegSz * MemSz * NumElems) &&
16692 "Non-power-of-two elements are not custom lowered!");
16694 // Attempt to load the original value using scalar loads.
16695 // Find the largest scalar type that divides the total loaded size.
16696 MVT SclrLoadTy = MVT::i8;
16697 for (MVT Tp : MVT::integer_valuetypes()) {
16698 if (TLI.isTypeLegal(Tp) && ((MemSz % Tp.getSizeInBits()) == 0)) {
16703 // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
16704 if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 &&
16706 SclrLoadTy = MVT::f64;
16708 // Calculate the number of scalar loads that we need to perform
16709 // in order to load our vector from memory.
16710 unsigned NumLoads = MemSz / SclrLoadTy.getSizeInBits();
16712 assert((Ext != ISD::SEXTLOAD || NumLoads == 1) &&
16713 "Can only lower sext loads with a single scalar load!");
16715 unsigned loadRegZize = RegSz;
16716 if (Ext == ISD::SEXTLOAD && RegSz == 256)
16719 // Represent our vector as a sequence of elements which are the
16720 // largest scalar that we can load.
16721 EVT LoadUnitVecVT = EVT::getVectorVT(
16722 *DAG.getContext(), SclrLoadTy, loadRegZize / SclrLoadTy.getSizeInBits());
16724 // Represent the data using the same element type that is stored in
16725 // memory. In practice, we ''widen'' MemVT.
16727 EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
16728 loadRegZize / MemVT.getScalarType().getSizeInBits());
16730 assert(WideVecVT.getSizeInBits() == LoadUnitVecVT.getSizeInBits() &&
16731 "Invalid vector type");
16733 // We can't shuffle using an illegal type.
16734 assert(TLI.isTypeLegal(WideVecVT) &&
16735 "We only lower types that form legal widened vector types");
16737 SmallVector<SDValue, 8> Chains;
16738 SDValue Ptr = Ld->getBasePtr();
16739 SDValue Increment =
16740 DAG.getConstant(SclrLoadTy.getSizeInBits() / 8, TLI.getPointerTy());
16741 SDValue Res = DAG.getUNDEF(LoadUnitVecVT);
16743 for (unsigned i = 0; i < NumLoads; ++i) {
16744 // Perform a single load.
16745 SDValue ScalarLoad =
16746 DAG.getLoad(SclrLoadTy, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
16747 Ld->isVolatile(), Ld->isNonTemporal(), Ld->isInvariant(),
16748 Ld->getAlignment());
16749 Chains.push_back(ScalarLoad.getValue(1));
16750 // Create the first element type using SCALAR_TO_VECTOR in order to avoid
16751 // another round of DAGCombining.
16753 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoadUnitVecVT, ScalarLoad);
16755 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, LoadUnitVecVT, Res,
16756 ScalarLoad, DAG.getIntPtrConstant(i));
16758 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
16761 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
16763 // Bitcast the loaded value to a vector of the original element type, in
16764 // the size of the target vector type.
16765 SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Res);
16766 unsigned SizeRatio = RegSz / MemSz;
16768 if (Ext == ISD::SEXTLOAD) {
16769 // If we have SSE4.1, we can directly emit a VSEXT node.
16770 if (Subtarget->hasSSE41()) {
16771 SDValue Sext = DAG.getNode(X86ISD::VSEXT, dl, RegVT, SlicedVec);
16772 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
16776 // Otherwise we'll shuffle the small elements in the high bits of the
16777 // larger type and perform an arithmetic shift. If the shift is not legal
16778 // it's better to scalarize.
16779 assert(TLI.isOperationLegalOrCustom(ISD::SRA, RegVT) &&
16780 "We can't implement a sext load without an arithmetic right shift!");
16782 // Redistribute the loaded elements into the different locations.
16783 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
16784 for (unsigned i = 0; i != NumElems; ++i)
16785 ShuffleVec[i * SizeRatio + SizeRatio - 1] = i;
16787 SDValue Shuff = DAG.getVectorShuffle(
16788 WideVecVT, dl, SlicedVec, DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
16790 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
16792 // Build the arithmetic shift.
16793 unsigned Amt = RegVT.getVectorElementType().getSizeInBits() -
16794 MemVT.getVectorElementType().getSizeInBits();
16796 DAG.getNode(ISD::SRA, dl, RegVT, Shuff, DAG.getConstant(Amt, RegVT));
16798 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
16802 // Redistribute the loaded elements into the different locations.
16803 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
16804 for (unsigned i = 0; i != NumElems; ++i)
16805 ShuffleVec[i * SizeRatio] = i;
16807 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
16808 DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
16810 // Bitcast to the requested type.
16811 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff);
16812 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), TF);
16816 // isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or
16817 // ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart
16818 // from the AND / OR.
16819 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
16820 Opc = Op.getOpcode();
16821 if (Opc != ISD::OR && Opc != ISD::AND)
16823 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16824 Op.getOperand(0).hasOneUse() &&
16825 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
16826 Op.getOperand(1).hasOneUse());
16829 // isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and
16830 // 1 and that the SETCC node has a single use.
16831 static bool isXor1OfSetCC(SDValue Op) {
16832 if (Op.getOpcode() != ISD::XOR)
16834 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
16835 if (N1C && N1C->getAPIntValue() == 1) {
16836 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16837 Op.getOperand(0).hasOneUse();
16842 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
16843 bool addTest = true;
16844 SDValue Chain = Op.getOperand(0);
16845 SDValue Cond = Op.getOperand(1);
16846 SDValue Dest = Op.getOperand(2);
16849 bool Inverted = false;
16851 if (Cond.getOpcode() == ISD::SETCC) {
16852 // Check for setcc([su]{add,sub,mul}o == 0).
16853 if (cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
16854 isa<ConstantSDNode>(Cond.getOperand(1)) &&
16855 cast<ConstantSDNode>(Cond.getOperand(1))->isNullValue() &&
16856 Cond.getOperand(0).getResNo() == 1 &&
16857 (Cond.getOperand(0).getOpcode() == ISD::SADDO ||
16858 Cond.getOperand(0).getOpcode() == ISD::UADDO ||
16859 Cond.getOperand(0).getOpcode() == ISD::SSUBO ||
16860 Cond.getOperand(0).getOpcode() == ISD::USUBO ||
16861 Cond.getOperand(0).getOpcode() == ISD::SMULO ||
16862 Cond.getOperand(0).getOpcode() == ISD::UMULO)) {
16864 Cond = Cond.getOperand(0);
16866 SDValue NewCond = LowerSETCC(Cond, DAG);
16867 if (NewCond.getNode())
16872 // FIXME: LowerXALUO doesn't handle these!!
16873 else if (Cond.getOpcode() == X86ISD::ADD ||
16874 Cond.getOpcode() == X86ISD::SUB ||
16875 Cond.getOpcode() == X86ISD::SMUL ||
16876 Cond.getOpcode() == X86ISD::UMUL)
16877 Cond = LowerXALUO(Cond, DAG);
16880 // Look pass (and (setcc_carry (cmp ...)), 1).
16881 if (Cond.getOpcode() == ISD::AND &&
16882 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
16883 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
16884 if (C && C->getAPIntValue() == 1)
16885 Cond = Cond.getOperand(0);
16888 // If condition flag is set by a X86ISD::CMP, then use it as the condition
16889 // setting operand in place of the X86ISD::SETCC.
16890 unsigned CondOpcode = Cond.getOpcode();
16891 if (CondOpcode == X86ISD::SETCC ||
16892 CondOpcode == X86ISD::SETCC_CARRY) {
16893 CC = Cond.getOperand(0);
16895 SDValue Cmp = Cond.getOperand(1);
16896 unsigned Opc = Cmp.getOpcode();
16897 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp??
16898 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
16902 switch (cast<ConstantSDNode>(CC)->getZExtValue()) {
16906 // These can only come from an arithmetic instruction with overflow,
16907 // e.g. SADDO, UADDO.
16908 Cond = Cond.getNode()->getOperand(1);
16914 CondOpcode = Cond.getOpcode();
16915 if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
16916 CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
16917 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) &&
16918 Cond.getOperand(0).getValueType() != MVT::i8)) {
16919 SDValue LHS = Cond.getOperand(0);
16920 SDValue RHS = Cond.getOperand(1);
16921 unsigned X86Opcode;
16924 // Keep this in sync with LowerXALUO, otherwise we might create redundant
16925 // instructions that can't be removed afterwards (i.e. X86ISD::ADD and
16927 switch (CondOpcode) {
16928 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
16930 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
16932 X86Opcode = X86ISD::INC; X86Cond = X86::COND_O;
16935 X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
16936 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
16938 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
16940 X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O;
16943 X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
16944 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
16945 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
16946 default: llvm_unreachable("unexpected overflowing operator");
16949 X86Cond = X86::GetOppositeBranchCondition((X86::CondCode)X86Cond);
16950 if (CondOpcode == ISD::UMULO)
16951 VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(),
16954 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
16956 SDValue X86Op = DAG.getNode(X86Opcode, dl, VTs, LHS, RHS);
16958 if (CondOpcode == ISD::UMULO)
16959 Cond = X86Op.getValue(2);
16961 Cond = X86Op.getValue(1);
16963 CC = DAG.getConstant(X86Cond, MVT::i8);
16967 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) {
16968 SDValue Cmp = Cond.getOperand(0).getOperand(1);
16969 if (CondOpc == ISD::OR) {
16970 // Also, recognize the pattern generated by an FCMP_UNE. We can emit
16971 // two branches instead of an explicit OR instruction with a
16973 if (Cmp == Cond.getOperand(1).getOperand(1) &&
16974 isX86LogicalCmp(Cmp)) {
16975 CC = Cond.getOperand(0).getOperand(0);
16976 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16977 Chain, Dest, CC, Cmp);
16978 CC = Cond.getOperand(1).getOperand(0);
16982 } else { // ISD::AND
16983 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit
16984 // two branches instead of an explicit AND instruction with a
16985 // separate test. However, we only do this if this block doesn't
16986 // have a fall-through edge, because this requires an explicit
16987 // jmp when the condition is false.
16988 if (Cmp == Cond.getOperand(1).getOperand(1) &&
16989 isX86LogicalCmp(Cmp) &&
16990 Op.getNode()->hasOneUse()) {
16991 X86::CondCode CCode =
16992 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
16993 CCode = X86::GetOppositeBranchCondition(CCode);
16994 CC = DAG.getConstant(CCode, MVT::i8);
16995 SDNode *User = *Op.getNode()->use_begin();
16996 // Look for an unconditional branch following this conditional branch.
16997 // We need this because we need to reverse the successors in order
16998 // to implement FCMP_OEQ.
16999 if (User->getOpcode() == ISD::BR) {
17000 SDValue FalseBB = User->getOperand(1);
17002 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
17003 assert(NewBR == User);
17007 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17008 Chain, Dest, CC, Cmp);
17009 X86::CondCode CCode =
17010 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
17011 CCode = X86::GetOppositeBranchCondition(CCode);
17012 CC = DAG.getConstant(CCode, MVT::i8);
17018 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) {
17019 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition.
17020 // It should be transformed during dag combiner except when the condition
17021 // is set by a arithmetics with overflow node.
17022 X86::CondCode CCode =
17023 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
17024 CCode = X86::GetOppositeBranchCondition(CCode);
17025 CC = DAG.getConstant(CCode, MVT::i8);
17026 Cond = Cond.getOperand(0).getOperand(1);
17028 } else if (Cond.getOpcode() == ISD::SETCC &&
17029 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETOEQ) {
17030 // For FCMP_OEQ, we can emit
17031 // two branches instead of an explicit AND instruction with a
17032 // separate test. However, we only do this if this block doesn't
17033 // have a fall-through edge, because this requires an explicit
17034 // jmp when the condition is false.
17035 if (Op.getNode()->hasOneUse()) {
17036 SDNode *User = *Op.getNode()->use_begin();
17037 // Look for an unconditional branch following this conditional branch.
17038 // We need this because we need to reverse the successors in order
17039 // to implement FCMP_OEQ.
17040 if (User->getOpcode() == ISD::BR) {
17041 SDValue FalseBB = User->getOperand(1);
17043 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
17044 assert(NewBR == User);
17048 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
17049 Cond.getOperand(0), Cond.getOperand(1));
17050 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
17051 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
17052 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17053 Chain, Dest, CC, Cmp);
17054 CC = DAG.getConstant(X86::COND_P, MVT::i8);
17059 } else if (Cond.getOpcode() == ISD::SETCC &&
17060 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) {
17061 // For FCMP_UNE, we can emit
17062 // two branches instead of an explicit AND instruction with a
17063 // separate test. However, we only do this if this block doesn't
17064 // have a fall-through edge, because this requires an explicit
17065 // jmp when the condition is false.
17066 if (Op.getNode()->hasOneUse()) {
17067 SDNode *User = *Op.getNode()->use_begin();
17068 // Look for an unconditional branch following this conditional branch.
17069 // We need this because we need to reverse the successors in order
17070 // to implement FCMP_UNE.
17071 if (User->getOpcode() == ISD::BR) {
17072 SDValue FalseBB = User->getOperand(1);
17074 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
17075 assert(NewBR == User);
17078 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
17079 Cond.getOperand(0), Cond.getOperand(1));
17080 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
17081 CC = DAG.getConstant(X86::COND_NE, MVT::i8);
17082 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17083 Chain, Dest, CC, Cmp);
17084 CC = DAG.getConstant(X86::COND_NP, MVT::i8);
17094 // Look pass the truncate if the high bits are known zero.
17095 if (isTruncWithZeroHighBitsInput(Cond, DAG))
17096 Cond = Cond.getOperand(0);
17098 // We know the result of AND is compared against zero. Try to match
17100 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
17101 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG);
17102 if (NewSetCC.getNode()) {
17103 CC = NewSetCC.getOperand(0);
17104 Cond = NewSetCC.getOperand(1);
17111 X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE;
17112 CC = DAG.getConstant(X86Cond, MVT::i8);
17113 Cond = EmitTest(Cond, X86Cond, dl, DAG);
17115 Cond = ConvertCmpIfNecessary(Cond, DAG);
17116 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
17117 Chain, Dest, CC, Cond);
17120 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
17121 // Calls to _alloca are needed to probe the stack when allocating more than 4k
17122 // bytes in one go. Touching the stack at 4K increments is necessary to ensure
17123 // that the guard pages used by the OS virtual memory manager are allocated in
17124 // correct sequence.
17126 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
17127 SelectionDAG &DAG) const {
17128 MachineFunction &MF = DAG.getMachineFunction();
17129 bool SplitStack = MF.shouldSplitStack();
17130 bool Lower = (Subtarget->isOSWindows() && !Subtarget->isTargetMachO()) ||
17135 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17136 SDNode* Node = Op.getNode();
17138 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
17139 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
17140 " not tell us which reg is the stack pointer!");
17141 EVT VT = Node->getValueType(0);
17142 SDValue Tmp1 = SDValue(Node, 0);
17143 SDValue Tmp2 = SDValue(Node, 1);
17144 SDValue Tmp3 = Node->getOperand(2);
17145 SDValue Chain = Tmp1.getOperand(0);
17147 // Chain the dynamic stack allocation so that it doesn't modify the stack
17148 // pointer when other instructions are using the stack.
17149 Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, true),
17152 SDValue Size = Tmp2.getOperand(1);
17153 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
17154 Chain = SP.getValue(1);
17155 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
17156 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
17157 unsigned StackAlign = TFI.getStackAlignment();
17158 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
17159 if (Align > StackAlign)
17160 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
17161 DAG.getConstant(-(uint64_t)Align, VT));
17162 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
17164 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, true),
17165 DAG.getIntPtrConstant(0, true), SDValue(),
17168 SDValue Ops[2] = { Tmp1, Tmp2 };
17169 return DAG.getMergeValues(Ops, dl);
17173 SDValue Chain = Op.getOperand(0);
17174 SDValue Size = Op.getOperand(1);
17175 unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
17176 EVT VT = Op.getNode()->getValueType(0);
17178 bool Is64Bit = Subtarget->is64Bit();
17179 EVT SPTy = getPointerTy();
17182 MachineRegisterInfo &MRI = MF.getRegInfo();
17185 // The 64 bit implementation of segmented stacks needs to clobber both r10
17186 // r11. This makes it impossible to use it along with nested parameters.
17187 const Function *F = MF.getFunction();
17189 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
17191 if (I->hasNestAttr())
17192 report_fatal_error("Cannot use segmented stacks with functions that "
17193 "have nested arguments.");
17196 const TargetRegisterClass *AddrRegClass =
17197 getRegClassFor(getPointerTy());
17198 unsigned Vreg = MRI.createVirtualRegister(AddrRegClass);
17199 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
17200 SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
17201 DAG.getRegister(Vreg, SPTy));
17202 SDValue Ops1[2] = { Value, Chain };
17203 return DAG.getMergeValues(Ops1, dl);
17206 const unsigned Reg = (Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX);
17208 Chain = DAG.getCopyToReg(Chain, dl, Reg, Size, Flag);
17209 Flag = Chain.getValue(1);
17210 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
17212 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
17214 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
17215 unsigned SPReg = RegInfo->getStackRegister();
17216 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
17217 Chain = SP.getValue(1);
17220 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
17221 DAG.getConstant(-(uint64_t)Align, VT));
17222 Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
17225 SDValue Ops1[2] = { SP, Chain };
17226 return DAG.getMergeValues(Ops1, dl);
17230 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
17231 MachineFunction &MF = DAG.getMachineFunction();
17232 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
17234 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
17237 if (!Subtarget->is64Bit() || Subtarget->isTargetWin64()) {
17238 // vastart just stores the address of the VarArgsFrameIndex slot into the
17239 // memory location argument.
17240 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
17242 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
17243 MachinePointerInfo(SV), false, false, 0);
17247 // gp_offset (0 - 6 * 8)
17248 // fp_offset (48 - 48 + 8 * 16)
17249 // overflow_arg_area (point to parameters coming in memory).
17251 SmallVector<SDValue, 8> MemOps;
17252 SDValue FIN = Op.getOperand(1);
17254 SDValue Store = DAG.getStore(Op.getOperand(0), DL,
17255 DAG.getConstant(FuncInfo->getVarArgsGPOffset(),
17257 FIN, MachinePointerInfo(SV), false, false, 0);
17258 MemOps.push_back(Store);
17261 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
17262 FIN, DAG.getIntPtrConstant(4));
17263 Store = DAG.getStore(Op.getOperand(0), DL,
17264 DAG.getConstant(FuncInfo->getVarArgsFPOffset(),
17266 FIN, MachinePointerInfo(SV, 4), false, false, 0);
17267 MemOps.push_back(Store);
17269 // Store ptr to overflow_arg_area
17270 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
17271 FIN, DAG.getIntPtrConstant(4));
17272 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
17274 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN,
17275 MachinePointerInfo(SV, 8),
17277 MemOps.push_back(Store);
17279 // Store ptr to reg_save_area.
17280 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(),
17281 FIN, DAG.getIntPtrConstant(8));
17282 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
17284 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN,
17285 MachinePointerInfo(SV, 16), false, false, 0);
17286 MemOps.push_back(Store);
17287 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
17290 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
17291 assert(Subtarget->is64Bit() &&
17292 "LowerVAARG only handles 64-bit va_arg!");
17293 assert((Subtarget->isTargetLinux() ||
17294 Subtarget->isTargetDarwin()) &&
17295 "Unhandled target in LowerVAARG");
17296 assert(Op.getNode()->getNumOperands() == 4);
17297 SDValue Chain = Op.getOperand(0);
17298 SDValue SrcPtr = Op.getOperand(1);
17299 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
17300 unsigned Align = Op.getConstantOperandVal(3);
17303 EVT ArgVT = Op.getNode()->getValueType(0);
17304 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
17305 uint32_t ArgSize = getDataLayout()->getTypeAllocSize(ArgTy);
17308 // Decide which area this value should be read from.
17309 // TODO: Implement the AMD64 ABI in its entirety. This simple
17310 // selection mechanism works only for the basic types.
17311 if (ArgVT == MVT::f80) {
17312 llvm_unreachable("va_arg for f80 not yet implemented");
17313 } else if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
17314 ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
17315 } else if (ArgVT.isInteger() && ArgSize <= 32 /*bytes*/) {
17316 ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
17318 llvm_unreachable("Unhandled argument type in LowerVAARG");
17321 if (ArgMode == 2) {
17322 // Sanity Check: Make sure using fp_offset makes sense.
17323 assert(!DAG.getTarget().Options.UseSoftFloat &&
17324 !(DAG.getMachineFunction().getFunction()->hasFnAttribute(
17325 Attribute::NoImplicitFloat)) &&
17326 Subtarget->hasSSE1());
17329 // Insert VAARG_64 node into the DAG
17330 // VAARG_64 returns two values: Variable Argument Address, Chain
17331 SmallVector<SDValue, 11> InstOps;
17332 InstOps.push_back(Chain);
17333 InstOps.push_back(SrcPtr);
17334 InstOps.push_back(DAG.getConstant(ArgSize, MVT::i32));
17335 InstOps.push_back(DAG.getConstant(ArgMode, MVT::i8));
17336 InstOps.push_back(DAG.getConstant(Align, MVT::i32));
17337 SDVTList VTs = DAG.getVTList(getPointerTy(), MVT::Other);
17338 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl,
17339 VTs, InstOps, MVT::i64,
17340 MachinePointerInfo(SV),
17342 /*Volatile=*/false,
17344 /*WriteMem=*/true);
17345 Chain = VAARG.getValue(1);
17347 // Load the next argument and return it
17348 return DAG.getLoad(ArgVT, dl,
17351 MachinePointerInfo(),
17352 false, false, false, 0);
17355 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget,
17356 SelectionDAG &DAG) {
17357 // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
17358 assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!");
17359 SDValue Chain = Op.getOperand(0);
17360 SDValue DstPtr = Op.getOperand(1);
17361 SDValue SrcPtr = Op.getOperand(2);
17362 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
17363 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
17366 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr,
17367 DAG.getIntPtrConstant(24), 8, /*isVolatile*/false,
17369 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
17372 // getTargetVShiftByConstNode - Handle vector element shifts where the shift
17373 // amount is a constant. Takes immediate version of shift as input.
17374 static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT,
17375 SDValue SrcOp, uint64_t ShiftAmt,
17376 SelectionDAG &DAG) {
17377 MVT ElementType = VT.getVectorElementType();
17379 // Fold this packed shift into its first operand if ShiftAmt is 0.
17383 // Check for ShiftAmt >= element width
17384 if (ShiftAmt >= ElementType.getSizeInBits()) {
17385 if (Opc == X86ISD::VSRAI)
17386 ShiftAmt = ElementType.getSizeInBits() - 1;
17388 return DAG.getConstant(0, VT);
17391 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
17392 && "Unknown target vector shift-by-constant node");
17394 // Fold this packed vector shift into a build vector if SrcOp is a
17395 // vector of Constants or UNDEFs, and SrcOp valuetype is the same as VT.
17396 if (VT == SrcOp.getSimpleValueType() &&
17397 ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
17398 SmallVector<SDValue, 8> Elts;
17399 unsigned NumElts = SrcOp->getNumOperands();
17400 ConstantSDNode *ND;
17403 default: llvm_unreachable(nullptr);
17404 case X86ISD::VSHLI:
17405 for (unsigned i=0; i!=NumElts; ++i) {
17406 SDValue CurrentOp = SrcOp->getOperand(i);
17407 if (CurrentOp->getOpcode() == ISD::UNDEF) {
17408 Elts.push_back(CurrentOp);
17411 ND = cast<ConstantSDNode>(CurrentOp);
17412 const APInt &C = ND->getAPIntValue();
17413 Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), ElementType));
17416 case X86ISD::VSRLI:
17417 for (unsigned i=0; i!=NumElts; ++i) {
17418 SDValue CurrentOp = SrcOp->getOperand(i);
17419 if (CurrentOp->getOpcode() == ISD::UNDEF) {
17420 Elts.push_back(CurrentOp);
17423 ND = cast<ConstantSDNode>(CurrentOp);
17424 const APInt &C = ND->getAPIntValue();
17425 Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), ElementType));
17428 case X86ISD::VSRAI:
17429 for (unsigned i=0; i!=NumElts; ++i) {
17430 SDValue CurrentOp = SrcOp->getOperand(i);
17431 if (CurrentOp->getOpcode() == ISD::UNDEF) {
17432 Elts.push_back(CurrentOp);
17435 ND = cast<ConstantSDNode>(CurrentOp);
17436 const APInt &C = ND->getAPIntValue();
17437 Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), ElementType));
17442 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
17445 return DAG.getNode(Opc, dl, VT, SrcOp, DAG.getConstant(ShiftAmt, MVT::i8));
17448 // getTargetVShiftNode - Handle vector element shifts where the shift amount
17449 // may or may not be a constant. Takes immediate version of shift as input.
17450 static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT,
17451 SDValue SrcOp, SDValue ShAmt,
17452 SelectionDAG &DAG) {
17453 MVT SVT = ShAmt.getSimpleValueType();
17454 assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!");
17456 // Catch shift-by-constant.
17457 if (ConstantSDNode *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
17458 return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp,
17459 CShAmt->getZExtValue(), DAG);
17461 // Change opcode to non-immediate version
17463 default: llvm_unreachable("Unknown target vector shift node");
17464 case X86ISD::VSHLI: Opc = X86ISD::VSHL; break;
17465 case X86ISD::VSRLI: Opc = X86ISD::VSRL; break;
17466 case X86ISD::VSRAI: Opc = X86ISD::VSRA; break;
17469 const X86Subtarget &Subtarget =
17470 static_cast<const X86Subtarget &>(DAG.getSubtarget());
17471 if (Subtarget.hasSSE41() && ShAmt.getOpcode() == ISD::ZERO_EXTEND &&
17472 ShAmt.getOperand(0).getSimpleValueType() == MVT::i16) {
17473 // Let the shuffle legalizer expand this shift amount node.
17474 SDValue Op0 = ShAmt.getOperand(0);
17475 Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(Op0), MVT::v8i16, Op0);
17476 ShAmt = getShuffleVectorZeroOrUndef(Op0, 0, true, &Subtarget, DAG);
17478 // Need to build a vector containing shift amount.
17479 // SSE/AVX packed shifts only use the lower 64-bit of the shift count.
17480 SmallVector<SDValue, 4> ShOps;
17481 ShOps.push_back(ShAmt);
17482 if (SVT == MVT::i32) {
17483 ShOps.push_back(DAG.getConstant(0, SVT));
17484 ShOps.push_back(DAG.getUNDEF(SVT));
17486 ShOps.push_back(DAG.getUNDEF(SVT));
17488 MVT BVT = SVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64;
17489 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, BVT, ShOps);
17492 // The return type has to be a 128-bit type with the same element
17493 // type as the input type.
17494 MVT EltVT = VT.getVectorElementType();
17495 EVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits());
17497 ShAmt = DAG.getNode(ISD::BITCAST, dl, ShVT, ShAmt);
17498 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
17501 /// \brief Return (and \p Op, \p Mask) for compare instructions or
17502 /// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
17503 /// necessary casting for \p Mask when lowering masking intrinsics.
17504 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
17505 SDValue PreservedSrc,
17506 const X86Subtarget *Subtarget,
17507 SelectionDAG &DAG) {
17508 EVT VT = Op.getValueType();
17509 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(),
17510 MVT::i1, VT.getVectorNumElements());
17511 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17512 Mask.getValueType().getSizeInBits());
17515 assert(MaskVT.isSimple() && "invalid mask type");
17517 if (isAllOnes(Mask))
17520 // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
17521 // are extracted by EXTRACT_SUBVECTOR.
17522 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
17523 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
17524 DAG.getIntPtrConstant(0));
17526 switch (Op.getOpcode()) {
17528 case X86ISD::PCMPEQM:
17529 case X86ISD::PCMPGTM:
17531 case X86ISD::CMPMU:
17532 return DAG.getNode(ISD::AND, dl, VT, Op, VMask);
17534 if (PreservedSrc.getOpcode() == ISD::UNDEF)
17535 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
17536 return DAG.getNode(ISD::VSELECT, dl, VT, VMask, Op, PreservedSrc);
17539 /// \brief Creates an SDNode for a predicated scalar operation.
17540 /// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
17541 /// The mask is comming as MVT::i8 and it should be truncated
17542 /// to MVT::i1 while lowering masking intrinsics.
17543 /// The main difference between ScalarMaskingNode and VectorMaskingNode is using
17544 /// "X86select" instead of "vselect". We just can't create the "vselect" node for
17545 /// a scalar instruction.
17546 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
17547 SDValue PreservedSrc,
17548 const X86Subtarget *Subtarget,
17549 SelectionDAG &DAG) {
17550 if (isAllOnes(Mask))
17553 EVT VT = Op.getValueType();
17555 // The mask should be of type MVT::i1
17556 SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask);
17558 if (PreservedSrc.getOpcode() == ISD::UNDEF)
17559 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
17560 return DAG.getNode(X86ISD::SELECT, dl, VT, IMask, Op, PreservedSrc);
17563 static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
17564 SelectionDAG &DAG) {
17566 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
17567 EVT VT = Op.getValueType();
17568 const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
17570 switch(IntrData->Type) {
17571 case INTR_TYPE_1OP:
17572 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1));
17573 case INTR_TYPE_2OP:
17574 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17576 case INTR_TYPE_3OP:
17577 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17578 Op.getOperand(2), Op.getOperand(3));
17579 case INTR_TYPE_1OP_MASK_RM: {
17580 SDValue Src = Op.getOperand(1);
17581 SDValue Src0 = Op.getOperand(2);
17582 SDValue Mask = Op.getOperand(3);
17583 SDValue RoundingMode = Op.getOperand(4);
17584 return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src,
17586 Mask, Src0, Subtarget, DAG);
17588 case INTR_TYPE_SCALAR_MASK_RM: {
17589 SDValue Src1 = Op.getOperand(1);
17590 SDValue Src2 = Op.getOperand(2);
17591 SDValue Src0 = Op.getOperand(3);
17592 SDValue Mask = Op.getOperand(4);
17593 SDValue RoundingMode = Op.getOperand(5);
17594 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2,
17596 Mask, Src0, Subtarget, DAG);
17598 case INTR_TYPE_2OP_MASK: {
17599 SDValue Mask = Op.getOperand(4);
17600 SDValue PassThru = Op.getOperand(3);
17601 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
17602 if (IntrWithRoundingModeOpcode != 0) {
17603 unsigned Round = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue();
17604 if (Round != X86::STATIC_ROUNDING::CUR_DIRECTION) {
17605 return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
17606 dl, Op.getValueType(),
17607 Op.getOperand(1), Op.getOperand(2),
17608 Op.getOperand(3), Op.getOperand(5)),
17609 Mask, PassThru, Subtarget, DAG);
17612 return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
17615 Mask, PassThru, Subtarget, DAG);
17617 case FMA_OP_MASK: {
17618 SDValue Src1 = Op.getOperand(1);
17619 SDValue Src2 = Op.getOperand(2);
17620 SDValue Src3 = Op.getOperand(3);
17621 SDValue Mask = Op.getOperand(4);
17622 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
17623 if (IntrWithRoundingModeOpcode != 0) {
17624 SDValue Rnd = Op.getOperand(5);
17625 if (cast<ConstantSDNode>(Rnd)->getZExtValue() !=
17626 X86::STATIC_ROUNDING::CUR_DIRECTION)
17627 return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
17628 dl, Op.getValueType(),
17629 Src1, Src2, Src3, Rnd),
17630 Mask, Src1, Subtarget, DAG);
17632 return getVectorMaskingNode(DAG.getNode(IntrData->Opc0,
17633 dl, Op.getValueType(),
17635 Mask, Src1, Subtarget, DAG);
17638 case CMP_MASK_CC: {
17639 // Comparison intrinsics with masks.
17640 // Example of transformation:
17641 // (i8 (int_x86_avx512_mask_pcmpeq_q_128
17642 // (v2i64 %a), (v2i64 %b), (i8 %mask))) ->
17644 // (v8i1 (insert_subvector undef,
17645 // (v2i1 (and (PCMPEQM %a, %b),
17646 // (extract_subvector
17647 // (v8i1 (bitcast %mask)), 0))), 0))))
17648 EVT VT = Op.getOperand(1).getValueType();
17649 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17650 VT.getVectorNumElements());
17651 SDValue Mask = Op.getOperand((IntrData->Type == CMP_MASK_CC) ? 4 : 3);
17652 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17653 Mask.getValueType().getSizeInBits());
17655 if (IntrData->Type == CMP_MASK_CC) {
17656 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
17657 Op.getOperand(2), Op.getOperand(3));
17659 assert(IntrData->Type == CMP_MASK && "Unexpected intrinsic type!");
17660 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
17663 SDValue CmpMask = getVectorMaskingNode(Cmp, Mask,
17664 DAG.getTargetConstant(0, MaskVT),
17666 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, BitcastVT,
17667 DAG.getUNDEF(BitcastVT), CmpMask,
17668 DAG.getIntPtrConstant(0));
17669 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res);
17671 case COMI: { // Comparison intrinsics
17672 ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
17673 SDValue LHS = Op.getOperand(1);
17674 SDValue RHS = Op.getOperand(2);
17675 unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG);
17676 assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!");
17677 SDValue Cond = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
17678 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17679 DAG.getConstant(X86CC, MVT::i8), Cond);
17680 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17683 return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
17684 Op.getOperand(1), Op.getOperand(2), DAG);
17686 return getVectorMaskingNode(getTargetVShiftNode(IntrData->Opc0, dl,
17687 Op.getSimpleValueType(),
17689 Op.getOperand(2), DAG),
17690 Op.getOperand(4), Op.getOperand(3), Subtarget,
17692 case COMPRESS_EXPAND_IN_REG: {
17693 SDValue Mask = Op.getOperand(3);
17694 SDValue DataToCompress = Op.getOperand(1);
17695 SDValue PassThru = Op.getOperand(2);
17696 if (isAllOnes(Mask)) // return data as is
17697 return Op.getOperand(1);
17698 EVT VT = Op.getValueType();
17699 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17700 VT.getVectorNumElements());
17701 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17702 Mask.getValueType().getSizeInBits());
17704 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
17705 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
17706 DAG.getIntPtrConstant(0));
17708 return DAG.getNode(IntrData->Opc0, dl, VT, VMask, DataToCompress,
17712 SDValue Mask = Op.getOperand(3);
17713 EVT VT = Op.getValueType();
17714 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17715 VT.getVectorNumElements());
17716 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17717 Mask.getValueType().getSizeInBits());
17719 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
17720 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
17721 DAG.getIntPtrConstant(0));
17722 return DAG.getNode(IntrData->Opc0, dl, VT, VMask, Op.getOperand(1),
17731 default: return SDValue(); // Don't custom lower most intrinsics.
17733 case Intrinsic::x86_avx512_mask_valign_q_512:
17734 case Intrinsic::x86_avx512_mask_valign_d_512:
17735 // Vector source operands are swapped.
17736 return getVectorMaskingNode(DAG.getNode(X86ISD::VALIGN, dl,
17737 Op.getValueType(), Op.getOperand(2),
17740 Op.getOperand(5), Op.getOperand(4),
17743 // ptest and testp intrinsics. The intrinsic these come from are designed to
17744 // return an integer value, not just an instruction so lower it to the ptest
17745 // or testp pattern and a setcc for the result.
17746 case Intrinsic::x86_sse41_ptestz:
17747 case Intrinsic::x86_sse41_ptestc:
17748 case Intrinsic::x86_sse41_ptestnzc:
17749 case Intrinsic::x86_avx_ptestz_256:
17750 case Intrinsic::x86_avx_ptestc_256:
17751 case Intrinsic::x86_avx_ptestnzc_256:
17752 case Intrinsic::x86_avx_vtestz_ps:
17753 case Intrinsic::x86_avx_vtestc_ps:
17754 case Intrinsic::x86_avx_vtestnzc_ps:
17755 case Intrinsic::x86_avx_vtestz_pd:
17756 case Intrinsic::x86_avx_vtestc_pd:
17757 case Intrinsic::x86_avx_vtestnzc_pd:
17758 case Intrinsic::x86_avx_vtestz_ps_256:
17759 case Intrinsic::x86_avx_vtestc_ps_256:
17760 case Intrinsic::x86_avx_vtestnzc_ps_256:
17761 case Intrinsic::x86_avx_vtestz_pd_256:
17762 case Intrinsic::x86_avx_vtestc_pd_256:
17763 case Intrinsic::x86_avx_vtestnzc_pd_256: {
17764 bool IsTestPacked = false;
17767 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
17768 case Intrinsic::x86_avx_vtestz_ps:
17769 case Intrinsic::x86_avx_vtestz_pd:
17770 case Intrinsic::x86_avx_vtestz_ps_256:
17771 case Intrinsic::x86_avx_vtestz_pd_256:
17772 IsTestPacked = true; // Fallthrough
17773 case Intrinsic::x86_sse41_ptestz:
17774 case Intrinsic::x86_avx_ptestz_256:
17776 X86CC = X86::COND_E;
17778 case Intrinsic::x86_avx_vtestc_ps:
17779 case Intrinsic::x86_avx_vtestc_pd:
17780 case Intrinsic::x86_avx_vtestc_ps_256:
17781 case Intrinsic::x86_avx_vtestc_pd_256:
17782 IsTestPacked = true; // Fallthrough
17783 case Intrinsic::x86_sse41_ptestc:
17784 case Intrinsic::x86_avx_ptestc_256:
17786 X86CC = X86::COND_B;
17788 case Intrinsic::x86_avx_vtestnzc_ps:
17789 case Intrinsic::x86_avx_vtestnzc_pd:
17790 case Intrinsic::x86_avx_vtestnzc_ps_256:
17791 case Intrinsic::x86_avx_vtestnzc_pd_256:
17792 IsTestPacked = true; // Fallthrough
17793 case Intrinsic::x86_sse41_ptestnzc:
17794 case Intrinsic::x86_avx_ptestnzc_256:
17796 X86CC = X86::COND_A;
17800 SDValue LHS = Op.getOperand(1);
17801 SDValue RHS = Op.getOperand(2);
17802 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
17803 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
17804 SDValue CC = DAG.getConstant(X86CC, MVT::i8);
17805 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
17806 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17808 case Intrinsic::x86_avx512_kortestz_w:
17809 case Intrinsic::x86_avx512_kortestc_w: {
17810 unsigned X86CC = (IntNo == Intrinsic::x86_avx512_kortestz_w)? X86::COND_E: X86::COND_B;
17811 SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1));
17812 SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2));
17813 SDValue CC = DAG.getConstant(X86CC, MVT::i8);
17814 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
17815 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test);
17816 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17819 case Intrinsic::x86_sse42_pcmpistria128:
17820 case Intrinsic::x86_sse42_pcmpestria128:
17821 case Intrinsic::x86_sse42_pcmpistric128:
17822 case Intrinsic::x86_sse42_pcmpestric128:
17823 case Intrinsic::x86_sse42_pcmpistrio128:
17824 case Intrinsic::x86_sse42_pcmpestrio128:
17825 case Intrinsic::x86_sse42_pcmpistris128:
17826 case Intrinsic::x86_sse42_pcmpestris128:
17827 case Intrinsic::x86_sse42_pcmpistriz128:
17828 case Intrinsic::x86_sse42_pcmpestriz128: {
17832 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
17833 case Intrinsic::x86_sse42_pcmpistria128:
17834 Opcode = X86ISD::PCMPISTRI;
17835 X86CC = X86::COND_A;
17837 case Intrinsic::x86_sse42_pcmpestria128:
17838 Opcode = X86ISD::PCMPESTRI;
17839 X86CC = X86::COND_A;
17841 case Intrinsic::x86_sse42_pcmpistric128:
17842 Opcode = X86ISD::PCMPISTRI;
17843 X86CC = X86::COND_B;
17845 case Intrinsic::x86_sse42_pcmpestric128:
17846 Opcode = X86ISD::PCMPESTRI;
17847 X86CC = X86::COND_B;
17849 case Intrinsic::x86_sse42_pcmpistrio128:
17850 Opcode = X86ISD::PCMPISTRI;
17851 X86CC = X86::COND_O;
17853 case Intrinsic::x86_sse42_pcmpestrio128:
17854 Opcode = X86ISD::PCMPESTRI;
17855 X86CC = X86::COND_O;
17857 case Intrinsic::x86_sse42_pcmpistris128:
17858 Opcode = X86ISD::PCMPISTRI;
17859 X86CC = X86::COND_S;
17861 case Intrinsic::x86_sse42_pcmpestris128:
17862 Opcode = X86ISD::PCMPESTRI;
17863 X86CC = X86::COND_S;
17865 case Intrinsic::x86_sse42_pcmpistriz128:
17866 Opcode = X86ISD::PCMPISTRI;
17867 X86CC = X86::COND_E;
17869 case Intrinsic::x86_sse42_pcmpestriz128:
17870 Opcode = X86ISD::PCMPESTRI;
17871 X86CC = X86::COND_E;
17874 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
17875 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
17876 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps);
17877 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17878 DAG.getConstant(X86CC, MVT::i8),
17879 SDValue(PCMP.getNode(), 1));
17880 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
17883 case Intrinsic::x86_sse42_pcmpistri128:
17884 case Intrinsic::x86_sse42_pcmpestri128: {
17886 if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
17887 Opcode = X86ISD::PCMPISTRI;
17889 Opcode = X86ISD::PCMPESTRI;
17891 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
17892 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
17893 return DAG.getNode(Opcode, dl, VTs, NewOps);
17898 static SDValue getGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
17899 SDValue Src, SDValue Mask, SDValue Base,
17900 SDValue Index, SDValue ScaleOp, SDValue Chain,
17901 const X86Subtarget * Subtarget) {
17903 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
17904 assert(C && "Invalid scale type");
17905 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
17906 EVT MaskVT = MVT::getVectorVT(MVT::i1,
17907 Index.getSimpleValueType().getVectorNumElements());
17909 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
17911 MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
17913 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
17914 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
17915 SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
17916 SDValue Segment = DAG.getRegister(0, MVT::i32);
17917 if (Src.getOpcode() == ISD::UNDEF)
17918 Src = getZeroVector(Op.getValueType(), Subtarget, DAG, dl);
17919 SDValue Ops[] = {Src, MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
17920 SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
17921 SDValue RetOps[] = { SDValue(Res, 0), SDValue(Res, 2) };
17922 return DAG.getMergeValues(RetOps, dl);
17925 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
17926 SDValue Src, SDValue Mask, SDValue Base,
17927 SDValue Index, SDValue ScaleOp, SDValue Chain) {
17929 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
17930 assert(C && "Invalid scale type");
17931 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
17932 SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
17933 SDValue Segment = DAG.getRegister(0, MVT::i32);
17934 EVT MaskVT = MVT::getVectorVT(MVT::i1,
17935 Index.getSimpleValueType().getVectorNumElements());
17937 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
17939 MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
17941 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
17942 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
17943 SDValue Ops[] = {Base, Scale, Index, Disp, Segment, MaskInReg, Src, Chain};
17944 SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
17945 return SDValue(Res, 1);
17948 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
17949 SDValue Mask, SDValue Base, SDValue Index,
17950 SDValue ScaleOp, SDValue Chain) {
17952 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ScaleOp);
17953 assert(C && "Invalid scale type");
17954 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), MVT::i8);
17955 SDValue Disp = DAG.getTargetConstant(0, MVT::i32);
17956 SDValue Segment = DAG.getRegister(0, MVT::i32);
17958 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
17960 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(Mask);
17962 MaskInReg = DAG.getTargetConstant(MaskC->getSExtValue(), MaskVT);
17964 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask);
17965 //SDVTList VTs = DAG.getVTList(MVT::Other);
17966 SDValue Ops[] = {MaskInReg, Base, Scale, Index, Disp, Segment, Chain};
17967 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
17968 return SDValue(Res, 0);
17971 // getReadPerformanceCounter - Handles the lowering of builtin intrinsics that
17972 // read performance monitor counters (x86_rdpmc).
17973 static void getReadPerformanceCounter(SDNode *N, SDLoc DL,
17974 SelectionDAG &DAG, const X86Subtarget *Subtarget,
17975 SmallVectorImpl<SDValue> &Results) {
17976 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
17977 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
17980 // The ECX register is used to select the index of the performance counter
17982 SDValue Chain = DAG.getCopyToReg(N->getOperand(0), DL, X86::ECX,
17984 SDValue rd = DAG.getNode(X86ISD::RDPMC_DAG, DL, Tys, Chain);
17986 // Reads the content of a 64-bit performance counter and returns it in the
17987 // registers EDX:EAX.
17988 if (Subtarget->is64Bit()) {
17989 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
17990 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
17993 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
17994 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
17997 Chain = HI.getValue(1);
17999 if (Subtarget->is64Bit()) {
18000 // The EAX register is loaded with the low-order 32 bits. The EDX register
18001 // is loaded with the supported high-order bits of the counter.
18002 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
18003 DAG.getConstant(32, MVT::i8));
18004 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
18005 Results.push_back(Chain);
18009 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
18010 SDValue Ops[] = { LO, HI };
18011 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
18012 Results.push_back(Pair);
18013 Results.push_back(Chain);
18016 // getReadTimeStampCounter - Handles the lowering of builtin intrinsics that
18017 // read the time stamp counter (x86_rdtsc and x86_rdtscp). This function is
18018 // also used to custom lower READCYCLECOUNTER nodes.
18019 static void getReadTimeStampCounter(SDNode *N, SDLoc DL, unsigned Opcode,
18020 SelectionDAG &DAG, const X86Subtarget *Subtarget,
18021 SmallVectorImpl<SDValue> &Results) {
18022 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
18023 SDValue rd = DAG.getNode(Opcode, DL, Tys, N->getOperand(0));
18026 // The processor's time-stamp counter (a 64-bit MSR) is stored into the
18027 // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
18028 // and the EAX register is loaded with the low-order 32 bits.
18029 if (Subtarget->is64Bit()) {
18030 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1));
18031 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
18034 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1));
18035 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
18038 SDValue Chain = HI.getValue(1);
18040 if (Opcode == X86ISD::RDTSCP_DAG) {
18041 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
18043 // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
18044 // the ECX register. Add 'ecx' explicitly to the chain.
18045 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32,
18047 // Explicitly store the content of ECX at the location passed in input
18048 // to the 'rdtscp' intrinsic.
18049 Chain = DAG.getStore(ecx.getValue(1), DL, ecx, N->getOperand(2),
18050 MachinePointerInfo(), false, false, 0);
18053 if (Subtarget->is64Bit()) {
18054 // The EDX register is loaded with the high-order 32 bits of the MSR, and
18055 // the EAX register is loaded with the low-order 32 bits.
18056 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
18057 DAG.getConstant(32, MVT::i8));
18058 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
18059 Results.push_back(Chain);
18063 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
18064 SDValue Ops[] = { LO, HI };
18065 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
18066 Results.push_back(Pair);
18067 Results.push_back(Chain);
18070 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
18071 SelectionDAG &DAG) {
18072 SmallVector<SDValue, 2> Results;
18074 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
18076 return DAG.getMergeValues(Results, DL);
18080 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget *Subtarget,
18081 SelectionDAG &DAG) {
18082 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
18084 const IntrinsicData* IntrData = getIntrinsicWithChain(IntNo);
18089 switch(IntrData->Type) {
18091 llvm_unreachable("Unknown Intrinsic Type");
18095 // Emit the node with the right value type.
18096 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other);
18097 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
18099 // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
18100 // Otherwise return the value from Rand, which is always 0, casted to i32.
18101 SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
18102 DAG.getConstant(1, Op->getValueType(1)),
18103 DAG.getConstant(X86::COND_B, MVT::i32),
18104 SDValue(Result.getNode(), 1) };
18105 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl,
18106 DAG.getVTList(Op->getValueType(1), MVT::Glue),
18109 // Return { result, isValid, chain }.
18110 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
18111 SDValue(Result.getNode(), 2));
18114 //gather(v1, mask, index, base, scale);
18115 SDValue Chain = Op.getOperand(0);
18116 SDValue Src = Op.getOperand(2);
18117 SDValue Base = Op.getOperand(3);
18118 SDValue Index = Op.getOperand(4);
18119 SDValue Mask = Op.getOperand(5);
18120 SDValue Scale = Op.getOperand(6);
18121 return getGatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain,
18125 //scatter(base, mask, index, v1, scale);
18126 SDValue Chain = Op.getOperand(0);
18127 SDValue Base = Op.getOperand(2);
18128 SDValue Mask = Op.getOperand(3);
18129 SDValue Index = Op.getOperand(4);
18130 SDValue Src = Op.getOperand(5);
18131 SDValue Scale = Op.getOperand(6);
18132 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale, Chain);
18135 SDValue Hint = Op.getOperand(6);
18137 if (dyn_cast<ConstantSDNode> (Hint) == nullptr ||
18138 (HintVal = dyn_cast<ConstantSDNode> (Hint)->getZExtValue()) > 1)
18139 llvm_unreachable("Wrong prefetch hint in intrinsic: should be 0 or 1");
18140 unsigned Opcode = (HintVal ? IntrData->Opc1 : IntrData->Opc0);
18141 SDValue Chain = Op.getOperand(0);
18142 SDValue Mask = Op.getOperand(2);
18143 SDValue Index = Op.getOperand(3);
18144 SDValue Base = Op.getOperand(4);
18145 SDValue Scale = Op.getOperand(5);
18146 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain);
18148 // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
18150 SmallVector<SDValue, 2> Results;
18151 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget, Results);
18152 return DAG.getMergeValues(Results, dl);
18154 // Read Performance Monitoring Counters.
18156 SmallVector<SDValue, 2> Results;
18157 getReadPerformanceCounter(Op.getNode(), dl, DAG, Subtarget, Results);
18158 return DAG.getMergeValues(Results, dl);
18160 // XTEST intrinsics.
18162 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
18163 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
18164 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18165 DAG.getConstant(X86::COND_NE, MVT::i8),
18167 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
18168 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
18169 Ret, SDValue(InTrans.getNode(), 1));
18173 SmallVector<SDValue, 2> Results;
18174 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
18175 SDVTList VTs = DAG.getVTList(Op.getOperand(3)->getValueType(0), MVT::Other);
18176 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(2),
18177 DAG.getConstant(-1, MVT::i8));
18178 SDValue Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(3),
18179 Op.getOperand(4), GenCF.getValue(1));
18180 SDValue Store = DAG.getStore(Op.getOperand(0), dl, Res.getValue(0),
18181 Op.getOperand(5), MachinePointerInfo(),
18183 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18184 DAG.getConstant(X86::COND_B, MVT::i8),
18186 Results.push_back(SetCC);
18187 Results.push_back(Store);
18188 return DAG.getMergeValues(Results, dl);
18190 case COMPRESS_TO_MEM: {
18192 SDValue Mask = Op.getOperand(4);
18193 SDValue DataToCompress = Op.getOperand(3);
18194 SDValue Addr = Op.getOperand(2);
18195 SDValue Chain = Op.getOperand(0);
18197 if (isAllOnes(Mask)) // return just a store
18198 return DAG.getStore(Chain, dl, DataToCompress, Addr,
18199 MachinePointerInfo(), false, false, 0);
18201 EVT VT = DataToCompress.getValueType();
18202 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18203 VT.getVectorNumElements());
18204 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18205 Mask.getValueType().getSizeInBits());
18206 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
18207 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
18208 DAG.getIntPtrConstant(0));
18210 SDValue Compressed = DAG.getNode(IntrData->Opc0, dl, VT, VMask,
18211 DataToCompress, DAG.getUNDEF(VT));
18212 return DAG.getStore(Chain, dl, Compressed, Addr,
18213 MachinePointerInfo(), false, false, 0);
18215 case EXPAND_FROM_MEM: {
18217 SDValue Mask = Op.getOperand(4);
18218 SDValue PathThru = Op.getOperand(3);
18219 SDValue Addr = Op.getOperand(2);
18220 SDValue Chain = Op.getOperand(0);
18221 EVT VT = Op.getValueType();
18223 if (isAllOnes(Mask)) // return just a load
18224 return DAG.getLoad(VT, dl, Chain, Addr, MachinePointerInfo(), false, false,
18226 EVT MaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18227 VT.getVectorNumElements());
18228 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
18229 Mask.getValueType().getSizeInBits());
18230 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
18231 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask),
18232 DAG.getIntPtrConstant(0));
18234 SDValue DataToExpand = DAG.getLoad(VT, dl, Chain, Addr, MachinePointerInfo(),
18235 false, false, false, 0);
18237 SmallVector<SDValue, 2> Results;
18238 Results.push_back(DAG.getNode(IntrData->Opc0, dl, VT, VMask, DataToExpand,
18240 Results.push_back(Chain);
18241 return DAG.getMergeValues(Results, dl);
18246 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
18247 SelectionDAG &DAG) const {
18248 MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
18249 MFI->setReturnAddressIsTaken(true);
18251 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
18254 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
18256 EVT PtrVT = getPointerTy();
18259 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
18260 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18261 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), PtrVT);
18262 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
18263 DAG.getNode(ISD::ADD, dl, PtrVT,
18264 FrameAddr, Offset),
18265 MachinePointerInfo(), false, false, false, 0);
18268 // Just load the return address.
18269 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
18270 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
18271 RetAddrFI, MachinePointerInfo(), false, false, false, 0);
18274 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
18275 MachineFunction &MF = DAG.getMachineFunction();
18276 MachineFrameInfo *MFI = MF.getFrameInfo();
18277 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
18278 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18279 EVT VT = Op.getValueType();
18281 MFI->setFrameAddressIsTaken(true);
18283 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
18284 // Depth > 0 makes no sense on targets which use Windows unwind codes. It
18285 // is not possible to crawl up the stack without looking at the unwind codes
18287 int FrameAddrIndex = FuncInfo->getFAIndex();
18288 if (!FrameAddrIndex) {
18289 // Set up a frame object for the return address.
18290 unsigned SlotSize = RegInfo->getSlotSize();
18291 FrameAddrIndex = MF.getFrameInfo()->CreateFixedObject(
18292 SlotSize, /*Offset=*/INT64_MIN, /*IsImmutable=*/false);
18293 FuncInfo->setFAIndex(FrameAddrIndex);
18295 return DAG.getFrameIndex(FrameAddrIndex, VT);
18298 unsigned FrameReg =
18299 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
18300 SDLoc dl(Op); // FIXME probably not meaningful
18301 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
18302 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
18303 (FrameReg == X86::EBP && VT == MVT::i32)) &&
18304 "Invalid Frame Register!");
18305 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
18307 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
18308 MachinePointerInfo(),
18309 false, false, false, 0);
18313 // FIXME? Maybe this could be a TableGen attribute on some registers and
18314 // this table could be generated automatically from RegInfo.
18315 unsigned X86TargetLowering::getRegisterByName(const char* RegName,
18317 unsigned Reg = StringSwitch<unsigned>(RegName)
18318 .Case("esp", X86::ESP)
18319 .Case("rsp", X86::RSP)
18323 report_fatal_error("Invalid register name global variable");
18326 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
18327 SelectionDAG &DAG) const {
18328 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18329 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize());
18332 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
18333 SDValue Chain = Op.getOperand(0);
18334 SDValue Offset = Op.getOperand(1);
18335 SDValue Handler = Op.getOperand(2);
18338 EVT PtrVT = getPointerTy();
18339 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
18340 unsigned FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
18341 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||
18342 (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&
18343 "Invalid Frame Register!");
18344 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
18345 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
18347 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
18348 DAG.getIntPtrConstant(RegInfo->getSlotSize()));
18349 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
18350 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(),
18352 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
18354 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
18355 DAG.getRegister(StoreAddrReg, PtrVT));
18358 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
18359 SelectionDAG &DAG) const {
18361 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
18362 DAG.getVTList(MVT::i32, MVT::Other),
18363 Op.getOperand(0), Op.getOperand(1));
18366 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
18367 SelectionDAG &DAG) const {
18369 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
18370 Op.getOperand(0), Op.getOperand(1));
18373 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
18374 return Op.getOperand(0);
18377 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
18378 SelectionDAG &DAG) const {
18379 SDValue Root = Op.getOperand(0);
18380 SDValue Trmp = Op.getOperand(1); // trampoline
18381 SDValue FPtr = Op.getOperand(2); // nested function
18382 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
18385 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
18386 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
18388 if (Subtarget->is64Bit()) {
18389 SDValue OutChains[6];
18391 // Large code-model.
18392 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
18393 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
18395 const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
18396 const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
18398 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
18400 // Load the pointer to the nested function into R11.
18401 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
18402 SDValue Addr = Trmp;
18403 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
18404 Addr, MachinePointerInfo(TrmpAddr),
18407 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18408 DAG.getConstant(2, MVT::i64));
18409 OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr,
18410 MachinePointerInfo(TrmpAddr, 2),
18413 // Load the 'nest' parameter value into R10.
18414 // R10 is specified in X86CallingConv.td
18415 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
18416 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18417 DAG.getConstant(10, MVT::i64));
18418 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
18419 Addr, MachinePointerInfo(TrmpAddr, 10),
18422 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18423 DAG.getConstant(12, MVT::i64));
18424 OutChains[3] = DAG.getStore(Root, dl, Nest, Addr,
18425 MachinePointerInfo(TrmpAddr, 12),
18428 // Jump to the nested function.
18429 OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
18430 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18431 DAG.getConstant(20, MVT::i64));
18432 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
18433 Addr, MachinePointerInfo(TrmpAddr, 20),
18436 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
18437 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
18438 DAG.getConstant(22, MVT::i64));
18439 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr,
18440 MachinePointerInfo(TrmpAddr, 22),
18443 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
18445 const Function *Func =
18446 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
18447 CallingConv::ID CC = Func->getCallingConv();
18452 llvm_unreachable("Unsupported calling convention");
18453 case CallingConv::C:
18454 case CallingConv::X86_StdCall: {
18455 // Pass 'nest' parameter in ECX.
18456 // Must be kept in sync with X86CallingConv.td
18457 NestReg = X86::ECX;
18459 // Check that ECX wasn't needed by an 'inreg' parameter.
18460 FunctionType *FTy = Func->getFunctionType();
18461 const AttributeSet &Attrs = Func->getAttributes();
18463 if (!Attrs.isEmpty() && !Func->isVarArg()) {
18464 unsigned InRegCount = 0;
18467 for (FunctionType::param_iterator I = FTy->param_begin(),
18468 E = FTy->param_end(); I != E; ++I, ++Idx)
18469 if (Attrs.hasAttribute(Idx, Attribute::InReg))
18470 // FIXME: should only count parameters that are lowered to integers.
18471 InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32;
18473 if (InRegCount > 2) {
18474 report_fatal_error("Nest register in use - reduce number of inreg"
18480 case CallingConv::X86_FastCall:
18481 case CallingConv::X86_ThisCall:
18482 case CallingConv::Fast:
18483 // Pass 'nest' parameter in EAX.
18484 // Must be kept in sync with X86CallingConv.td
18485 NestReg = X86::EAX;
18489 SDValue OutChains[4];
18490 SDValue Addr, Disp;
18492 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18493 DAG.getConstant(10, MVT::i32));
18494 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
18496 // This is storing the opcode for MOV32ri.
18497 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
18498 const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
18499 OutChains[0] = DAG.getStore(Root, dl,
18500 DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
18501 Trmp, MachinePointerInfo(TrmpAddr),
18504 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18505 DAG.getConstant(1, MVT::i32));
18506 OutChains[1] = DAG.getStore(Root, dl, Nest, Addr,
18507 MachinePointerInfo(TrmpAddr, 1),
18510 const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
18511 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18512 DAG.getConstant(5, MVT::i32));
18513 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr,
18514 MachinePointerInfo(TrmpAddr, 5),
18517 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
18518 DAG.getConstant(6, MVT::i32));
18519 OutChains[3] = DAG.getStore(Root, dl, Disp, Addr,
18520 MachinePointerInfo(TrmpAddr, 6),
18523 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
18527 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
18528 SelectionDAG &DAG) const {
18530 The rounding mode is in bits 11:10 of FPSR, and has the following
18532 00 Round to nearest
18537 FLT_ROUNDS, on the other hand, expects the following:
18544 To perform the conversion, we do:
18545 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3)
18548 MachineFunction &MF = DAG.getMachineFunction();
18549 const TargetFrameLowering &TFI = *Subtarget->getFrameLowering();
18550 unsigned StackAlignment = TFI.getStackAlignment();
18551 MVT VT = Op.getSimpleValueType();
18554 // Save FP Control Word to stack slot
18555 int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment, false);
18556 SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
18558 MachineMemOperand *MMO =
18559 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(SSFI),
18560 MachineMemOperand::MOStore, 2, 2);
18562 SDValue Ops[] = { DAG.getEntryNode(), StackSlot };
18563 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
18564 DAG.getVTList(MVT::Other),
18565 Ops, MVT::i16, MMO);
18567 // Load FP Control Word from stack slot
18568 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot,
18569 MachinePointerInfo(), false, false, false, 0);
18571 // Transform as necessary
18573 DAG.getNode(ISD::SRL, DL, MVT::i16,
18574 DAG.getNode(ISD::AND, DL, MVT::i16,
18575 CWD, DAG.getConstant(0x800, MVT::i16)),
18576 DAG.getConstant(11, MVT::i8));
18578 DAG.getNode(ISD::SRL, DL, MVT::i16,
18579 DAG.getNode(ISD::AND, DL, MVT::i16,
18580 CWD, DAG.getConstant(0x400, MVT::i16)),
18581 DAG.getConstant(9, MVT::i8));
18584 DAG.getNode(ISD::AND, DL, MVT::i16,
18585 DAG.getNode(ISD::ADD, DL, MVT::i16,
18586 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2),
18587 DAG.getConstant(1, MVT::i16)),
18588 DAG.getConstant(3, MVT::i16));
18590 return DAG.getNode((VT.getSizeInBits() < 16 ?
18591 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
18594 static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) {
18595 MVT VT = Op.getSimpleValueType();
18597 unsigned NumBits = VT.getSizeInBits();
18600 Op = Op.getOperand(0);
18601 if (VT == MVT::i8) {
18602 // Zero extend to i32 since there is not an i8 bsr.
18604 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
18607 // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
18608 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
18609 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
18611 // If src is zero (i.e. bsr sets ZF), returns NumBits.
18614 DAG.getConstant(NumBits+NumBits-1, OpVT),
18615 DAG.getConstant(X86::COND_E, MVT::i8),
18618 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
18620 // Finally xor with NumBits-1.
18621 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
18624 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
18628 static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
18629 MVT VT = Op.getSimpleValueType();
18631 unsigned NumBits = VT.getSizeInBits();
18634 Op = Op.getOperand(0);
18635 if (VT == MVT::i8) {
18636 // Zero extend to i32 since there is not an i8 bsr.
18638 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
18641 // Issue a bsr (scan bits in reverse).
18642 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
18643 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
18645 // And xor with NumBits-1.
18646 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
18649 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
18653 static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
18654 MVT VT = Op.getSimpleValueType();
18655 unsigned NumBits = VT.getSizeInBits();
18657 Op = Op.getOperand(0);
18659 // Issue a bsf (scan bits forward) which also sets EFLAGS.
18660 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
18661 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
18663 // If src is zero (i.e. bsf sets ZF), returns NumBits.
18666 DAG.getConstant(NumBits, VT),
18667 DAG.getConstant(X86::COND_E, MVT::i8),
18670 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
18673 // Lower256IntArith - Break a 256-bit integer operation into two new 128-bit
18674 // ones, and then concatenate the result back.
18675 static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
18676 MVT VT = Op.getSimpleValueType();
18678 assert(VT.is256BitVector() && VT.isInteger() &&
18679 "Unsupported value type for operation");
18681 unsigned NumElems = VT.getVectorNumElements();
18684 // Extract the LHS vectors
18685 SDValue LHS = Op.getOperand(0);
18686 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
18687 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
18689 // Extract the RHS vectors
18690 SDValue RHS = Op.getOperand(1);
18691 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, dl);
18692 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, dl);
18694 MVT EltVT = VT.getVectorElementType();
18695 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
18697 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
18698 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
18699 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
18702 static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
18703 assert(Op.getSimpleValueType().is256BitVector() &&
18704 Op.getSimpleValueType().isInteger() &&
18705 "Only handle AVX 256-bit vector integer operation");
18706 return Lower256IntArith(Op, DAG);
18709 static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) {
18710 assert(Op.getSimpleValueType().is256BitVector() &&
18711 Op.getSimpleValueType().isInteger() &&
18712 "Only handle AVX 256-bit vector integer operation");
18713 return Lower256IntArith(Op, DAG);
18716 static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
18717 SelectionDAG &DAG) {
18719 MVT VT = Op.getSimpleValueType();
18721 // Decompose 256-bit ops into smaller 128-bit ops.
18722 if (VT.is256BitVector() && !Subtarget->hasInt256())
18723 return Lower256IntArith(Op, DAG);
18725 SDValue A = Op.getOperand(0);
18726 SDValue B = Op.getOperand(1);
18728 // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
18729 if (VT == MVT::v4i32) {
18730 assert(Subtarget->hasSSE2() && !Subtarget->hasSSE41() &&
18731 "Should not custom lower when pmuldq is available!");
18733 // Extract the odd parts.
18734 static const int UnpackMask[] = { 1, -1, 3, -1 };
18735 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
18736 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
18738 // Multiply the even parts.
18739 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B);
18740 // Now multiply odd parts.
18741 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds);
18743 Evens = DAG.getNode(ISD::BITCAST, dl, VT, Evens);
18744 Odds = DAG.getNode(ISD::BITCAST, dl, VT, Odds);
18746 // Merge the two vectors back together with a shuffle. This expands into 2
18748 static const int ShufMask[] = { 0, 4, 2, 6 };
18749 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
18752 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
18753 "Only know how to lower V2I64/V4I64/V8I64 multiply");
18755 // Ahi = psrlqi(a, 32);
18756 // Bhi = psrlqi(b, 32);
18758 // AloBlo = pmuludq(a, b);
18759 // AloBhi = pmuludq(a, Bhi);
18760 // AhiBlo = pmuludq(Ahi, b);
18762 // AloBhi = psllqi(AloBhi, 32);
18763 // AhiBlo = psllqi(AhiBlo, 32);
18764 // return AloBlo + AloBhi + AhiBlo;
18766 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
18767 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
18769 // Bit cast to 32-bit vectors for MULUDQ
18770 EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 :
18771 (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32;
18772 A = DAG.getNode(ISD::BITCAST, dl, MulVT, A);
18773 B = DAG.getNode(ISD::BITCAST, dl, MulVT, B);
18774 Ahi = DAG.getNode(ISD::BITCAST, dl, MulVT, Ahi);
18775 Bhi = DAG.getNode(ISD::BITCAST, dl, MulVT, Bhi);
18777 SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
18778 SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
18779 SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
18781 AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
18782 AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
18784 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
18785 return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
18788 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
18789 assert(Subtarget->isTargetWin64() && "Unexpected target");
18790 EVT VT = Op.getValueType();
18791 assert(VT.isInteger() && VT.getSizeInBits() == 128 &&
18792 "Unexpected return type for lowering");
18796 switch (Op->getOpcode()) {
18797 default: llvm_unreachable("Unexpected request for libcall!");
18798 case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
18799 case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
18800 case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
18801 case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
18802 case ISD::SDIVREM: isSigned = true; LC = RTLIB::SDIVREM_I128; break;
18803 case ISD::UDIVREM: isSigned = false; LC = RTLIB::UDIVREM_I128; break;
18807 SDValue InChain = DAG.getEntryNode();
18809 TargetLowering::ArgListTy Args;
18810 TargetLowering::ArgListEntry Entry;
18811 for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
18812 EVT ArgVT = Op->getOperand(i).getValueType();
18813 assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
18814 "Unexpected argument type for lowering");
18815 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
18816 Entry.Node = StackPtr;
18817 InChain = DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MachinePointerInfo(),
18819 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
18820 Entry.Ty = PointerType::get(ArgTy,0);
18821 Entry.isSExt = false;
18822 Entry.isZExt = false;
18823 Args.push_back(Entry);
18826 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
18829 TargetLowering::CallLoweringInfo CLI(DAG);
18830 CLI.setDebugLoc(dl).setChain(InChain)
18831 .setCallee(getLibcallCallingConv(LC),
18832 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()),
18833 Callee, std::move(Args), 0)
18834 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
18836 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
18837 return DAG.getNode(ISD::BITCAST, dl, VT, CallInfo.first);
18840 static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget *Subtarget,
18841 SelectionDAG &DAG) {
18842 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
18843 EVT VT = Op0.getValueType();
18846 assert((VT == MVT::v4i32 && Subtarget->hasSSE2()) ||
18847 (VT == MVT::v8i32 && Subtarget->hasInt256()));
18849 // PMULxD operations multiply each even value (starting at 0) of LHS with
18850 // the related value of RHS and produce a widen result.
18851 // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
18852 // => <2 x i64> <ae|cg>
18854 // In other word, to have all the results, we need to perform two PMULxD:
18855 // 1. one with the even values.
18856 // 2. one with the odd values.
18857 // To achieve #2, with need to place the odd values at an even position.
18859 // Place the odd value at an even position (basically, shift all values 1
18860 // step to the left):
18861 const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1};
18862 // <a|b|c|d> => <b|undef|d|undef>
18863 SDValue Odd0 = DAG.getVectorShuffle(VT, dl, Op0, Op0, Mask);
18864 // <e|f|g|h> => <f|undef|h|undef>
18865 SDValue Odd1 = DAG.getVectorShuffle(VT, dl, Op1, Op1, Mask);
18867 // Emit two multiplies, one for the lower 2 ints and one for the higher 2
18869 MVT MulVT = VT == MVT::v4i32 ? MVT::v2i64 : MVT::v4i64;
18870 bool IsSigned = Op->getOpcode() == ISD::SMUL_LOHI;
18872 (!IsSigned || !Subtarget->hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
18873 // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
18874 // => <2 x i64> <ae|cg>
18875 SDValue Mul1 = DAG.getNode(ISD::BITCAST, dl, VT,
18876 DAG.getNode(Opcode, dl, MulVT, Op0, Op1));
18877 // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
18878 // => <2 x i64> <bf|dh>
18879 SDValue Mul2 = DAG.getNode(ISD::BITCAST, dl, VT,
18880 DAG.getNode(Opcode, dl, MulVT, Odd0, Odd1));
18882 // Shuffle it back into the right order.
18883 SDValue Highs, Lows;
18884 if (VT == MVT::v8i32) {
18885 const int HighMask[] = {1, 9, 3, 11, 5, 13, 7, 15};
18886 Highs = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, HighMask);
18887 const int LowMask[] = {0, 8, 2, 10, 4, 12, 6, 14};
18888 Lows = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, LowMask);
18890 const int HighMask[] = {1, 5, 3, 7};
18891 Highs = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, HighMask);
18892 const int LowMask[] = {0, 4, 2, 6};
18893 Lows = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, LowMask);
18896 // If we have a signed multiply but no PMULDQ fix up the high parts of a
18897 // unsigned multiply.
18898 if (IsSigned && !Subtarget->hasSSE41()) {
18900 DAG.getConstant(31, DAG.getTargetLoweringInfo().getShiftAmountTy(VT));
18901 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
18902 DAG.getNode(ISD::SRA, dl, VT, Op0, ShAmt), Op1);
18903 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
18904 DAG.getNode(ISD::SRA, dl, VT, Op1, ShAmt), Op0);
18906 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
18907 Highs = DAG.getNode(ISD::SUB, dl, VT, Highs, Fixup);
18910 // The first result of MUL_LOHI is actually the low value, followed by the
18912 SDValue Ops[] = {Lows, Highs};
18913 return DAG.getMergeValues(Ops, dl);
18916 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
18917 const X86Subtarget *Subtarget) {
18918 MVT VT = Op.getSimpleValueType();
18920 SDValue R = Op.getOperand(0);
18921 SDValue Amt = Op.getOperand(1);
18923 // Optimize shl/srl/sra with constant shift amount.
18924 if (auto *BVAmt = dyn_cast<BuildVectorSDNode>(Amt)) {
18925 if (auto *ShiftConst = BVAmt->getConstantSplatNode()) {
18926 uint64_t ShiftAmt = ShiftConst->getZExtValue();
18928 if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
18929 (Subtarget->hasInt256() &&
18930 (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16)) ||
18931 (Subtarget->hasAVX512() &&
18932 (VT == MVT::v8i64 || VT == MVT::v16i32))) {
18933 if (Op.getOpcode() == ISD::SHL)
18934 return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
18936 if (Op.getOpcode() == ISD::SRL)
18937 return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
18939 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64)
18940 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
18944 if (VT == MVT::v16i8) {
18945 if (Op.getOpcode() == ISD::SHL) {
18946 // Make a large shift.
18947 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
18948 MVT::v8i16, R, ShiftAmt,
18950 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
18951 // Zero out the rightmost bits.
18952 SmallVector<SDValue, 16> V(16,
18953 DAG.getConstant(uint8_t(-1U << ShiftAmt),
18955 return DAG.getNode(ISD::AND, dl, VT, SHL,
18956 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
18958 if (Op.getOpcode() == ISD::SRL) {
18959 // Make a large shift.
18960 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
18961 MVT::v8i16, R, ShiftAmt,
18963 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
18964 // Zero out the leftmost bits.
18965 SmallVector<SDValue, 16> V(16,
18966 DAG.getConstant(uint8_t(-1U) >> ShiftAmt,
18968 return DAG.getNode(ISD::AND, dl, VT, SRL,
18969 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
18971 if (Op.getOpcode() == ISD::SRA) {
18972 if (ShiftAmt == 7) {
18973 // R s>> 7 === R s< 0
18974 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
18975 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
18978 // R s>> a === ((R u>> a) ^ m) - m
18979 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
18980 SmallVector<SDValue, 16> V(16, DAG.getConstant(128 >> ShiftAmt,
18982 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
18983 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
18984 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
18987 llvm_unreachable("Unknown shift opcode.");
18990 if (Subtarget->hasInt256() && VT == MVT::v32i8) {
18991 if (Op.getOpcode() == ISD::SHL) {
18992 // Make a large shift.
18993 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
18994 MVT::v16i16, R, ShiftAmt,
18996 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
18997 // Zero out the rightmost bits.
18998 SmallVector<SDValue, 32> V(32,
18999 DAG.getConstant(uint8_t(-1U << ShiftAmt),
19001 return DAG.getNode(ISD::AND, dl, VT, SHL,
19002 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
19004 if (Op.getOpcode() == ISD::SRL) {
19005 // Make a large shift.
19006 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
19007 MVT::v16i16, R, ShiftAmt,
19009 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
19010 // Zero out the leftmost bits.
19011 SmallVector<SDValue, 32> V(32,
19012 DAG.getConstant(uint8_t(-1U) >> ShiftAmt,
19014 return DAG.getNode(ISD::AND, dl, VT, SRL,
19015 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V));
19017 if (Op.getOpcode() == ISD::SRA) {
19018 if (ShiftAmt == 7) {
19019 // R s>> 7 === R s< 0
19020 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
19021 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
19024 // R s>> a === ((R u>> a) ^ m) - m
19025 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
19026 SmallVector<SDValue, 32> V(32, DAG.getConstant(128 >> ShiftAmt,
19028 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V);
19029 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
19030 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
19033 llvm_unreachable("Unknown shift opcode.");
19038 // Special case in 32-bit mode, where i64 is expanded into high and low parts.
19039 if (!Subtarget->is64Bit() &&
19040 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) &&
19041 Amt.getOpcode() == ISD::BITCAST &&
19042 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
19043 Amt = Amt.getOperand(0);
19044 unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
19045 VT.getVectorNumElements();
19046 unsigned RatioInLog2 = Log2_32_Ceil(Ratio);
19047 uint64_t ShiftAmt = 0;
19048 for (unsigned i = 0; i != Ratio; ++i) {
19049 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Amt.getOperand(i));
19053 ShiftAmt |= C->getZExtValue() << (i * (1 << (6 - RatioInLog2)));
19055 // Check remaining shift amounts.
19056 for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
19057 uint64_t ShAmt = 0;
19058 for (unsigned j = 0; j != Ratio; ++j) {
19059 ConstantSDNode *C =
19060 dyn_cast<ConstantSDNode>(Amt.getOperand(i + j));
19064 ShAmt |= C->getZExtValue() << (j * (1 << (6 - RatioInLog2)));
19066 if (ShAmt != ShiftAmt)
19069 switch (Op.getOpcode()) {
19071 llvm_unreachable("Unknown shift opcode!");
19073 return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
19076 return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
19079 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
19087 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
19088 const X86Subtarget* Subtarget) {
19089 MVT VT = Op.getSimpleValueType();
19091 SDValue R = Op.getOperand(0);
19092 SDValue Amt = Op.getOperand(1);
19094 if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) ||
19095 VT == MVT::v4i32 || VT == MVT::v8i16 ||
19096 (Subtarget->hasInt256() &&
19097 ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) ||
19098 VT == MVT::v8i32 || VT == MVT::v16i16)) ||
19099 (Subtarget->hasAVX512() && (VT == MVT::v8i64 || VT == MVT::v16i32))) {
19101 EVT EltVT = VT.getVectorElementType();
19103 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Amt)) {
19104 // Check if this build_vector node is doing a splat.
19105 // If so, then set BaseShAmt equal to the splat value.
19106 BaseShAmt = BV->getSplatValue();
19107 if (BaseShAmt && BaseShAmt.getOpcode() == ISD::UNDEF)
19108 BaseShAmt = SDValue();
19110 if (Amt.getOpcode() == ISD::EXTRACT_SUBVECTOR)
19111 Amt = Amt.getOperand(0);
19113 ShuffleVectorSDNode *SVN = dyn_cast<ShuffleVectorSDNode>(Amt);
19114 if (SVN && SVN->isSplat()) {
19115 unsigned SplatIdx = (unsigned)SVN->getSplatIndex();
19116 SDValue InVec = Amt.getOperand(0);
19117 if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
19118 assert((SplatIdx < InVec.getValueType().getVectorNumElements()) &&
19119 "Unexpected shuffle index found!");
19120 BaseShAmt = InVec.getOperand(SplatIdx);
19121 } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) {
19122 if (ConstantSDNode *C =
19123 dyn_cast<ConstantSDNode>(InVec.getOperand(2))) {
19124 if (C->getZExtValue() == SplatIdx)
19125 BaseShAmt = InVec.getOperand(1);
19130 // Avoid introducing an extract element from a shuffle.
19131 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InVec,
19132 DAG.getIntPtrConstant(SplatIdx));
19136 if (BaseShAmt.getNode()) {
19137 assert(EltVT.bitsLE(MVT::i64) && "Unexpected element type!");
19138 if (EltVT != MVT::i64 && EltVT.bitsGT(MVT::i32))
19139 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt);
19140 else if (EltVT.bitsLT(MVT::i32))
19141 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt);
19143 switch (Op.getOpcode()) {
19145 llvm_unreachable("Unknown shift opcode!");
19147 switch (VT.SimpleTy) {
19148 default: return SDValue();
19157 return getTargetVShiftNode(X86ISD::VSHLI, dl, VT, R, BaseShAmt, DAG);
19160 switch (VT.SimpleTy) {
19161 default: return SDValue();
19168 return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, R, BaseShAmt, DAG);
19171 switch (VT.SimpleTy) {
19172 default: return SDValue();
19181 return getTargetVShiftNode(X86ISD::VSRLI, dl, VT, R, BaseShAmt, DAG);
19187 // Special case in 32-bit mode, where i64 is expanded into high and low parts.
19188 if (!Subtarget->is64Bit() &&
19189 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64) ||
19190 (Subtarget->hasAVX512() && VT == MVT::v8i64)) &&
19191 Amt.getOpcode() == ISD::BITCAST &&
19192 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
19193 Amt = Amt.getOperand(0);
19194 unsigned Ratio = Amt.getSimpleValueType().getVectorNumElements() /
19195 VT.getVectorNumElements();
19196 std::vector<SDValue> Vals(Ratio);
19197 for (unsigned i = 0; i != Ratio; ++i)
19198 Vals[i] = Amt.getOperand(i);
19199 for (unsigned i = Ratio; i != Amt.getNumOperands(); i += Ratio) {
19200 for (unsigned j = 0; j != Ratio; ++j)
19201 if (Vals[j] != Amt.getOperand(i + j))
19204 switch (Op.getOpcode()) {
19206 llvm_unreachable("Unknown shift opcode!");
19208 return DAG.getNode(X86ISD::VSHL, dl, VT, R, Op.getOperand(1));
19210 return DAG.getNode(X86ISD::VSRL, dl, VT, R, Op.getOperand(1));
19212 return DAG.getNode(X86ISD::VSRA, dl, VT, R, Op.getOperand(1));
19219 static SDValue LowerShift(SDValue Op, const X86Subtarget* Subtarget,
19220 SelectionDAG &DAG) {
19221 MVT VT = Op.getSimpleValueType();
19223 SDValue R = Op.getOperand(0);
19224 SDValue Amt = Op.getOperand(1);
19227 assert(VT.isVector() && "Custom lowering only for vector shifts!");
19228 assert(Subtarget->hasSSE2() && "Only custom lower when we have SSE2!");
19230 V = LowerScalarImmediateShift(Op, DAG, Subtarget);
19234 V = LowerScalarVariableShift(Op, DAG, Subtarget);
19238 if (Subtarget->hasAVX512() && (VT == MVT::v16i32 || VT == MVT::v8i64))
19240 // AVX2 has VPSLLV/VPSRAV/VPSRLV.
19241 if (Subtarget->hasInt256()) {
19242 if (Op.getOpcode() == ISD::SRL &&
19243 (VT == MVT::v2i64 || VT == MVT::v4i32 ||
19244 VT == MVT::v4i64 || VT == MVT::v8i32))
19246 if (Op.getOpcode() == ISD::SHL &&
19247 (VT == MVT::v2i64 || VT == MVT::v4i32 ||
19248 VT == MVT::v4i64 || VT == MVT::v8i32))
19250 if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32))
19254 // If possible, lower this packed shift into a vector multiply instead of
19255 // expanding it into a sequence of scalar shifts.
19256 // Do this only if the vector shift count is a constant build_vector.
19257 if (Op.getOpcode() == ISD::SHL &&
19258 (VT == MVT::v8i16 || VT == MVT::v4i32 ||
19259 (Subtarget->hasInt256() && VT == MVT::v16i16)) &&
19260 ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
19261 SmallVector<SDValue, 8> Elts;
19262 EVT SVT = VT.getScalarType();
19263 unsigned SVTBits = SVT.getSizeInBits();
19264 const APInt &One = APInt(SVTBits, 1);
19265 unsigned NumElems = VT.getVectorNumElements();
19267 for (unsigned i=0; i !=NumElems; ++i) {
19268 SDValue Op = Amt->getOperand(i);
19269 if (Op->getOpcode() == ISD::UNDEF) {
19270 Elts.push_back(Op);
19274 ConstantSDNode *ND = cast<ConstantSDNode>(Op);
19275 const APInt &C = APInt(SVTBits, ND->getAPIntValue().getZExtValue());
19276 uint64_t ShAmt = C.getZExtValue();
19277 if (ShAmt >= SVTBits) {
19278 Elts.push_back(DAG.getUNDEF(SVT));
19281 Elts.push_back(DAG.getConstant(One.shl(ShAmt), SVT));
19283 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts);
19284 return DAG.getNode(ISD::MUL, dl, VT, R, BV);
19287 // Lower SHL with variable shift amount.
19288 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
19289 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT));
19291 Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT));
19292 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op);
19293 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
19294 return DAG.getNode(ISD::MUL, dl, VT, Op, R);
19297 // If possible, lower this shift as a sequence of two shifts by
19298 // constant plus a MOVSS/MOVSD instead of scalarizing it.
19300 // (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
19302 // Could be rewritten as:
19303 // (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
19305 // The advantage is that the two shifts from the example would be
19306 // lowered as X86ISD::VSRLI nodes. This would be cheaper than scalarizing
19307 // the vector shift into four scalar shifts plus four pairs of vector
19309 if ((VT == MVT::v8i16 || VT == MVT::v4i32) &&
19310 ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
19311 unsigned TargetOpcode = X86ISD::MOVSS;
19312 bool CanBeSimplified;
19313 // The splat value for the first packed shift (the 'X' from the example).
19314 SDValue Amt1 = Amt->getOperand(0);
19315 // The splat value for the second packed shift (the 'Y' from the example).
19316 SDValue Amt2 = (VT == MVT::v4i32) ? Amt->getOperand(1) :
19317 Amt->getOperand(2);
19319 // See if it is possible to replace this node with a sequence of
19320 // two shifts followed by a MOVSS/MOVSD
19321 if (VT == MVT::v4i32) {
19322 // Check if it is legal to use a MOVSS.
19323 CanBeSimplified = Amt2 == Amt->getOperand(2) &&
19324 Amt2 == Amt->getOperand(3);
19325 if (!CanBeSimplified) {
19326 // Otherwise, check if we can still simplify this node using a MOVSD.
19327 CanBeSimplified = Amt1 == Amt->getOperand(1) &&
19328 Amt->getOperand(2) == Amt->getOperand(3);
19329 TargetOpcode = X86ISD::MOVSD;
19330 Amt2 = Amt->getOperand(2);
19333 // Do similar checks for the case where the machine value type
19335 CanBeSimplified = Amt1 == Amt->getOperand(1);
19336 for (unsigned i=3; i != 8 && CanBeSimplified; ++i)
19337 CanBeSimplified = Amt2 == Amt->getOperand(i);
19339 if (!CanBeSimplified) {
19340 TargetOpcode = X86ISD::MOVSD;
19341 CanBeSimplified = true;
19342 Amt2 = Amt->getOperand(4);
19343 for (unsigned i=0; i != 4 && CanBeSimplified; ++i)
19344 CanBeSimplified = Amt1 == Amt->getOperand(i);
19345 for (unsigned j=4; j != 8 && CanBeSimplified; ++j)
19346 CanBeSimplified = Amt2 == Amt->getOperand(j);
19350 if (CanBeSimplified && isa<ConstantSDNode>(Amt1) &&
19351 isa<ConstantSDNode>(Amt2)) {
19352 // Replace this node with two shifts followed by a MOVSS/MOVSD.
19353 EVT CastVT = MVT::v4i32;
19355 DAG.getConstant(cast<ConstantSDNode>(Amt1)->getAPIntValue(), VT);
19356 SDValue Shift1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat1);
19358 DAG.getConstant(cast<ConstantSDNode>(Amt2)->getAPIntValue(), VT);
19359 SDValue Shift2 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat2);
19360 if (TargetOpcode == X86ISD::MOVSD)
19361 CastVT = MVT::v2i64;
19362 SDValue BitCast1 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift1);
19363 SDValue BitCast2 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift2);
19364 SDValue Result = getTargetShuffleNode(TargetOpcode, dl, CastVT, BitCast2,
19366 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
19370 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
19371 assert(Subtarget->hasSSE2() && "Need SSE2 for pslli/pcmpeq.");
19374 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT));
19375 Op = DAG.getNode(ISD::BITCAST, dl, VT, Op);
19377 // Turn 'a' into a mask suitable for VSELECT
19378 SDValue VSelM = DAG.getConstant(0x80, VT);
19379 SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
19380 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
19382 SDValue CM1 = DAG.getConstant(0x0f, VT);
19383 SDValue CM2 = DAG.getConstant(0x3f, VT);
19385 // r = VSELECT(r, psllw(r & (char16)15, 4), a);
19386 SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1);
19387 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG);
19388 M = DAG.getNode(ISD::BITCAST, dl, VT, M);
19389 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
19392 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
19393 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
19394 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
19396 // r = VSELECT(r, psllw(r & (char16)63, 2), a);
19397 M = DAG.getNode(ISD::AND, dl, VT, R, CM2);
19398 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG);
19399 M = DAG.getNode(ISD::BITCAST, dl, VT, M);
19400 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
19403 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
19404 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
19405 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
19407 // return VSELECT(r, r+r, a);
19408 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel,
19409 DAG.getNode(ISD::ADD, dl, VT, R, R), R);
19413 // It's worth extending once and using the v8i32 shifts for 16-bit types, but
19414 // the extra overheads to get from v16i8 to v8i32 make the existing SSE
19415 // solution better.
19416 if (Subtarget->hasInt256() && VT == MVT::v8i16) {
19417 MVT NewVT = VT == MVT::v8i16 ? MVT::v8i32 : MVT::v16i16;
19419 Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
19420 R = DAG.getNode(ExtOpc, dl, NewVT, R);
19421 Amt = DAG.getNode(ISD::ANY_EXTEND, dl, NewVT, Amt);
19422 return DAG.getNode(ISD::TRUNCATE, dl, VT,
19423 DAG.getNode(Op.getOpcode(), dl, NewVT, R, Amt));
19426 // Decompose 256-bit shifts into smaller 128-bit shifts.
19427 if (VT.is256BitVector()) {
19428 unsigned NumElems = VT.getVectorNumElements();
19429 MVT EltVT = VT.getVectorElementType();
19430 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
19432 // Extract the two vectors
19433 SDValue V1 = Extract128BitVector(R, 0, DAG, dl);
19434 SDValue V2 = Extract128BitVector(R, NumElems/2, DAG, dl);
19436 // Recreate the shift amount vectors
19437 SDValue Amt1, Amt2;
19438 if (Amt.getOpcode() == ISD::BUILD_VECTOR) {
19439 // Constant shift amount
19440 SmallVector<SDValue, 4> Amt1Csts;
19441 SmallVector<SDValue, 4> Amt2Csts;
19442 for (unsigned i = 0; i != NumElems/2; ++i)
19443 Amt1Csts.push_back(Amt->getOperand(i));
19444 for (unsigned i = NumElems/2; i != NumElems; ++i)
19445 Amt2Csts.push_back(Amt->getOperand(i));
19447 Amt1 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt1Csts);
19448 Amt2 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt2Csts);
19450 // Variable shift amount
19451 Amt1 = Extract128BitVector(Amt, 0, DAG, dl);
19452 Amt2 = Extract128BitVector(Amt, NumElems/2, DAG, dl);
19455 // Issue new vector shifts for the smaller types
19456 V1 = DAG.getNode(Op.getOpcode(), dl, NewVT, V1, Amt1);
19457 V2 = DAG.getNode(Op.getOpcode(), dl, NewVT, V2, Amt2);
19459 // Concatenate the result back
19460 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2);
19466 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
19467 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
19468 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
19469 // looks for this combo and may remove the "setcc" instruction if the "setcc"
19470 // has only one use.
19471 SDNode *N = Op.getNode();
19472 SDValue LHS = N->getOperand(0);
19473 SDValue RHS = N->getOperand(1);
19474 unsigned BaseOp = 0;
19477 switch (Op.getOpcode()) {
19478 default: llvm_unreachable("Unknown ovf instruction!");
19480 // A subtract of one will be selected as a INC. Note that INC doesn't
19481 // set CF, so we can't do this for UADDO.
19482 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
19484 BaseOp = X86ISD::INC;
19485 Cond = X86::COND_O;
19488 BaseOp = X86ISD::ADD;
19489 Cond = X86::COND_O;
19492 BaseOp = X86ISD::ADD;
19493 Cond = X86::COND_B;
19496 // A subtract of one will be selected as a DEC. Note that DEC doesn't
19497 // set CF, so we can't do this for USUBO.
19498 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS))
19500 BaseOp = X86ISD::DEC;
19501 Cond = X86::COND_O;
19504 BaseOp = X86ISD::SUB;
19505 Cond = X86::COND_O;
19508 BaseOp = X86ISD::SUB;
19509 Cond = X86::COND_B;
19512 BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL;
19513 Cond = X86::COND_O;
19515 case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs
19516 if (N->getValueType(0) == MVT::i8) {
19517 BaseOp = X86ISD::UMUL8;
19518 Cond = X86::COND_O;
19521 SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0),
19523 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
19526 DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
19527 DAG.getConstant(X86::COND_O, MVT::i32),
19528 SDValue(Sum.getNode(), 2));
19530 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
19534 // Also sets EFLAGS.
19535 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
19536 SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
19539 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
19540 DAG.getConstant(Cond, MVT::i32),
19541 SDValue(Sum.getNode(), 1));
19543 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
19546 // Sign extension of the low part of vector elements. This may be used either
19547 // when sign extend instructions are not available or if the vector element
19548 // sizes already match the sign-extended size. If the vector elements are in
19549 // their pre-extended size and sign extend instructions are available, that will
19550 // be handled by LowerSIGN_EXTEND.
19551 SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
19552 SelectionDAG &DAG) const {
19554 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
19555 MVT VT = Op.getSimpleValueType();
19557 if (!Subtarget->hasSSE2() || !VT.isVector())
19560 unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
19561 ExtraVT.getScalarType().getSizeInBits();
19563 switch (VT.SimpleTy) {
19564 default: return SDValue();
19567 if (!Subtarget->hasFp256())
19569 if (!Subtarget->hasInt256()) {
19570 // needs to be split
19571 unsigned NumElems = VT.getVectorNumElements();
19573 // Extract the LHS vectors
19574 SDValue LHS = Op.getOperand(0);
19575 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, dl);
19576 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, dl);
19578 MVT EltVT = VT.getVectorElementType();
19579 EVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
19581 EVT ExtraEltVT = ExtraVT.getVectorElementType();
19582 unsigned ExtraNumElems = ExtraVT.getVectorNumElements();
19583 ExtraVT = EVT::getVectorVT(*DAG.getContext(), ExtraEltVT,
19585 SDValue Extra = DAG.getValueType(ExtraVT);
19587 LHS1 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, Extra);
19588 LHS2 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, Extra);
19590 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, LHS1, LHS2);
19595 SDValue Op0 = Op.getOperand(0);
19597 // This is a sign extension of some low part of vector elements without
19598 // changing the size of the vector elements themselves:
19599 // Shift-Left + Shift-Right-Algebraic.
19600 SDValue Shl = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Op0,
19602 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Shl, BitsDiff,
19608 /// Returns true if the operand type is exactly twice the native width, and
19609 /// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
19610 /// Used to know whether to use cmpxchg8/16b when expanding atomic operations
19611 /// (otherwise we leave them alone to become __sync_fetch_and_... calls).
19612 bool X86TargetLowering::needsCmpXchgNb(const Type *MemType) const {
19613 unsigned OpWidth = MemType->getPrimitiveSizeInBits();
19616 return !Subtarget->is64Bit(); // FIXME this should be Subtarget.hasCmpxchg8b
19617 else if (OpWidth == 128)
19618 return Subtarget->hasCmpxchg16b();
19623 bool X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
19624 return needsCmpXchgNb(SI->getValueOperand()->getType());
19627 // Note: this turns large loads into lock cmpxchg8b/16b.
19628 // FIXME: On 32 bits x86, fild/movq might be faster than lock cmpxchg8b.
19629 bool X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
19630 auto PTy = cast<PointerType>(LI->getPointerOperand()->getType());
19631 return needsCmpXchgNb(PTy->getElementType());
19634 bool X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
19635 unsigned NativeWidth = Subtarget->is64Bit() ? 64 : 32;
19636 const Type *MemType = AI->getType();
19638 // If the operand is too big, we must see if cmpxchg8/16b is available
19639 // and default to library calls otherwise.
19640 if (MemType->getPrimitiveSizeInBits() > NativeWidth)
19641 return needsCmpXchgNb(MemType);
19643 AtomicRMWInst::BinOp Op = AI->getOperation();
19646 llvm_unreachable("Unknown atomic operation");
19647 case AtomicRMWInst::Xchg:
19648 case AtomicRMWInst::Add:
19649 case AtomicRMWInst::Sub:
19650 // It's better to use xadd, xsub or xchg for these in all cases.
19652 case AtomicRMWInst::Or:
19653 case AtomicRMWInst::And:
19654 case AtomicRMWInst::Xor:
19655 // If the atomicrmw's result isn't actually used, we can just add a "lock"
19656 // prefix to a normal instruction for these operations.
19657 return !AI->use_empty();
19658 case AtomicRMWInst::Nand:
19659 case AtomicRMWInst::Max:
19660 case AtomicRMWInst::Min:
19661 case AtomicRMWInst::UMax:
19662 case AtomicRMWInst::UMin:
19663 // These always require a non-trivial set of data operations on x86. We must
19664 // use a cmpxchg loop.
19669 static bool hasMFENCE(const X86Subtarget& Subtarget) {
19670 // Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
19671 // no-sse2). There isn't any reason to disable it if the target processor
19673 return Subtarget.hasSSE2() || Subtarget.is64Bit();
19677 X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
19678 unsigned NativeWidth = Subtarget->is64Bit() ? 64 : 32;
19679 const Type *MemType = AI->getType();
19680 // Accesses larger than the native width are turned into cmpxchg/libcalls, so
19681 // there is no benefit in turning such RMWs into loads, and it is actually
19682 // harmful as it introduces a mfence.
19683 if (MemType->getPrimitiveSizeInBits() > NativeWidth)
19686 auto Builder = IRBuilder<>(AI);
19687 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
19688 auto SynchScope = AI->getSynchScope();
19689 // We must restrict the ordering to avoid generating loads with Release or
19690 // ReleaseAcquire orderings.
19691 auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
19692 auto Ptr = AI->getPointerOperand();
19694 // Before the load we need a fence. Here is an example lifted from
19695 // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
19698 // x.store(1, relaxed);
19699 // r1 = y.fetch_add(0, release);
19701 // y.fetch_add(42, acquire);
19702 // r2 = x.load(relaxed);
19703 // r1 = r2 = 0 is impossible, but becomes possible if the idempotent rmw is
19704 // lowered to just a load without a fence. A mfence flushes the store buffer,
19705 // making the optimization clearly correct.
19706 // FIXME: it is required if isAtLeastRelease(Order) but it is not clear
19707 // otherwise, we might be able to be more agressive on relaxed idempotent
19708 // rmw. In practice, they do not look useful, so we don't try to be
19709 // especially clever.
19710 if (SynchScope == SingleThread) {
19711 // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at
19712 // the IR level, so we must wrap it in an intrinsic.
19714 } else if (hasMFENCE(*Subtarget)) {
19715 Function *MFence = llvm::Intrinsic::getDeclaration(M,
19716 Intrinsic::x86_sse2_mfence);
19717 Builder.CreateCall(MFence);
19719 // FIXME: it might make sense to use a locked operation here but on a
19720 // different cache-line to prevent cache-line bouncing. In practice it
19721 // is probably a small win, and x86 processors without mfence are rare
19722 // enough that we do not bother.
19726 // Finally we can emit the atomic load.
19727 LoadInst *Loaded = Builder.CreateAlignedLoad(Ptr,
19728 AI->getType()->getPrimitiveSizeInBits());
19729 Loaded->setAtomic(Order, SynchScope);
19730 AI->replaceAllUsesWith(Loaded);
19731 AI->eraseFromParent();
19735 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget *Subtarget,
19736 SelectionDAG &DAG) {
19738 AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
19739 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
19740 SynchronizationScope FenceScope = static_cast<SynchronizationScope>(
19741 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
19743 // The only fence that needs an instruction is a sequentially-consistent
19744 // cross-thread fence.
19745 if (FenceOrdering == SequentiallyConsistent && FenceScope == CrossThread) {
19746 if (hasMFENCE(*Subtarget))
19747 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
19749 SDValue Chain = Op.getOperand(0);
19750 SDValue Zero = DAG.getConstant(0, MVT::i32);
19752 DAG.getRegister(X86::ESP, MVT::i32), // Base
19753 DAG.getTargetConstant(1, MVT::i8), // Scale
19754 DAG.getRegister(0, MVT::i32), // Index
19755 DAG.getTargetConstant(0, MVT::i32), // Disp
19756 DAG.getRegister(0, MVT::i32), // Segment.
19760 SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops);
19761 return SDValue(Res, 0);
19764 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
19765 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
19768 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget,
19769 SelectionDAG &DAG) {
19770 MVT T = Op.getSimpleValueType();
19774 switch(T.SimpleTy) {
19775 default: llvm_unreachable("Invalid value type!");
19776 case MVT::i8: Reg = X86::AL; size = 1; break;
19777 case MVT::i16: Reg = X86::AX; size = 2; break;
19778 case MVT::i32: Reg = X86::EAX; size = 4; break;
19780 assert(Subtarget->is64Bit() && "Node not type legal!");
19781 Reg = X86::RAX; size = 8;
19784 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
19785 Op.getOperand(2), SDValue());
19786 SDValue Ops[] = { cpIn.getValue(0),
19789 DAG.getTargetConstant(size, MVT::i8),
19790 cpIn.getValue(1) };
19791 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
19792 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
19793 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
19797 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
19798 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
19799 MVT::i32, cpOut.getValue(2));
19800 SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
19801 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
19803 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), cpOut);
19804 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
19805 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), EFLAGS.getValue(1));
19809 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget *Subtarget,
19810 SelectionDAG &DAG) {
19811 MVT SrcVT = Op.getOperand(0).getSimpleValueType();
19812 MVT DstVT = Op.getSimpleValueType();
19814 if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) {
19815 assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
19816 if (DstVT != MVT::f64)
19817 // This conversion needs to be expanded.
19820 SDValue InVec = Op->getOperand(0);
19822 unsigned NumElts = SrcVT.getVectorNumElements();
19823 EVT SVT = SrcVT.getVectorElementType();
19825 // Widen the vector in input in the case of MVT::v2i32.
19826 // Example: from MVT::v2i32 to MVT::v4i32.
19827 SmallVector<SDValue, 16> Elts;
19828 for (unsigned i = 0, e = NumElts; i != e; ++i)
19829 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, InVec,
19830 DAG.getIntPtrConstant(i)));
19832 // Explicitly mark the extra elements as Undef.
19833 Elts.append(NumElts, DAG.getUNDEF(SVT));
19835 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
19836 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Elts);
19837 SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV);
19838 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64,
19839 DAG.getIntPtrConstant(0));
19842 assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
19843 Subtarget->hasMMX() && "Unexpected custom BITCAST");
19844 assert((DstVT == MVT::i64 ||
19845 (DstVT.isVector() && DstVT.getSizeInBits()==64)) &&
19846 "Unexpected custom BITCAST");
19847 // i64 <=> MMX conversions are Legal.
19848 if (SrcVT==MVT::i64 && DstVT.isVector())
19850 if (DstVT==MVT::i64 && SrcVT.isVector())
19852 // MMX <=> MMX conversions are Legal.
19853 if (SrcVT.isVector() && DstVT.isVector())
19855 // All other conversions need to be expanded.
19859 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget *Subtarget,
19860 SelectionDAG &DAG) {
19861 SDNode *Node = Op.getNode();
19864 Op = Op.getOperand(0);
19865 EVT VT = Op.getValueType();
19866 assert((VT.is128BitVector() || VT.is256BitVector()) &&
19867 "CTPOP lowering only implemented for 128/256-bit wide vector types");
19869 unsigned NumElts = VT.getVectorNumElements();
19870 EVT EltVT = VT.getVectorElementType();
19871 unsigned Len = EltVT.getSizeInBits();
19873 // This is the vectorized version of the "best" algorithm from
19874 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
19875 // with a minor tweak to use a series of adds + shifts instead of vector
19876 // multiplications. Implemented for the v2i64, v4i64, v4i32, v8i32 types:
19878 // v2i64, v4i64, v4i32 => Only profitable w/ popcnt disabled
19879 // v8i32 => Always profitable
19881 // FIXME: There a couple of possible improvements:
19883 // 1) Support for i8 and i16 vectors (needs measurements if popcnt enabled).
19884 // 2) Use strategies from http://wm.ite.pl/articles/sse-popcount.html
19886 assert(EltVT.isInteger() && (Len == 32 || Len == 64) && Len % 8 == 0 &&
19887 "CTPOP not implemented for this vector element type.");
19889 // X86 canonicalize ANDs to vXi64, generate the appropriate bitcasts to avoid
19890 // extra legalization.
19891 bool NeedsBitcast = EltVT == MVT::i32;
19892 MVT BitcastVT = VT.is256BitVector() ? MVT::v4i64 : MVT::v2i64;
19894 SDValue Cst55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), EltVT);
19895 SDValue Cst33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), EltVT);
19896 SDValue Cst0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), EltVT);
19898 // v = v - ((v >> 1) & 0x55555555...)
19899 SmallVector<SDValue, 8> Ones(NumElts, DAG.getConstant(1, EltVT));
19900 SDValue OnesV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ones);
19901 SDValue Srl = DAG.getNode(ISD::SRL, dl, VT, Op, OnesV);
19903 Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl);
19905 SmallVector<SDValue, 8> Mask55(NumElts, Cst55);
19906 SDValue M55 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask55);
19908 M55 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M55);
19910 SDValue And = DAG.getNode(ISD::AND, dl, Srl.getValueType(), Srl, M55);
19911 if (VT != And.getValueType())
19912 And = DAG.getNode(ISD::BITCAST, dl, VT, And);
19913 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Op, And);
19915 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
19916 SmallVector<SDValue, 8> Mask33(NumElts, Cst33);
19917 SDValue M33 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask33);
19918 SmallVector<SDValue, 8> Twos(NumElts, DAG.getConstant(2, EltVT));
19919 SDValue TwosV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Twos);
19921 Srl = DAG.getNode(ISD::SRL, dl, VT, Sub, TwosV);
19922 if (NeedsBitcast) {
19923 Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl);
19924 M33 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M33);
19925 Sub = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Sub);
19928 SDValue AndRHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Srl, M33);
19929 SDValue AndLHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Sub, M33);
19930 if (VT != AndRHS.getValueType()) {
19931 AndRHS = DAG.getNode(ISD::BITCAST, dl, VT, AndRHS);
19932 AndLHS = DAG.getNode(ISD::BITCAST, dl, VT, AndLHS);
19934 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, AndLHS, AndRHS);
19936 // v = (v + (v >> 4)) & 0x0F0F0F0F...
19937 SmallVector<SDValue, 8> Fours(NumElts, DAG.getConstant(4, EltVT));
19938 SDValue FoursV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Fours);
19939 Srl = DAG.getNode(ISD::SRL, dl, VT, Add, FoursV);
19940 Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl);
19942 SmallVector<SDValue, 8> Mask0F(NumElts, Cst0F);
19943 SDValue M0F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask0F);
19944 if (NeedsBitcast) {
19945 Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add);
19946 M0F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M0F);
19948 And = DAG.getNode(ISD::AND, dl, M0F.getValueType(), Add, M0F);
19949 if (VT != And.getValueType())
19950 And = DAG.getNode(ISD::BITCAST, dl, VT, And);
19952 // The algorithm mentioned above uses:
19953 // v = (v * 0x01010101...) >> (Len - 8)
19955 // Change it to use vector adds + vector shifts which yield faster results on
19956 // Haswell than using vector integer multiplication.
19958 // For i32 elements:
19959 // v = v + (v >> 8)
19960 // v = v + (v >> 16)
19962 // For i64 elements:
19963 // v = v + (v >> 8)
19964 // v = v + (v >> 16)
19965 // v = v + (v >> 32)
19968 SmallVector<SDValue, 8> Csts;
19969 for (unsigned i = 8; i <= Len/2; i *= 2) {
19970 Csts.assign(NumElts, DAG.getConstant(i, EltVT));
19971 SDValue CstsV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Csts);
19972 Srl = DAG.getNode(ISD::SRL, dl, VT, Add, CstsV);
19973 Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl);
19977 // The result is on the least significant 6-bits on i32 and 7-bits on i64.
19978 SDValue Cst3F = DAG.getConstant(APInt(Len, Len == 32 ? 0x3F : 0x7F), EltVT);
19979 SmallVector<SDValue, 8> Cst3FV(NumElts, Cst3F);
19980 SDValue M3F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Cst3FV);
19981 if (NeedsBitcast) {
19982 Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add);
19983 M3F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M3F);
19985 And = DAG.getNode(ISD::AND, dl, M3F.getValueType(), Add, M3F);
19986 if (VT != And.getValueType())
19987 And = DAG.getNode(ISD::BITCAST, dl, VT, And);
19992 static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
19993 SDNode *Node = Op.getNode();
19995 EVT T = Node->getValueType(0);
19996 SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
19997 DAG.getConstant(0, T), Node->getOperand(2));
19998 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
19999 cast<AtomicSDNode>(Node)->getMemoryVT(),
20000 Node->getOperand(0),
20001 Node->getOperand(1), negOp,
20002 cast<AtomicSDNode>(Node)->getMemOperand(),
20003 cast<AtomicSDNode>(Node)->getOrdering(),
20004 cast<AtomicSDNode>(Node)->getSynchScope());
20007 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
20008 SDNode *Node = Op.getNode();
20010 EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
20012 // Convert seq_cst store -> xchg
20013 // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
20014 // FIXME: On 32-bit, store -> fist or movq would be more efficient
20015 // (The only way to get a 16-byte store is cmpxchg16b)
20016 // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
20017 if (cast<AtomicSDNode>(Node)->getOrdering() == SequentiallyConsistent ||
20018 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
20019 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
20020 cast<AtomicSDNode>(Node)->getMemoryVT(),
20021 Node->getOperand(0),
20022 Node->getOperand(1), Node->getOperand(2),
20023 cast<AtomicSDNode>(Node)->getMemOperand(),
20024 cast<AtomicSDNode>(Node)->getOrdering(),
20025 cast<AtomicSDNode>(Node)->getSynchScope());
20026 return Swap.getValue(1);
20028 // Other atomic stores have a simple pattern.
20032 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
20033 EVT VT = Op.getNode()->getSimpleValueType(0);
20035 // Let legalize expand this if it isn't a legal type yet.
20036 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
20039 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
20042 bool ExtraOp = false;
20043 switch (Op.getOpcode()) {
20044 default: llvm_unreachable("Invalid code");
20045 case ISD::ADDC: Opc = X86ISD::ADD; break;
20046 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
20047 case ISD::SUBC: Opc = X86ISD::SUB; break;
20048 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
20052 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
20054 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
20055 Op.getOperand(1), Op.getOperand(2));
20058 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget *Subtarget,
20059 SelectionDAG &DAG) {
20060 assert(Subtarget->isTargetDarwin() && Subtarget->is64Bit());
20062 // For MacOSX, we want to call an alternative entry point: __sincos_stret,
20063 // which returns the values as { float, float } (in XMM0) or
20064 // { double, double } (which is returned in XMM0, XMM1).
20066 SDValue Arg = Op.getOperand(0);
20067 EVT ArgVT = Arg.getValueType();
20068 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
20070 TargetLowering::ArgListTy Args;
20071 TargetLowering::ArgListEntry Entry;
20075 Entry.isSExt = false;
20076 Entry.isZExt = false;
20077 Args.push_back(Entry);
20079 bool isF64 = ArgVT == MVT::f64;
20080 // Only optimize x86_64 for now. i386 is a bit messy. For f32,
20081 // the small struct {f32, f32} is returned in (eax, edx). For f64,
20082 // the results are returned via SRet in memory.
20083 const char *LibcallName = isF64 ? "__sincos_stret" : "__sincosf_stret";
20084 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20085 SDValue Callee = DAG.getExternalSymbol(LibcallName, TLI.getPointerTy());
20087 Type *RetTy = isF64
20088 ? (Type*)StructType::get(ArgTy, ArgTy, nullptr)
20089 : (Type*)VectorType::get(ArgTy, 4);
20091 TargetLowering::CallLoweringInfo CLI(DAG);
20092 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
20093 .setCallee(CallingConv::C, RetTy, Callee, std::move(Args), 0);
20095 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
20098 // Returned in xmm0 and xmm1.
20099 return CallResult.first;
20101 // Returned in bits 0:31 and 32:64 xmm0.
20102 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
20103 CallResult.first, DAG.getIntPtrConstant(0));
20104 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
20105 CallResult.first, DAG.getIntPtrConstant(1));
20106 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
20107 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
20110 /// LowerOperation - Provide custom lowering hooks for some operations.
20112 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
20113 switch (Op.getOpcode()) {
20114 default: llvm_unreachable("Should not custom lower this!");
20115 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG);
20116 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
20117 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
20118 return LowerCMP_SWAP(Op, Subtarget, DAG);
20119 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG);
20120 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);
20121 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG);
20122 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
20123 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
20124 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
20125 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
20126 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
20127 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
20128 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
20129 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
20130 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
20131 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
20132 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
20133 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
20134 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
20135 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
20136 case ISD::SHL_PARTS:
20137 case ISD::SRA_PARTS:
20138 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
20139 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
20140 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
20141 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
20142 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG);
20143 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG);
20144 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG);
20145 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
20146 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
20147 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
20148 case ISD::LOAD: return LowerExtendedLoad(Op, Subtarget, DAG);
20150 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG);
20151 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
20152 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
20153 case ISD::SETCC: return LowerSETCC(Op, DAG);
20154 case ISD::SELECT: return LowerSELECT(Op, DAG);
20155 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
20156 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
20157 case ISD::VASTART: return LowerVASTART(Op, DAG);
20158 case ISD::VAARG: return LowerVAARG(Op, DAG);
20159 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
20160 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, Subtarget, DAG);
20161 case ISD::INTRINSIC_VOID:
20162 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
20163 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
20164 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
20165 case ISD::FRAME_TO_ARGS_OFFSET:
20166 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
20167 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
20168 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
20169 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
20170 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
20171 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
20172 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
20173 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
20174 case ISD::CTLZ: return LowerCTLZ(Op, DAG);
20175 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ_ZERO_UNDEF(Op, DAG);
20176 case ISD::CTTZ: return LowerCTTZ(Op, DAG);
20177 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
20178 case ISD::UMUL_LOHI:
20179 case ISD::SMUL_LOHI: return LowerMUL_LOHI(Op, Subtarget, DAG);
20182 case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
20188 case ISD::UMULO: return LowerXALUO(Op, DAG);
20189 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
20190 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG);
20194 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
20195 case ISD::ADD: return LowerADD(Op, DAG);
20196 case ISD::SUB: return LowerSUB(Op, DAG);
20197 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG);
20201 /// ReplaceNodeResults - Replace a node with an illegal result type
20202 /// with a new node built out of custom code.
20203 void X86TargetLowering::ReplaceNodeResults(SDNode *N,
20204 SmallVectorImpl<SDValue>&Results,
20205 SelectionDAG &DAG) const {
20207 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20208 switch (N->getOpcode()) {
20210 llvm_unreachable("Do not know how to custom type legalize this operation!");
20211 // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
20212 case X86ISD::FMINC:
20214 case X86ISD::FMAXC:
20215 case X86ISD::FMAX: {
20216 EVT VT = N->getValueType(0);
20217 if (VT != MVT::v2f32)
20218 llvm_unreachable("Unexpected type (!= v2f32) on FMIN/FMAX.");
20219 SDValue UNDEF = DAG.getUNDEF(VT);
20220 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
20221 N->getOperand(0), UNDEF);
20222 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
20223 N->getOperand(1), UNDEF);
20224 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));
20227 case ISD::SIGN_EXTEND_INREG:
20232 // We don't want to expand or promote these.
20239 case ISD::UDIVREM: {
20240 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
20241 Results.push_back(V);
20244 case ISD::FP_TO_SINT:
20245 case ISD::FP_TO_UINT: {
20246 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
20248 if (!IsSigned && !isIntegerTypeFTOL(SDValue(N, 0).getValueType()))
20251 std::pair<SDValue,SDValue> Vals =
20252 FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, /*IsReplace=*/ true);
20253 SDValue FIST = Vals.first, StackSlot = Vals.second;
20254 if (FIST.getNode()) {
20255 EVT VT = N->getValueType(0);
20256 // Return a load from the stack slot.
20257 if (StackSlot.getNode())
20258 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot,
20259 MachinePointerInfo(),
20260 false, false, false, 0));
20262 Results.push_back(FIST);
20266 case ISD::UINT_TO_FP: {
20267 assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
20268 if (N->getOperand(0).getValueType() != MVT::v2i32 ||
20269 N->getValueType(0) != MVT::v2f32)
20271 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64,
20273 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
20275 SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias);
20276 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
20277 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias));
20278 Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or);
20279 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
20280 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
20283 case ISD::FP_ROUND: {
20284 if (!TLI.isTypeLegal(N->getOperand(0).getValueType()))
20286 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
20287 Results.push_back(V);
20290 case ISD::INTRINSIC_W_CHAIN: {
20291 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
20293 default : llvm_unreachable("Do not know how to custom type "
20294 "legalize this intrinsic operation!");
20295 case Intrinsic::x86_rdtsc:
20296 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
20298 case Intrinsic::x86_rdtscp:
20299 return getReadTimeStampCounter(N, dl, X86ISD::RDTSCP_DAG, DAG, Subtarget,
20301 case Intrinsic::x86_rdpmc:
20302 return getReadPerformanceCounter(N, dl, DAG, Subtarget, Results);
20305 case ISD::READCYCLECOUNTER: {
20306 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
20309 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
20310 EVT T = N->getValueType(0);
20311 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");
20312 bool Regs64bit = T == MVT::i128;
20313 EVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
20314 SDValue cpInL, cpInH;
20315 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
20316 DAG.getConstant(0, HalfT));
20317 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
20318 DAG.getConstant(1, HalfT));
20319 cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
20320 Regs64bit ? X86::RAX : X86::EAX,
20322 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl,
20323 Regs64bit ? X86::RDX : X86::EDX,
20324 cpInH, cpInL.getValue(1));
20325 SDValue swapInL, swapInH;
20326 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
20327 DAG.getConstant(0, HalfT));
20328 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
20329 DAG.getConstant(1, HalfT));
20330 swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl,
20331 Regs64bit ? X86::RBX : X86::EBX,
20332 swapInL, cpInH.getValue(1));
20333 swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl,
20334 Regs64bit ? X86::RCX : X86::ECX,
20335 swapInH, swapInL.getValue(1));
20336 SDValue Ops[] = { swapInH.getValue(0),
20338 swapInH.getValue(1) };
20339 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
20340 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
20341 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG :
20342 X86ISD::LCMPXCHG8_DAG;
20343 SDValue Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, T, MMO);
20344 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
20345 Regs64bit ? X86::RAX : X86::EAX,
20346 HalfT, Result.getValue(1));
20347 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
20348 Regs64bit ? X86::RDX : X86::EDX,
20349 HalfT, cpOutL.getValue(2));
20350 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
20352 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
20353 MVT::i32, cpOutH.getValue(2));
20355 DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
20356 DAG.getConstant(X86::COND_E, MVT::i8), EFLAGS);
20357 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
20359 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
20360 Results.push_back(Success);
20361 Results.push_back(EFLAGS.getValue(1));
20364 case ISD::ATOMIC_SWAP:
20365 case ISD::ATOMIC_LOAD_ADD:
20366 case ISD::ATOMIC_LOAD_SUB:
20367 case ISD::ATOMIC_LOAD_AND:
20368 case ISD::ATOMIC_LOAD_OR:
20369 case ISD::ATOMIC_LOAD_XOR:
20370 case ISD::ATOMIC_LOAD_NAND:
20371 case ISD::ATOMIC_LOAD_MIN:
20372 case ISD::ATOMIC_LOAD_MAX:
20373 case ISD::ATOMIC_LOAD_UMIN:
20374 case ISD::ATOMIC_LOAD_UMAX:
20375 case ISD::ATOMIC_LOAD: {
20376 // Delegate to generic TypeLegalization. Situations we can really handle
20377 // should have already been dealt with by AtomicExpandPass.cpp.
20380 case ISD::BITCAST: {
20381 assert(Subtarget->hasSSE2() && "Requires at least SSE2!");
20382 EVT DstVT = N->getValueType(0);
20383 EVT SrcVT = N->getOperand(0)->getValueType(0);
20385 if (SrcVT != MVT::f64 ||
20386 (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8))
20389 unsigned NumElts = DstVT.getVectorNumElements();
20390 EVT SVT = DstVT.getVectorElementType();
20391 EVT WiderVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
20392 SDValue Expanded = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
20393 MVT::v2f64, N->getOperand(0));
20394 SDValue ToVecInt = DAG.getNode(ISD::BITCAST, dl, WiderVT, Expanded);
20396 if (ExperimentalVectorWideningLegalization) {
20397 // If we are legalizing vectors by widening, we already have the desired
20398 // legal vector type, just return it.
20399 Results.push_back(ToVecInt);
20403 SmallVector<SDValue, 8> Elts;
20404 for (unsigned i = 0, e = NumElts; i != e; ++i)
20405 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT,
20406 ToVecInt, DAG.getIntPtrConstant(i)));
20408 Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, dl, DstVT, Elts));
20413 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
20415 default: return nullptr;
20416 case X86ISD::BSF: return "X86ISD::BSF";
20417 case X86ISD::BSR: return "X86ISD::BSR";
20418 case X86ISD::SHLD: return "X86ISD::SHLD";
20419 case X86ISD::SHRD: return "X86ISD::SHRD";
20420 case X86ISD::FAND: return "X86ISD::FAND";
20421 case X86ISD::FANDN: return "X86ISD::FANDN";
20422 case X86ISD::FOR: return "X86ISD::FOR";
20423 case X86ISD::FXOR: return "X86ISD::FXOR";
20424 case X86ISD::FSRL: return "X86ISD::FSRL";
20425 case X86ISD::FILD: return "X86ISD::FILD";
20426 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
20427 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
20428 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
20429 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
20430 case X86ISD::FLD: return "X86ISD::FLD";
20431 case X86ISD::FST: return "X86ISD::FST";
20432 case X86ISD::CALL: return "X86ISD::CALL";
20433 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
20434 case X86ISD::RDTSCP_DAG: return "X86ISD::RDTSCP_DAG";
20435 case X86ISD::RDPMC_DAG: return "X86ISD::RDPMC_DAG";
20436 case X86ISD::BT: return "X86ISD::BT";
20437 case X86ISD::CMP: return "X86ISD::CMP";
20438 case X86ISD::COMI: return "X86ISD::COMI";
20439 case X86ISD::UCOMI: return "X86ISD::UCOMI";
20440 case X86ISD::CMPM: return "X86ISD::CMPM";
20441 case X86ISD::CMPMU: return "X86ISD::CMPMU";
20442 case X86ISD::SETCC: return "X86ISD::SETCC";
20443 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
20444 case X86ISD::FSETCC: return "X86ISD::FSETCC";
20445 case X86ISD::CMOV: return "X86ISD::CMOV";
20446 case X86ISD::BRCOND: return "X86ISD::BRCOND";
20447 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
20448 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
20449 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
20450 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
20451 case X86ISD::Wrapper: return "X86ISD::Wrapper";
20452 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
20453 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
20454 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
20455 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
20456 case X86ISD::PINSRB: return "X86ISD::PINSRB";
20457 case X86ISD::PINSRW: return "X86ISD::PINSRW";
20458 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
20459 case X86ISD::ANDNP: return "X86ISD::ANDNP";
20460 case X86ISD::PSIGN: return "X86ISD::PSIGN";
20461 case X86ISD::BLENDI: return "X86ISD::BLENDI";
20462 case X86ISD::SHRUNKBLEND: return "X86ISD::SHRUNKBLEND";
20463 case X86ISD::SUBUS: return "X86ISD::SUBUS";
20464 case X86ISD::HADD: return "X86ISD::HADD";
20465 case X86ISD::HSUB: return "X86ISD::HSUB";
20466 case X86ISD::FHADD: return "X86ISD::FHADD";
20467 case X86ISD::FHSUB: return "X86ISD::FHSUB";
20468 case X86ISD::UMAX: return "X86ISD::UMAX";
20469 case X86ISD::UMIN: return "X86ISD::UMIN";
20470 case X86ISD::SMAX: return "X86ISD::SMAX";
20471 case X86ISD::SMIN: return "X86ISD::SMIN";
20472 case X86ISD::FMAX: return "X86ISD::FMAX";
20473 case X86ISD::FMIN: return "X86ISD::FMIN";
20474 case X86ISD::FMAXC: return "X86ISD::FMAXC";
20475 case X86ISD::FMINC: return "X86ISD::FMINC";
20476 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
20477 case X86ISD::FRCP: return "X86ISD::FRCP";
20478 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
20479 case X86ISD::TLSBASEADDR: return "X86ISD::TLSBASEADDR";
20480 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
20481 case X86ISD::EH_SJLJ_SETJMP: return "X86ISD::EH_SJLJ_SETJMP";
20482 case X86ISD::EH_SJLJ_LONGJMP: return "X86ISD::EH_SJLJ_LONGJMP";
20483 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
20484 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
20485 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
20486 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r";
20487 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
20488 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
20489 case X86ISD::LCMPXCHG16_DAG: return "X86ISD::LCMPXCHG16_DAG";
20490 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
20491 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
20492 case X86ISD::VZEXT: return "X86ISD::VZEXT";
20493 case X86ISD::VSEXT: return "X86ISD::VSEXT";
20494 case X86ISD::VTRUNC: return "X86ISD::VTRUNC";
20495 case X86ISD::VTRUNCM: return "X86ISD::VTRUNCM";
20496 case X86ISD::VINSERT: return "X86ISD::VINSERT";
20497 case X86ISD::VFPEXT: return "X86ISD::VFPEXT";
20498 case X86ISD::VFPROUND: return "X86ISD::VFPROUND";
20499 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ";
20500 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ";
20501 case X86ISD::VSHL: return "X86ISD::VSHL";
20502 case X86ISD::VSRL: return "X86ISD::VSRL";
20503 case X86ISD::VSRA: return "X86ISD::VSRA";
20504 case X86ISD::VSHLI: return "X86ISD::VSHLI";
20505 case X86ISD::VSRLI: return "X86ISD::VSRLI";
20506 case X86ISD::VSRAI: return "X86ISD::VSRAI";
20507 case X86ISD::CMPP: return "X86ISD::CMPP";
20508 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ";
20509 case X86ISD::PCMPGT: return "X86ISD::PCMPGT";
20510 case X86ISD::PCMPEQM: return "X86ISD::PCMPEQM";
20511 case X86ISD::PCMPGTM: return "X86ISD::PCMPGTM";
20512 case X86ISD::ADD: return "X86ISD::ADD";
20513 case X86ISD::SUB: return "X86ISD::SUB";
20514 case X86ISD::ADC: return "X86ISD::ADC";
20515 case X86ISD::SBB: return "X86ISD::SBB";
20516 case X86ISD::SMUL: return "X86ISD::SMUL";
20517 case X86ISD::UMUL: return "X86ISD::UMUL";
20518 case X86ISD::SMUL8: return "X86ISD::SMUL8";
20519 case X86ISD::UMUL8: return "X86ISD::UMUL8";
20520 case X86ISD::SDIVREM8_SEXT_HREG: return "X86ISD::SDIVREM8_SEXT_HREG";
20521 case X86ISD::UDIVREM8_ZEXT_HREG: return "X86ISD::UDIVREM8_ZEXT_HREG";
20522 case X86ISD::INC: return "X86ISD::INC";
20523 case X86ISD::DEC: return "X86ISD::DEC";
20524 case X86ISD::OR: return "X86ISD::OR";
20525 case X86ISD::XOR: return "X86ISD::XOR";
20526 case X86ISD::AND: return "X86ISD::AND";
20527 case X86ISD::BEXTR: return "X86ISD::BEXTR";
20528 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
20529 case X86ISD::PTEST: return "X86ISD::PTEST";
20530 case X86ISD::TESTP: return "X86ISD::TESTP";
20531 case X86ISD::TESTM: return "X86ISD::TESTM";
20532 case X86ISD::TESTNM: return "X86ISD::TESTNM";
20533 case X86ISD::KORTEST: return "X86ISD::KORTEST";
20534 case X86ISD::PACKSS: return "X86ISD::PACKSS";
20535 case X86ISD::PACKUS: return "X86ISD::PACKUS";
20536 case X86ISD::PALIGNR: return "X86ISD::PALIGNR";
20537 case X86ISD::VALIGN: return "X86ISD::VALIGN";
20538 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
20539 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
20540 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
20541 case X86ISD::SHUFP: return "X86ISD::SHUFP";
20542 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
20543 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD";
20544 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
20545 case X86ISD::MOVLPS: return "X86ISD::MOVLPS";
20546 case X86ISD::MOVLPD: return "X86ISD::MOVLPD";
20547 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
20548 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
20549 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
20550 case X86ISD::MOVSD: return "X86ISD::MOVSD";
20551 case X86ISD::MOVSS: return "X86ISD::MOVSS";
20552 case X86ISD::UNPCKL: return "X86ISD::UNPCKL";
20553 case X86ISD::UNPCKH: return "X86ISD::UNPCKH";
20554 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
20555 case X86ISD::VBROADCASTM: return "X86ISD::VBROADCASTM";
20556 case X86ISD::VEXTRACT: return "X86ISD::VEXTRACT";
20557 case X86ISD::VPERMILPI: return "X86ISD::VPERMILPI";
20558 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128";
20559 case X86ISD::VPERMV: return "X86ISD::VPERMV";
20560 case X86ISD::VPERMV3: return "X86ISD::VPERMV3";
20561 case X86ISD::VPERMIV3: return "X86ISD::VPERMIV3";
20562 case X86ISD::VPERMI: return "X86ISD::VPERMI";
20563 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ";
20564 case X86ISD::PMULDQ: return "X86ISD::PMULDQ";
20565 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
20566 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
20567 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
20568 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
20569 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
20570 case X86ISD::WIN_FTOL: return "X86ISD::WIN_FTOL";
20571 case X86ISD::SAHF: return "X86ISD::SAHF";
20572 case X86ISD::RDRAND: return "X86ISD::RDRAND";
20573 case X86ISD::RDSEED: return "X86ISD::RDSEED";
20574 case X86ISD::FMADD: return "X86ISD::FMADD";
20575 case X86ISD::FMSUB: return "X86ISD::FMSUB";
20576 case X86ISD::FNMADD: return "X86ISD::FNMADD";
20577 case X86ISD::FNMSUB: return "X86ISD::FNMSUB";
20578 case X86ISD::FMADDSUB: return "X86ISD::FMADDSUB";
20579 case X86ISD::FMSUBADD: return "X86ISD::FMSUBADD";
20580 case X86ISD::PCMPESTRI: return "X86ISD::PCMPESTRI";
20581 case X86ISD::PCMPISTRI: return "X86ISD::PCMPISTRI";
20582 case X86ISD::XTEST: return "X86ISD::XTEST";
20583 case X86ISD::COMPRESS: return "X86ISD::COMPRESS";
20584 case X86ISD::EXPAND: return "X86ISD::EXPAND";
20585 case X86ISD::SELECT: return "X86ISD::SELECT";
20586 case X86ISD::ADDSUB: return "X86ISD::ADDSUB";
20587 case X86ISD::RCP28: return "X86ISD::RCP28";
20588 case X86ISD::RSQRT28: return "X86ISD::RSQRT28";
20592 // isLegalAddressingMode - Return true if the addressing mode represented
20593 // by AM is legal for this target, for a load/store of the specified type.
20594 bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
20596 // X86 supports extremely general addressing modes.
20597 CodeModel::Model M = getTargetMachine().getCodeModel();
20598 Reloc::Model R = getTargetMachine().getRelocationModel();
20600 // X86 allows a sign-extended 32-bit immediate field as a displacement.
20601 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
20606 Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine());
20608 // If a reference to this global requires an extra load, we can't fold it.
20609 if (isGlobalStubReference(GVFlags))
20612 // If BaseGV requires a register for the PIC base, we cannot also have a
20613 // BaseReg specified.
20614 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
20617 // If lower 4G is not available, then we must use rip-relative addressing.
20618 if ((M != CodeModel::Small || R != Reloc::Static) &&
20619 Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1))
20623 switch (AM.Scale) {
20629 // These scales always work.
20634 // These scales are formed with basereg+scalereg. Only accept if there is
20639 default: // Other stuff never works.
20646 bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
20647 unsigned Bits = Ty->getScalarSizeInBits();
20649 // 8-bit shifts are always expensive, but versions with a scalar amount aren't
20650 // particularly cheaper than those without.
20654 // On AVX2 there are new vpsllv[dq] instructions (and other shifts), that make
20655 // variable shifts just as cheap as scalar ones.
20656 if (Subtarget->hasInt256() && (Bits == 32 || Bits == 64))
20659 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
20660 // fully general vector.
20664 bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
20665 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
20667 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
20668 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
20669 return NumBits1 > NumBits2;
20672 bool X86TargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
20673 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
20676 if (!isTypeLegal(EVT::getEVT(Ty1)))
20679 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
20681 // Assuming the caller doesn't have a zeroext or signext return parameter,
20682 // truncation all the way down to i1 is valid.
20686 bool X86TargetLowering::isLegalICmpImmediate(int64_t Imm) const {
20687 return isInt<32>(Imm);
20690 bool X86TargetLowering::isLegalAddImmediate(int64_t Imm) const {
20691 // Can also use sub to handle negated immediates.
20692 return isInt<32>(Imm);
20695 bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
20696 if (!VT1.isInteger() || !VT2.isInteger())
20698 unsigned NumBits1 = VT1.getSizeInBits();
20699 unsigned NumBits2 = VT2.getSizeInBits();
20700 return NumBits1 > NumBits2;
20703 bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
20704 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
20705 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit();
20708 bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
20709 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
20710 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit();
20713 bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
20714 EVT VT1 = Val.getValueType();
20715 if (isZExtFree(VT1, VT2))
20718 if (Val.getOpcode() != ISD::LOAD)
20721 if (!VT1.isSimple() || !VT1.isInteger() ||
20722 !VT2.isSimple() || !VT2.isInteger())
20725 switch (VT1.getSimpleVT().SimpleTy) {
20730 // X86 has 8, 16, and 32-bit zero-extending loads.
20737 bool X86TargetLowering::isVectorLoadExtDesirable(SDValue) const { return true; }
20740 X86TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
20741 if (!(Subtarget->hasFMA() || Subtarget->hasFMA4()))
20744 VT = VT.getScalarType();
20746 if (!VT.isSimple())
20749 switch (VT.getSimpleVT().SimpleTy) {
20760 bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const {
20761 // i16 instructions are longer (0x66 prefix) and potentially slower.
20762 return !(VT1 == MVT::i32 && VT2 == MVT::i16);
20765 /// isShuffleMaskLegal - Targets can use this to indicate that they only
20766 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
20767 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
20768 /// are assumed to be legal.
20770 X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
20772 if (!VT.isSimple())
20775 MVT SVT = VT.getSimpleVT();
20777 // Very little shuffling can be done for 64-bit vectors right now.
20778 if (VT.getSizeInBits() == 64)
20781 // This is an experimental legality test that is tailored to match the
20782 // legality test of the experimental lowering more closely. They are gated
20783 // separately to ease testing of performance differences.
20784 if (ExperimentalVectorShuffleLegality)
20785 // We only care that the types being shuffled are legal. The lowering can
20786 // handle any possible shuffle mask that results.
20787 return isTypeLegal(SVT);
20789 // If this is a single-input shuffle with no 128 bit lane crossings we can
20790 // lower it into pshufb.
20791 if ((SVT.is128BitVector() && Subtarget->hasSSSE3()) ||
20792 (SVT.is256BitVector() && Subtarget->hasInt256())) {
20793 bool isLegal = true;
20794 for (unsigned I = 0, E = M.size(); I != E; ++I) {
20795 if (M[I] >= (int)SVT.getVectorNumElements() ||
20796 ShuffleCrosses128bitLane(SVT, I, M[I])) {
20805 // FIXME: blends, shifts.
20806 return (SVT.getVectorNumElements() == 2 ||
20807 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
20808 isMOVLMask(M, SVT) ||
20809 isCommutedMOVLMask(M, SVT) ||
20810 isMOVHLPSMask(M, SVT) ||
20811 isSHUFPMask(M, SVT) ||
20812 isSHUFPMask(M, SVT, /* Commuted */ true) ||
20813 isPSHUFDMask(M, SVT) ||
20814 isPSHUFDMask(M, SVT, /* SecondOperand */ true) ||
20815 isPSHUFHWMask(M, SVT, Subtarget->hasInt256()) ||
20816 isPSHUFLWMask(M, SVT, Subtarget->hasInt256()) ||
20817 isPALIGNRMask(M, SVT, Subtarget) ||
20818 isUNPCKLMask(M, SVT, Subtarget->hasInt256()) ||
20819 isUNPCKHMask(M, SVT, Subtarget->hasInt256()) ||
20820 isUNPCKL_v_undef_Mask(M, SVT, Subtarget->hasInt256()) ||
20821 isUNPCKH_v_undef_Mask(M, SVT, Subtarget->hasInt256()) ||
20822 isBlendMask(M, SVT, Subtarget->hasSSE41(), Subtarget->hasInt256()) ||
20823 (Subtarget->hasSSE41() && isINSERTPSMask(M, SVT)));
20827 X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
20829 if (!VT.isSimple())
20832 MVT SVT = VT.getSimpleVT();
20834 // This is an experimental legality test that is tailored to match the
20835 // legality test of the experimental lowering more closely. They are gated
20836 // separately to ease testing of performance differences.
20837 if (ExperimentalVectorShuffleLegality)
20838 // The new vector shuffle lowering is very good at managing zero-inputs.
20839 return isShuffleMaskLegal(Mask, VT);
20841 unsigned NumElts = SVT.getVectorNumElements();
20842 // FIXME: This collection of masks seems suspect.
20845 if (NumElts == 4 && SVT.is128BitVector()) {
20846 return (isMOVLMask(Mask, SVT) ||
20847 isCommutedMOVLMask(Mask, SVT, true) ||
20848 isSHUFPMask(Mask, SVT) ||
20849 isSHUFPMask(Mask, SVT, /* Commuted */ true) ||
20850 isBlendMask(Mask, SVT, Subtarget->hasSSE41(),
20851 Subtarget->hasInt256()));
20856 //===----------------------------------------------------------------------===//
20857 // X86 Scheduler Hooks
20858 //===----------------------------------------------------------------------===//
20860 /// Utility function to emit xbegin specifying the start of an RTM region.
20861 static MachineBasicBlock *EmitXBegin(MachineInstr *MI, MachineBasicBlock *MBB,
20862 const TargetInstrInfo *TII) {
20863 DebugLoc DL = MI->getDebugLoc();
20865 const BasicBlock *BB = MBB->getBasicBlock();
20866 MachineFunction::iterator I = MBB;
20869 // For the v = xbegin(), we generate
20880 MachineBasicBlock *thisMBB = MBB;
20881 MachineFunction *MF = MBB->getParent();
20882 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
20883 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
20884 MF->insert(I, mainMBB);
20885 MF->insert(I, sinkMBB);
20887 // Transfer the remainder of BB and its successor edges to sinkMBB.
20888 sinkMBB->splice(sinkMBB->begin(), MBB,
20889 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
20890 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
20894 // # fallthrough to mainMBB
20895 // # abortion to sinkMBB
20896 BuildMI(thisMBB, DL, TII->get(X86::XBEGIN_4)).addMBB(sinkMBB);
20897 thisMBB->addSuccessor(mainMBB);
20898 thisMBB->addSuccessor(sinkMBB);
20902 BuildMI(mainMBB, DL, TII->get(X86::MOV32ri), X86::EAX).addImm(-1);
20903 mainMBB->addSuccessor(sinkMBB);
20906 // EAX is live into the sinkMBB
20907 sinkMBB->addLiveIn(X86::EAX);
20908 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
20909 TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
20912 MI->eraseFromParent();
20916 // FIXME: When we get size specific XMM0 registers, i.e. XMM0_V16I8
20917 // or XMM0_V32I8 in AVX all of this code can be replaced with that
20918 // in the .td file.
20919 static MachineBasicBlock *EmitPCMPSTRM(MachineInstr *MI, MachineBasicBlock *BB,
20920 const TargetInstrInfo *TII) {
20922 switch (MI->getOpcode()) {
20923 default: llvm_unreachable("illegal opcode!");
20924 case X86::PCMPISTRM128REG: Opc = X86::PCMPISTRM128rr; break;
20925 case X86::VPCMPISTRM128REG: Opc = X86::VPCMPISTRM128rr; break;
20926 case X86::PCMPISTRM128MEM: Opc = X86::PCMPISTRM128rm; break;
20927 case X86::VPCMPISTRM128MEM: Opc = X86::VPCMPISTRM128rm; break;
20928 case X86::PCMPESTRM128REG: Opc = X86::PCMPESTRM128rr; break;
20929 case X86::VPCMPESTRM128REG: Opc = X86::VPCMPESTRM128rr; break;
20930 case X86::PCMPESTRM128MEM: Opc = X86::PCMPESTRM128rm; break;
20931 case X86::VPCMPESTRM128MEM: Opc = X86::VPCMPESTRM128rm; break;
20934 DebugLoc dl = MI->getDebugLoc();
20935 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
20937 unsigned NumArgs = MI->getNumOperands();
20938 for (unsigned i = 1; i < NumArgs; ++i) {
20939 MachineOperand &Op = MI->getOperand(i);
20940 if (!(Op.isReg() && Op.isImplicit()))
20941 MIB.addOperand(Op);
20943 if (MI->hasOneMemOperand())
20944 MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
20946 BuildMI(*BB, MI, dl,
20947 TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
20948 .addReg(X86::XMM0);
20950 MI->eraseFromParent();
20954 // FIXME: Custom handling because TableGen doesn't support multiple implicit
20955 // defs in an instruction pattern
20956 static MachineBasicBlock *EmitPCMPSTRI(MachineInstr *MI, MachineBasicBlock *BB,
20957 const TargetInstrInfo *TII) {
20959 switch (MI->getOpcode()) {
20960 default: llvm_unreachable("illegal opcode!");
20961 case X86::PCMPISTRIREG: Opc = X86::PCMPISTRIrr; break;
20962 case X86::VPCMPISTRIREG: Opc = X86::VPCMPISTRIrr; break;
20963 case X86::PCMPISTRIMEM: Opc = X86::PCMPISTRIrm; break;
20964 case X86::VPCMPISTRIMEM: Opc = X86::VPCMPISTRIrm; break;
20965 case X86::PCMPESTRIREG: Opc = X86::PCMPESTRIrr; break;
20966 case X86::VPCMPESTRIREG: Opc = X86::VPCMPESTRIrr; break;
20967 case X86::PCMPESTRIMEM: Opc = X86::PCMPESTRIrm; break;
20968 case X86::VPCMPESTRIMEM: Opc = X86::VPCMPESTRIrm; break;
20971 DebugLoc dl = MI->getDebugLoc();
20972 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
20974 unsigned NumArgs = MI->getNumOperands(); // remove the results
20975 for (unsigned i = 1; i < NumArgs; ++i) {
20976 MachineOperand &Op = MI->getOperand(i);
20977 if (!(Op.isReg() && Op.isImplicit()))
20978 MIB.addOperand(Op);
20980 if (MI->hasOneMemOperand())
20981 MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
20983 BuildMI(*BB, MI, dl,
20984 TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
20987 MI->eraseFromParent();
20991 static MachineBasicBlock *EmitMonitor(MachineInstr *MI, MachineBasicBlock *BB,
20992 const X86Subtarget *Subtarget) {
20993 DebugLoc dl = MI->getDebugLoc();
20994 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
20995 // Address into RAX/EAX, other two args into ECX, EDX.
20996 unsigned MemOpc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r;
20997 unsigned MemReg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
20998 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(MemOpc), MemReg);
20999 for (int i = 0; i < X86::AddrNumOperands; ++i)
21000 MIB.addOperand(MI->getOperand(i));
21002 unsigned ValOps = X86::AddrNumOperands;
21003 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::ECX)
21004 .addReg(MI->getOperand(ValOps).getReg());
21005 BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), X86::EDX)
21006 .addReg(MI->getOperand(ValOps+1).getReg());
21008 // The instruction doesn't actually take any operands though.
21009 BuildMI(*BB, MI, dl, TII->get(X86::MONITORrrr));
21011 MI->eraseFromParent(); // The pseudo is gone now.
21015 MachineBasicBlock *
21016 X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr *MI,
21017 MachineBasicBlock *MBB) const {
21018 // Emit va_arg instruction on X86-64.
21020 // Operands to this pseudo-instruction:
21021 // 0 ) Output : destination address (reg)
21022 // 1-5) Input : va_list address (addr, i64mem)
21023 // 6 ) ArgSize : Size (in bytes) of vararg type
21024 // 7 ) ArgMode : 0=overflow only, 1=use gp_offset, 2=use fp_offset
21025 // 8 ) Align : Alignment of type
21026 // 9 ) EFLAGS (implicit-def)
21028 assert(MI->getNumOperands() == 10 && "VAARG_64 should have 10 operands!");
21029 assert(X86::AddrNumOperands == 5 && "VAARG_64 assumes 5 address operands");
21031 unsigned DestReg = MI->getOperand(0).getReg();
21032 MachineOperand &Base = MI->getOperand(1);
21033 MachineOperand &Scale = MI->getOperand(2);
21034 MachineOperand &Index = MI->getOperand(3);
21035 MachineOperand &Disp = MI->getOperand(4);
21036 MachineOperand &Segment = MI->getOperand(5);
21037 unsigned ArgSize = MI->getOperand(6).getImm();
21038 unsigned ArgMode = MI->getOperand(7).getImm();
21039 unsigned Align = MI->getOperand(8).getImm();
21041 // Memory Reference
21042 assert(MI->hasOneMemOperand() && "Expected VAARG_64 to have one memoperand");
21043 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
21044 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
21046 // Machine Information
21047 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21048 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
21049 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64);
21050 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
21051 DebugLoc DL = MI->getDebugLoc();
21053 // struct va_list {
21056 // i64 overflow_area (address)
21057 // i64 reg_save_area (address)
21059 // sizeof(va_list) = 24
21060 // alignment(va_list) = 8
21062 unsigned TotalNumIntRegs = 6;
21063 unsigned TotalNumXMMRegs = 8;
21064 bool UseGPOffset = (ArgMode == 1);
21065 bool UseFPOffset = (ArgMode == 2);
21066 unsigned MaxOffset = TotalNumIntRegs * 8 +
21067 (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
21069 /* Align ArgSize to a multiple of 8 */
21070 unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
21071 bool NeedsAlign = (Align > 8);
21073 MachineBasicBlock *thisMBB = MBB;
21074 MachineBasicBlock *overflowMBB;
21075 MachineBasicBlock *offsetMBB;
21076 MachineBasicBlock *endMBB;
21078 unsigned OffsetDestReg = 0; // Argument address computed by offsetMBB
21079 unsigned OverflowDestReg = 0; // Argument address computed by overflowMBB
21080 unsigned OffsetReg = 0;
21082 if (!UseGPOffset && !UseFPOffset) {
21083 // If we only pull from the overflow region, we don't create a branch.
21084 // We don't need to alter control flow.
21085 OffsetDestReg = 0; // unused
21086 OverflowDestReg = DestReg;
21088 offsetMBB = nullptr;
21089 overflowMBB = thisMBB;
21092 // First emit code to check if gp_offset (or fp_offset) is below the bound.
21093 // If so, pull the argument from reg_save_area. (branch to offsetMBB)
21094 // If not, pull from overflow_area. (branch to overflowMBB)
21099 // offsetMBB overflowMBB
21104 // Registers for the PHI in endMBB
21105 OffsetDestReg = MRI.createVirtualRegister(AddrRegClass);
21106 OverflowDestReg = MRI.createVirtualRegister(AddrRegClass);
21108 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
21109 MachineFunction *MF = MBB->getParent();
21110 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21111 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21112 endMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21114 MachineFunction::iterator MBBIter = MBB;
21117 // Insert the new basic blocks
21118 MF->insert(MBBIter, offsetMBB);
21119 MF->insert(MBBIter, overflowMBB);
21120 MF->insert(MBBIter, endMBB);
21122 // Transfer the remainder of MBB and its successor edges to endMBB.
21123 endMBB->splice(endMBB->begin(), thisMBB,
21124 std::next(MachineBasicBlock::iterator(MI)), thisMBB->end());
21125 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
21127 // Make offsetMBB and overflowMBB successors of thisMBB
21128 thisMBB->addSuccessor(offsetMBB);
21129 thisMBB->addSuccessor(overflowMBB);
21131 // endMBB is a successor of both offsetMBB and overflowMBB
21132 offsetMBB->addSuccessor(endMBB);
21133 overflowMBB->addSuccessor(endMBB);
21135 // Load the offset value into a register
21136 OffsetReg = MRI.createVirtualRegister(OffsetRegClass);
21137 BuildMI(thisMBB, DL, TII->get(X86::MOV32rm), OffsetReg)
21141 .addDisp(Disp, UseFPOffset ? 4 : 0)
21142 .addOperand(Segment)
21143 .setMemRefs(MMOBegin, MMOEnd);
21145 // Check if there is enough room left to pull this argument.
21146 BuildMI(thisMBB, DL, TII->get(X86::CMP32ri))
21148 .addImm(MaxOffset + 8 - ArgSizeA8);
21150 // Branch to "overflowMBB" if offset >= max
21151 // Fall through to "offsetMBB" otherwise
21152 BuildMI(thisMBB, DL, TII->get(X86::GetCondBranchFromCond(X86::COND_AE)))
21153 .addMBB(overflowMBB);
21156 // In offsetMBB, emit code to use the reg_save_area.
21158 assert(OffsetReg != 0);
21160 // Read the reg_save_area address.
21161 unsigned RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
21162 BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg)
21167 .addOperand(Segment)
21168 .setMemRefs(MMOBegin, MMOEnd);
21170 // Zero-extend the offset
21171 unsigned OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
21172 BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
21175 .addImm(X86::sub_32bit);
21177 // Add the offset to the reg_save_area to get the final address.
21178 BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg)
21179 .addReg(OffsetReg64)
21180 .addReg(RegSaveReg);
21182 // Compute the offset for the next argument
21183 unsigned NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
21184 BuildMI(offsetMBB, DL, TII->get(X86::ADD32ri), NextOffsetReg)
21186 .addImm(UseFPOffset ? 16 : 8);
21188 // Store it back into the va_list.
21189 BuildMI(offsetMBB, DL, TII->get(X86::MOV32mr))
21193 .addDisp(Disp, UseFPOffset ? 4 : 0)
21194 .addOperand(Segment)
21195 .addReg(NextOffsetReg)
21196 .setMemRefs(MMOBegin, MMOEnd);
21199 BuildMI(offsetMBB, DL, TII->get(X86::JMP_1))
21204 // Emit code to use overflow area
21207 // Load the overflow_area address into a register.
21208 unsigned OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
21209 BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg)
21214 .addOperand(Segment)
21215 .setMemRefs(MMOBegin, MMOEnd);
21217 // If we need to align it, do so. Otherwise, just copy the address
21218 // to OverflowDestReg.
21220 // Align the overflow address
21221 assert((Align & (Align-1)) == 0 && "Alignment must be a power of 2");
21222 unsigned TmpReg = MRI.createVirtualRegister(AddrRegClass);
21224 // aligned_addr = (addr + (align-1)) & ~(align-1)
21225 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg)
21226 .addReg(OverflowAddrReg)
21229 BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg)
21231 .addImm(~(uint64_t)(Align-1));
21233 BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg)
21234 .addReg(OverflowAddrReg);
21237 // Compute the next overflow address after this argument.
21238 // (the overflow address should be kept 8-byte aligned)
21239 unsigned NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
21240 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg)
21241 .addReg(OverflowDestReg)
21242 .addImm(ArgSizeA8);
21244 // Store the new overflow address.
21245 BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr))
21250 .addOperand(Segment)
21251 .addReg(NextAddrReg)
21252 .setMemRefs(MMOBegin, MMOEnd);
21254 // If we branched, emit the PHI to the front of endMBB.
21256 BuildMI(*endMBB, endMBB->begin(), DL,
21257 TII->get(X86::PHI), DestReg)
21258 .addReg(OffsetDestReg).addMBB(offsetMBB)
21259 .addReg(OverflowDestReg).addMBB(overflowMBB);
21262 // Erase the pseudo instruction
21263 MI->eraseFromParent();
21268 MachineBasicBlock *
21269 X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
21271 MachineBasicBlock *MBB) const {
21272 // Emit code to save XMM registers to the stack. The ABI says that the
21273 // number of registers to save is given in %al, so it's theoretically
21274 // possible to do an indirect jump trick to avoid saving all of them,
21275 // however this code takes a simpler approach and just executes all
21276 // of the stores if %al is non-zero. It's less code, and it's probably
21277 // easier on the hardware branch predictor, and stores aren't all that
21278 // expensive anyway.
21280 // Create the new basic blocks. One block contains all the XMM stores,
21281 // and one block is the final destination regardless of whether any
21282 // stores were performed.
21283 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
21284 MachineFunction *F = MBB->getParent();
21285 MachineFunction::iterator MBBIter = MBB;
21287 MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB);
21288 MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB);
21289 F->insert(MBBIter, XMMSaveMBB);
21290 F->insert(MBBIter, EndMBB);
21292 // Transfer the remainder of MBB and its successor edges to EndMBB.
21293 EndMBB->splice(EndMBB->begin(), MBB,
21294 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
21295 EndMBB->transferSuccessorsAndUpdatePHIs(MBB);
21297 // The original block will now fall through to the XMM save block.
21298 MBB->addSuccessor(XMMSaveMBB);
21299 // The XMMSaveMBB will fall through to the end block.
21300 XMMSaveMBB->addSuccessor(EndMBB);
21302 // Now add the instructions.
21303 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21304 DebugLoc DL = MI->getDebugLoc();
21306 unsigned CountReg = MI->getOperand(0).getReg();
21307 int64_t RegSaveFrameIndex = MI->getOperand(1).getImm();
21308 int64_t VarArgsFPOffset = MI->getOperand(2).getImm();
21310 if (!Subtarget->isTargetWin64()) {
21311 // If %al is 0, branch around the XMM save block.
21312 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg);
21313 BuildMI(MBB, DL, TII->get(X86::JE_1)).addMBB(EndMBB);
21314 MBB->addSuccessor(EndMBB);
21317 // Make sure the last operand is EFLAGS, which gets clobbered by the branch
21318 // that was just emitted, but clearly shouldn't be "saved".
21319 assert((MI->getNumOperands() <= 3 ||
21320 !MI->getOperand(MI->getNumOperands() - 1).isReg() ||
21321 MI->getOperand(MI->getNumOperands() - 1).getReg() == X86::EFLAGS)
21322 && "Expected last argument to be EFLAGS");
21323 unsigned MOVOpc = Subtarget->hasFp256() ? X86::VMOVAPSmr : X86::MOVAPSmr;
21324 // In the XMM save block, save all the XMM argument registers.
21325 for (int i = 3, e = MI->getNumOperands() - 1; i != e; ++i) {
21326 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset;
21327 MachineMemOperand *MMO =
21328 F->getMachineMemOperand(
21329 MachinePointerInfo::getFixedStack(RegSaveFrameIndex, Offset),
21330 MachineMemOperand::MOStore,
21331 /*Size=*/16, /*Align=*/16);
21332 BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc))
21333 .addFrameIndex(RegSaveFrameIndex)
21334 .addImm(/*Scale=*/1)
21335 .addReg(/*IndexReg=*/0)
21336 .addImm(/*Disp=*/Offset)
21337 .addReg(/*Segment=*/0)
21338 .addReg(MI->getOperand(i).getReg())
21339 .addMemOperand(MMO);
21342 MI->eraseFromParent(); // The pseudo instruction is gone now.
21347 // The EFLAGS operand of SelectItr might be missing a kill marker
21348 // because there were multiple uses of EFLAGS, and ISel didn't know
21349 // which to mark. Figure out whether SelectItr should have had a
21350 // kill marker, and set it if it should. Returns the correct kill
21352 static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
21353 MachineBasicBlock* BB,
21354 const TargetRegisterInfo* TRI) {
21355 // Scan forward through BB for a use/def of EFLAGS.
21356 MachineBasicBlock::iterator miI(std::next(SelectItr));
21357 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
21358 const MachineInstr& mi = *miI;
21359 if (mi.readsRegister(X86::EFLAGS))
21361 if (mi.definesRegister(X86::EFLAGS))
21362 break; // Should have kill-flag - update below.
21365 // If we hit the end of the block, check whether EFLAGS is live into a
21367 if (miI == BB->end()) {
21368 for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
21369 sEnd = BB->succ_end();
21370 sItr != sEnd; ++sItr) {
21371 MachineBasicBlock* succ = *sItr;
21372 if (succ->isLiveIn(X86::EFLAGS))
21377 // We found a def, or hit the end of the basic block and EFLAGS wasn't live
21378 // out. SelectMI should have a kill flag on EFLAGS.
21379 SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
21383 MachineBasicBlock *
21384 X86TargetLowering::EmitLoweredSelect(MachineInstr *MI,
21385 MachineBasicBlock *BB) const {
21386 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21387 DebugLoc DL = MI->getDebugLoc();
21389 // To "insert" a SELECT_CC instruction, we actually have to insert the
21390 // diamond control-flow pattern. The incoming instruction knows the
21391 // destination vreg to set, the condition code register to branch on, the
21392 // true/false values to select between, and a branch opcode to use.
21393 const BasicBlock *LLVM_BB = BB->getBasicBlock();
21394 MachineFunction::iterator It = BB;
21400 // cmpTY ccX, r1, r2
21402 // fallthrough --> copy0MBB
21403 MachineBasicBlock *thisMBB = BB;
21404 MachineFunction *F = BB->getParent();
21405 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
21406 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
21407 F->insert(It, copy0MBB);
21408 F->insert(It, sinkMBB);
21410 // If the EFLAGS register isn't dead in the terminator, then claim that it's
21411 // live into the sink and copy blocks.
21412 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
21413 if (!MI->killsRegister(X86::EFLAGS) &&
21414 !checkAndUpdateEFLAGSKill(MI, BB, TRI)) {
21415 copy0MBB->addLiveIn(X86::EFLAGS);
21416 sinkMBB->addLiveIn(X86::EFLAGS);
21419 // Transfer the remainder of BB and its successor edges to sinkMBB.
21420 sinkMBB->splice(sinkMBB->begin(), BB,
21421 std::next(MachineBasicBlock::iterator(MI)), BB->end());
21422 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
21424 // Add the true and fallthrough blocks as its successors.
21425 BB->addSuccessor(copy0MBB);
21426 BB->addSuccessor(sinkMBB);
21428 // Create the conditional branch instruction.
21430 X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
21431 BuildMI(BB, DL, TII->get(Opc)).addMBB(sinkMBB);
21434 // %FalseValue = ...
21435 // # fallthrough to sinkMBB
21436 copy0MBB->addSuccessor(sinkMBB);
21439 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
21441 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
21442 TII->get(X86::PHI), MI->getOperand(0).getReg())
21443 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
21444 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
21446 MI->eraseFromParent(); // The pseudo instruction is gone now.
21450 MachineBasicBlock *
21451 X86TargetLowering::EmitLoweredSegAlloca(MachineInstr *MI,
21452 MachineBasicBlock *BB) const {
21453 MachineFunction *MF = BB->getParent();
21454 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21455 DebugLoc DL = MI->getDebugLoc();
21456 const BasicBlock *LLVM_BB = BB->getBasicBlock();
21458 assert(MF->shouldSplitStack());
21460 const bool Is64Bit = Subtarget->is64Bit();
21461 const bool IsLP64 = Subtarget->isTarget64BitLP64();
21463 const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
21464 const unsigned TlsOffset = IsLP64 ? 0x70 : Is64Bit ? 0x40 : 0x30;
21467 // ... [Till the alloca]
21468 // If stacklet is not large enough, jump to mallocMBB
21471 // Allocate by subtracting from RSP
21472 // Jump to continueMBB
21475 // Allocate by call to runtime
21479 // [rest of original BB]
21482 MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21483 MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21484 MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
21486 MachineRegisterInfo &MRI = MF->getRegInfo();
21487 const TargetRegisterClass *AddrRegClass =
21488 getRegClassFor(getPointerTy());
21490 unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
21491 bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
21492 tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
21493 SPLimitVReg = MRI.createVirtualRegister(AddrRegClass),
21494 sizeVReg = MI->getOperand(1).getReg(),
21495 physSPReg = IsLP64 || Subtarget->isTargetNaCl64() ? X86::RSP : X86::ESP;
21497 MachineFunction::iterator MBBIter = BB;
21500 MF->insert(MBBIter, bumpMBB);
21501 MF->insert(MBBIter, mallocMBB);
21502 MF->insert(MBBIter, continueMBB);
21504 continueMBB->splice(continueMBB->begin(), BB,
21505 std::next(MachineBasicBlock::iterator(MI)), BB->end());
21506 continueMBB->transferSuccessorsAndUpdatePHIs(BB);
21508 // Add code to the main basic block to check if the stack limit has been hit,
21509 // and if so, jump to mallocMBB otherwise to bumpMBB.
21510 BuildMI(BB, DL, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
21511 BuildMI(BB, DL, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
21512 .addReg(tmpSPVReg).addReg(sizeVReg);
21513 BuildMI(BB, DL, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
21514 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
21515 .addReg(SPLimitVReg);
21516 BuildMI(BB, DL, TII->get(X86::JG_1)).addMBB(mallocMBB);
21518 // bumpMBB simply decreases the stack pointer, since we know the current
21519 // stacklet has enough space.
21520 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), physSPReg)
21521 .addReg(SPLimitVReg);
21522 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
21523 .addReg(SPLimitVReg);
21524 BuildMI(bumpMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
21526 // Calls into a routine in libgcc to allocate more space from the heap.
21527 const uint32_t *RegMask =
21528 Subtarget->getRegisterInfo()->getCallPreservedMask(CallingConv::C);
21530 BuildMI(mallocMBB, DL, TII->get(X86::MOV64rr), X86::RDI)
21532 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
21533 .addExternalSymbol("__morestack_allocate_stack_space")
21534 .addRegMask(RegMask)
21535 .addReg(X86::RDI, RegState::Implicit)
21536 .addReg(X86::RAX, RegState::ImplicitDefine);
21537 } else if (Is64Bit) {
21538 BuildMI(mallocMBB, DL, TII->get(X86::MOV32rr), X86::EDI)
21540 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
21541 .addExternalSymbol("__morestack_allocate_stack_space")
21542 .addRegMask(RegMask)
21543 .addReg(X86::EDI, RegState::Implicit)
21544 .addReg(X86::EAX, RegState::ImplicitDefine);
21546 BuildMI(mallocMBB, DL, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
21548 BuildMI(mallocMBB, DL, TII->get(X86::PUSH32r)).addReg(sizeVReg);
21549 BuildMI(mallocMBB, DL, TII->get(X86::CALLpcrel32))
21550 .addExternalSymbol("__morestack_allocate_stack_space")
21551 .addRegMask(RegMask)
21552 .addReg(X86::EAX, RegState::ImplicitDefine);
21556 BuildMI(mallocMBB, DL, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
21559 BuildMI(mallocMBB, DL, TII->get(TargetOpcode::COPY), mallocPtrVReg)
21560 .addReg(IsLP64 ? X86::RAX : X86::EAX);
21561 BuildMI(mallocMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
21563 // Set up the CFG correctly.
21564 BB->addSuccessor(bumpMBB);
21565 BB->addSuccessor(mallocMBB);
21566 mallocMBB->addSuccessor(continueMBB);
21567 bumpMBB->addSuccessor(continueMBB);
21569 // Take care of the PHI nodes.
21570 BuildMI(*continueMBB, continueMBB->begin(), DL, TII->get(X86::PHI),
21571 MI->getOperand(0).getReg())
21572 .addReg(mallocPtrVReg).addMBB(mallocMBB)
21573 .addReg(bumpSPPtrVReg).addMBB(bumpMBB);
21575 // Delete the original pseudo instruction.
21576 MI->eraseFromParent();
21579 return continueMBB;
21582 MachineBasicBlock *
21583 X86TargetLowering::EmitLoweredWinAlloca(MachineInstr *MI,
21584 MachineBasicBlock *BB) const {
21585 DebugLoc DL = MI->getDebugLoc();
21587 assert(!Subtarget->isTargetMachO());
21589 X86FrameLowering::emitStackProbeCall(*BB->getParent(), *BB, MI, DL);
21591 MI->eraseFromParent(); // The pseudo instruction is gone now.
21595 MachineBasicBlock *
21596 X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI,
21597 MachineBasicBlock *BB) const {
21598 // This is pretty easy. We're taking the value that we received from
21599 // our load from the relocation, sticking it in either RDI (x86-64)
21600 // or EAX and doing an indirect call. The return value will then
21601 // be in the normal return register.
21602 MachineFunction *F = BB->getParent();
21603 const X86InstrInfo *TII = Subtarget->getInstrInfo();
21604 DebugLoc DL = MI->getDebugLoc();
21606 assert(Subtarget->isTargetDarwin() && "Darwin only instr emitted?");
21607 assert(MI->getOperand(3).isGlobal() && "This should be a global");
21609 // Get a register mask for the lowered call.
21610 // FIXME: The 32-bit calls have non-standard calling conventions. Use a
21611 // proper register mask.
21612 const uint32_t *RegMask =
21613 Subtarget->getRegisterInfo()->getCallPreservedMask(CallingConv::C);
21614 if (Subtarget->is64Bit()) {
21615 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
21616 TII->get(X86::MOV64rm), X86::RDI)
21618 .addImm(0).addReg(0)
21619 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
21620 MI->getOperand(3).getTargetFlags())
21622 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m));
21623 addDirectMem(MIB, X86::RDI);
21624 MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
21625 } else if (F->getTarget().getRelocationModel() != Reloc::PIC_) {
21626 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
21627 TII->get(X86::MOV32rm), X86::EAX)
21629 .addImm(0).addReg(0)
21630 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
21631 MI->getOperand(3).getTargetFlags())
21633 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
21634 addDirectMem(MIB, X86::EAX);
21635 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
21637 MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
21638 TII->get(X86::MOV32rm), X86::EAX)
21639 .addReg(TII->getGlobalBaseReg(F))
21640 .addImm(0).addReg(0)
21641 .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
21642 MI->getOperand(3).getTargetFlags())
21644 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
21645 addDirectMem(MIB, X86::EAX);
21646 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
21649 MI->eraseFromParent(); // The pseudo instruction is gone now.
21653 MachineBasicBlock *
21654 X86TargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
21655 MachineBasicBlock *MBB) const {
21656 DebugLoc DL = MI->getDebugLoc();
21657 MachineFunction *MF = MBB->getParent();
21658 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21659 MachineRegisterInfo &MRI = MF->getRegInfo();
21661 const BasicBlock *BB = MBB->getBasicBlock();
21662 MachineFunction::iterator I = MBB;
21665 // Memory Reference
21666 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
21667 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
21670 unsigned MemOpndSlot = 0;
21672 unsigned CurOp = 0;
21674 DstReg = MI->getOperand(CurOp++).getReg();
21675 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
21676 assert(RC->hasType(MVT::i32) && "Invalid destination!");
21677 unsigned mainDstReg = MRI.createVirtualRegister(RC);
21678 unsigned restoreDstReg = MRI.createVirtualRegister(RC);
21680 MemOpndSlot = CurOp;
21682 MVT PVT = getPointerTy();
21683 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
21684 "Invalid Pointer Size!");
21686 // For v = setjmp(buf), we generate
21689 // buf[LabelOffset] = restoreMBB
21690 // SjLjSetup restoreMBB
21696 // v = phi(main, restore)
21699 // if base pointer being used, load it from frame
21702 MachineBasicBlock *thisMBB = MBB;
21703 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
21704 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
21705 MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
21706 MF->insert(I, mainMBB);
21707 MF->insert(I, sinkMBB);
21708 MF->push_back(restoreMBB);
21710 MachineInstrBuilder MIB;
21712 // Transfer the remainder of BB and its successor edges to sinkMBB.
21713 sinkMBB->splice(sinkMBB->begin(), MBB,
21714 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
21715 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
21718 unsigned PtrStoreOpc = 0;
21719 unsigned LabelReg = 0;
21720 const int64_t LabelOffset = 1 * PVT.getStoreSize();
21721 Reloc::Model RM = MF->getTarget().getRelocationModel();
21722 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
21723 (RM == Reloc::Static || RM == Reloc::DynamicNoPIC);
21725 // Prepare IP either in reg or imm.
21726 if (!UseImmLabel) {
21727 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
21728 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
21729 LabelReg = MRI.createVirtualRegister(PtrRC);
21730 if (Subtarget->is64Bit()) {
21731 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA64r), LabelReg)
21735 .addMBB(restoreMBB)
21738 const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
21739 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA32r), LabelReg)
21740 .addReg(XII->getGlobalBaseReg(MF))
21743 .addMBB(restoreMBB, Subtarget->ClassifyBlockAddressReference())
21747 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
21749 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrStoreOpc));
21750 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
21751 if (i == X86::AddrDisp)
21752 MIB.addDisp(MI->getOperand(MemOpndSlot + i), LabelOffset);
21754 MIB.addOperand(MI->getOperand(MemOpndSlot + i));
21757 MIB.addReg(LabelReg);
21759 MIB.addMBB(restoreMBB);
21760 MIB.setMemRefs(MMOBegin, MMOEnd);
21762 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::EH_SjLj_Setup))
21763 .addMBB(restoreMBB);
21765 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
21766 MIB.addRegMask(RegInfo->getNoPreservedMask());
21767 thisMBB->addSuccessor(mainMBB);
21768 thisMBB->addSuccessor(restoreMBB);
21772 BuildMI(mainMBB, DL, TII->get(X86::MOV32r0), mainDstReg);
21773 mainMBB->addSuccessor(sinkMBB);
21776 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
21777 TII->get(X86::PHI), DstReg)
21778 .addReg(mainDstReg).addMBB(mainMBB)
21779 .addReg(restoreDstReg).addMBB(restoreMBB);
21782 if (RegInfo->hasBasePointer(*MF)) {
21783 const bool Uses64BitFramePtr =
21784 Subtarget->isTarget64BitLP64() || Subtarget->isTargetNaCl64();
21785 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
21786 X86FI->setRestoreBasePointer(MF);
21787 unsigned FramePtr = RegInfo->getFrameRegister(*MF);
21788 unsigned BasePtr = RegInfo->getBaseRegister();
21789 unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
21790 addRegOffset(BuildMI(restoreMBB, DL, TII->get(Opm), BasePtr),
21791 FramePtr, true, X86FI->getRestoreBasePointerOffset())
21792 .setMIFlag(MachineInstr::FrameSetup);
21794 BuildMI(restoreMBB, DL, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
21795 BuildMI(restoreMBB, DL, TII->get(X86::JMP_1)).addMBB(sinkMBB);
21796 restoreMBB->addSuccessor(sinkMBB);
21798 MI->eraseFromParent();
21802 MachineBasicBlock *
21803 X86TargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
21804 MachineBasicBlock *MBB) const {
21805 DebugLoc DL = MI->getDebugLoc();
21806 MachineFunction *MF = MBB->getParent();
21807 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
21808 MachineRegisterInfo &MRI = MF->getRegInfo();
21810 // Memory Reference
21811 MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
21812 MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
21814 MVT PVT = getPointerTy();
21815 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
21816 "Invalid Pointer Size!");
21818 const TargetRegisterClass *RC =
21819 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
21820 unsigned Tmp = MRI.createVirtualRegister(RC);
21821 // Since FP is only updated here but NOT referenced, it's treated as GPR.
21822 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
21823 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
21824 unsigned SP = RegInfo->getStackRegister();
21826 MachineInstrBuilder MIB;
21828 const int64_t LabelOffset = 1 * PVT.getStoreSize();
21829 const int64_t SPOffset = 2 * PVT.getStoreSize();
21831 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
21832 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
21835 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), FP);
21836 for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
21837 MIB.addOperand(MI->getOperand(i));
21838 MIB.setMemRefs(MMOBegin, MMOEnd);
21840 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), Tmp);
21841 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
21842 if (i == X86::AddrDisp)
21843 MIB.addDisp(MI->getOperand(i), LabelOffset);
21845 MIB.addOperand(MI->getOperand(i));
21847 MIB.setMemRefs(MMOBegin, MMOEnd);
21849 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), SP);
21850 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
21851 if (i == X86::AddrDisp)
21852 MIB.addDisp(MI->getOperand(i), SPOffset);
21854 MIB.addOperand(MI->getOperand(i));
21856 MIB.setMemRefs(MMOBegin, MMOEnd);
21858 BuildMI(*MBB, MI, DL, TII->get(IJmpOpc)).addReg(Tmp);
21860 MI->eraseFromParent();
21864 // Replace 213-type (isel default) FMA3 instructions with 231-type for
21865 // accumulator loops. Writing back to the accumulator allows the coalescer
21866 // to remove extra copies in the loop.
21867 MachineBasicBlock *
21868 X86TargetLowering::emitFMA3Instr(MachineInstr *MI,
21869 MachineBasicBlock *MBB) const {
21870 MachineOperand &AddendOp = MI->getOperand(3);
21872 // Bail out early if the addend isn't a register - we can't switch these.
21873 if (!AddendOp.isReg())
21876 MachineFunction &MF = *MBB->getParent();
21877 MachineRegisterInfo &MRI = MF.getRegInfo();
21879 // Check whether the addend is defined by a PHI:
21880 assert(MRI.hasOneDef(AddendOp.getReg()) && "Multiple defs in SSA?");
21881 MachineInstr &AddendDef = *MRI.def_instr_begin(AddendOp.getReg());
21882 if (!AddendDef.isPHI())
21885 // Look for the following pattern:
21887 // %addend = phi [%entry, 0], [%loop, %result]
21889 // %result<tied1> = FMA213 %m2<tied0>, %m1, %addend
21893 // %addend = phi [%entry, 0], [%loop, %result]
21895 // %result<tied1> = FMA231 %addend<tied0>, %m1, %m2
21897 for (unsigned i = 1, e = AddendDef.getNumOperands(); i < e; i += 2) {
21898 assert(AddendDef.getOperand(i).isReg());
21899 MachineOperand PHISrcOp = AddendDef.getOperand(i);
21900 MachineInstr &PHISrcInst = *MRI.def_instr_begin(PHISrcOp.getReg());
21901 if (&PHISrcInst == MI) {
21902 // Found a matching instruction.
21903 unsigned NewFMAOpc = 0;
21904 switch (MI->getOpcode()) {
21905 case X86::VFMADDPDr213r: NewFMAOpc = X86::VFMADDPDr231r; break;
21906 case X86::VFMADDPSr213r: NewFMAOpc = X86::VFMADDPSr231r; break;
21907 case X86::VFMADDSDr213r: NewFMAOpc = X86::VFMADDSDr231r; break;
21908 case X86::VFMADDSSr213r: NewFMAOpc = X86::VFMADDSSr231r; break;
21909 case X86::VFMSUBPDr213r: NewFMAOpc = X86::VFMSUBPDr231r; break;
21910 case X86::VFMSUBPSr213r: NewFMAOpc = X86::VFMSUBPSr231r; break;
21911 case X86::VFMSUBSDr213r: NewFMAOpc = X86::VFMSUBSDr231r; break;
21912 case X86::VFMSUBSSr213r: NewFMAOpc = X86::VFMSUBSSr231r; break;
21913 case X86::VFNMADDPDr213r: NewFMAOpc = X86::VFNMADDPDr231r; break;
21914 case X86::VFNMADDPSr213r: NewFMAOpc = X86::VFNMADDPSr231r; break;
21915 case X86::VFNMADDSDr213r: NewFMAOpc = X86::VFNMADDSDr231r; break;
21916 case X86::VFNMADDSSr213r: NewFMAOpc = X86::VFNMADDSSr231r; break;
21917 case X86::VFNMSUBPDr213r: NewFMAOpc = X86::VFNMSUBPDr231r; break;
21918 case X86::VFNMSUBPSr213r: NewFMAOpc = X86::VFNMSUBPSr231r; break;
21919 case X86::VFNMSUBSDr213r: NewFMAOpc = X86::VFNMSUBSDr231r; break;
21920 case X86::VFNMSUBSSr213r: NewFMAOpc = X86::VFNMSUBSSr231r; break;
21921 case X86::VFMADDSUBPDr213r: NewFMAOpc = X86::VFMADDSUBPDr231r; break;
21922 case X86::VFMADDSUBPSr213r: NewFMAOpc = X86::VFMADDSUBPSr231r; break;
21923 case X86::VFMSUBADDPDr213r: NewFMAOpc = X86::VFMSUBADDPDr231r; break;
21924 case X86::VFMSUBADDPSr213r: NewFMAOpc = X86::VFMSUBADDPSr231r; break;
21926 case X86::VFMADDPDr213rY: NewFMAOpc = X86::VFMADDPDr231rY; break;
21927 case X86::VFMADDPSr213rY: NewFMAOpc = X86::VFMADDPSr231rY; break;
21928 case X86::VFMSUBPDr213rY: NewFMAOpc = X86::VFMSUBPDr231rY; break;
21929 case X86::VFMSUBPSr213rY: NewFMAOpc = X86::VFMSUBPSr231rY; break;
21930 case X86::VFNMADDPDr213rY: NewFMAOpc = X86::VFNMADDPDr231rY; break;
21931 case X86::VFNMADDPSr213rY: NewFMAOpc = X86::VFNMADDPSr231rY; break;
21932 case X86::VFNMSUBPDr213rY: NewFMAOpc = X86::VFNMSUBPDr231rY; break;
21933 case X86::VFNMSUBPSr213rY: NewFMAOpc = X86::VFNMSUBPSr231rY; break;
21934 case X86::VFMADDSUBPDr213rY: NewFMAOpc = X86::VFMADDSUBPDr231rY; break;
21935 case X86::VFMADDSUBPSr213rY: NewFMAOpc = X86::VFMADDSUBPSr231rY; break;
21936 case X86::VFMSUBADDPDr213rY: NewFMAOpc = X86::VFMSUBADDPDr231rY; break;
21937 case X86::VFMSUBADDPSr213rY: NewFMAOpc = X86::VFMSUBADDPSr231rY; break;
21938 default: llvm_unreachable("Unrecognized FMA variant.");
21941 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
21942 MachineInstrBuilder MIB =
21943 BuildMI(MF, MI->getDebugLoc(), TII.get(NewFMAOpc))
21944 .addOperand(MI->getOperand(0))
21945 .addOperand(MI->getOperand(3))
21946 .addOperand(MI->getOperand(2))
21947 .addOperand(MI->getOperand(1));
21948 MBB->insert(MachineBasicBlock::iterator(MI), MIB);
21949 MI->eraseFromParent();
21956 MachineBasicBlock *
21957 X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
21958 MachineBasicBlock *BB) const {
21959 switch (MI->getOpcode()) {
21960 default: llvm_unreachable("Unexpected instr type to insert");
21961 case X86::TAILJMPd64:
21962 case X86::TAILJMPr64:
21963 case X86::TAILJMPm64:
21964 case X86::TAILJMPd64_REX:
21965 case X86::TAILJMPr64_REX:
21966 case X86::TAILJMPm64_REX:
21967 llvm_unreachable("TAILJMP64 would not be touched here.");
21968 case X86::TCRETURNdi64:
21969 case X86::TCRETURNri64:
21970 case X86::TCRETURNmi64:
21972 case X86::WIN_ALLOCA:
21973 return EmitLoweredWinAlloca(MI, BB);
21974 case X86::SEG_ALLOCA_32:
21975 case X86::SEG_ALLOCA_64:
21976 return EmitLoweredSegAlloca(MI, BB);
21977 case X86::TLSCall_32:
21978 case X86::TLSCall_64:
21979 return EmitLoweredTLSCall(MI, BB);
21980 case X86::CMOV_GR8:
21981 case X86::CMOV_FR32:
21982 case X86::CMOV_FR64:
21983 case X86::CMOV_V4F32:
21984 case X86::CMOV_V2F64:
21985 case X86::CMOV_V2I64:
21986 case X86::CMOV_V8F32:
21987 case X86::CMOV_V4F64:
21988 case X86::CMOV_V4I64:
21989 case X86::CMOV_V16F32:
21990 case X86::CMOV_V8F64:
21991 case X86::CMOV_V8I64:
21992 case X86::CMOV_GR16:
21993 case X86::CMOV_GR32:
21994 case X86::CMOV_RFP32:
21995 case X86::CMOV_RFP64:
21996 case X86::CMOV_RFP80:
21997 return EmitLoweredSelect(MI, BB);
21999 case X86::FP32_TO_INT16_IN_MEM:
22000 case X86::FP32_TO_INT32_IN_MEM:
22001 case X86::FP32_TO_INT64_IN_MEM:
22002 case X86::FP64_TO_INT16_IN_MEM:
22003 case X86::FP64_TO_INT32_IN_MEM:
22004 case X86::FP64_TO_INT64_IN_MEM:
22005 case X86::FP80_TO_INT16_IN_MEM:
22006 case X86::FP80_TO_INT32_IN_MEM:
22007 case X86::FP80_TO_INT64_IN_MEM: {
22008 MachineFunction *F = BB->getParent();
22009 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
22010 DebugLoc DL = MI->getDebugLoc();
22012 // Change the floating point control register to use "round towards zero"
22013 // mode when truncating to an integer value.
22014 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2, false);
22015 addFrameReference(BuildMI(*BB, MI, DL,
22016 TII->get(X86::FNSTCW16m)), CWFrameIdx);
22018 // Load the old value of the high byte of the control word...
22020 F->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
22021 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16rm), OldCW),
22024 // Set the high part to be round to zero...
22025 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mi)), CWFrameIdx)
22028 // Reload the modified control word now...
22029 addFrameReference(BuildMI(*BB, MI, DL,
22030 TII->get(X86::FLDCW16m)), CWFrameIdx);
22032 // Restore the memory image of control word to original value
22033 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)), CWFrameIdx)
22036 // Get the X86 opcode to use.
22038 switch (MI->getOpcode()) {
22039 default: llvm_unreachable("illegal opcode!");
22040 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
22041 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
22042 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
22043 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
22044 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
22045 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
22046 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
22047 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
22048 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
22052 MachineOperand &Op = MI->getOperand(0);
22054 AM.BaseType = X86AddressMode::RegBase;
22055 AM.Base.Reg = Op.getReg();
22057 AM.BaseType = X86AddressMode::FrameIndexBase;
22058 AM.Base.FrameIndex = Op.getIndex();
22060 Op = MI->getOperand(1);
22062 AM.Scale = Op.getImm();
22063 Op = MI->getOperand(2);
22065 AM.IndexReg = Op.getImm();
22066 Op = MI->getOperand(3);
22067 if (Op.isGlobal()) {
22068 AM.GV = Op.getGlobal();
22070 AM.Disp = Op.getImm();
22072 addFullAddress(BuildMI(*BB, MI, DL, TII->get(Opc)), AM)
22073 .addReg(MI->getOperand(X86::AddrNumOperands).getReg());
22075 // Reload the original control word now.
22076 addFrameReference(BuildMI(*BB, MI, DL,
22077 TII->get(X86::FLDCW16m)), CWFrameIdx);
22079 MI->eraseFromParent(); // The pseudo instruction is gone now.
22082 // String/text processing lowering.
22083 case X86::PCMPISTRM128REG:
22084 case X86::VPCMPISTRM128REG:
22085 case X86::PCMPISTRM128MEM:
22086 case X86::VPCMPISTRM128MEM:
22087 case X86::PCMPESTRM128REG:
22088 case X86::VPCMPESTRM128REG:
22089 case X86::PCMPESTRM128MEM:
22090 case X86::VPCMPESTRM128MEM:
22091 assert(Subtarget->hasSSE42() &&
22092 "Target must have SSE4.2 or AVX features enabled");
22093 return EmitPCMPSTRM(MI, BB, Subtarget->getInstrInfo());
22095 // String/text processing lowering.
22096 case X86::PCMPISTRIREG:
22097 case X86::VPCMPISTRIREG:
22098 case X86::PCMPISTRIMEM:
22099 case X86::VPCMPISTRIMEM:
22100 case X86::PCMPESTRIREG:
22101 case X86::VPCMPESTRIREG:
22102 case X86::PCMPESTRIMEM:
22103 case X86::VPCMPESTRIMEM:
22104 assert(Subtarget->hasSSE42() &&
22105 "Target must have SSE4.2 or AVX features enabled");
22106 return EmitPCMPSTRI(MI, BB, Subtarget->getInstrInfo());
22108 // Thread synchronization.
22110 return EmitMonitor(MI, BB, Subtarget);
22114 return EmitXBegin(MI, BB, Subtarget->getInstrInfo());
22116 case X86::VASTART_SAVE_XMM_REGS:
22117 return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB);
22119 case X86::VAARG_64:
22120 return EmitVAARG64WithCustomInserter(MI, BB);
22122 case X86::EH_SjLj_SetJmp32:
22123 case X86::EH_SjLj_SetJmp64:
22124 return emitEHSjLjSetJmp(MI, BB);
22126 case X86::EH_SjLj_LongJmp32:
22127 case X86::EH_SjLj_LongJmp64:
22128 return emitEHSjLjLongJmp(MI, BB);
22130 case TargetOpcode::STATEPOINT:
22131 // As an implementation detail, STATEPOINT shares the STACKMAP format at
22132 // this point in the process. We diverge later.
22133 return emitPatchPoint(MI, BB);
22135 case TargetOpcode::STACKMAP:
22136 case TargetOpcode::PATCHPOINT:
22137 return emitPatchPoint(MI, BB);
22139 case X86::VFMADDPDr213r:
22140 case X86::VFMADDPSr213r:
22141 case X86::VFMADDSDr213r:
22142 case X86::VFMADDSSr213r:
22143 case X86::VFMSUBPDr213r:
22144 case X86::VFMSUBPSr213r:
22145 case X86::VFMSUBSDr213r:
22146 case X86::VFMSUBSSr213r:
22147 case X86::VFNMADDPDr213r:
22148 case X86::VFNMADDPSr213r:
22149 case X86::VFNMADDSDr213r:
22150 case X86::VFNMADDSSr213r:
22151 case X86::VFNMSUBPDr213r:
22152 case X86::VFNMSUBPSr213r:
22153 case X86::VFNMSUBSDr213r:
22154 case X86::VFNMSUBSSr213r:
22155 case X86::VFMADDSUBPDr213r:
22156 case X86::VFMADDSUBPSr213r:
22157 case X86::VFMSUBADDPDr213r:
22158 case X86::VFMSUBADDPSr213r:
22159 case X86::VFMADDPDr213rY:
22160 case X86::VFMADDPSr213rY:
22161 case X86::VFMSUBPDr213rY:
22162 case X86::VFMSUBPSr213rY:
22163 case X86::VFNMADDPDr213rY:
22164 case X86::VFNMADDPSr213rY:
22165 case X86::VFNMSUBPDr213rY:
22166 case X86::VFNMSUBPSr213rY:
22167 case X86::VFMADDSUBPDr213rY:
22168 case X86::VFMADDSUBPSr213rY:
22169 case X86::VFMSUBADDPDr213rY:
22170 case X86::VFMSUBADDPSr213rY:
22171 return emitFMA3Instr(MI, BB);
22175 //===----------------------------------------------------------------------===//
22176 // X86 Optimization Hooks
22177 //===----------------------------------------------------------------------===//
22179 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
22182 const SelectionDAG &DAG,
22183 unsigned Depth) const {
22184 unsigned BitWidth = KnownZero.getBitWidth();
22185 unsigned Opc = Op.getOpcode();
22186 assert((Opc >= ISD::BUILTIN_OP_END ||
22187 Opc == ISD::INTRINSIC_WO_CHAIN ||
22188 Opc == ISD::INTRINSIC_W_CHAIN ||
22189 Opc == ISD::INTRINSIC_VOID) &&
22190 "Should use MaskedValueIsZero if you don't know whether Op"
22191 " is a target node!");
22193 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
22207 // These nodes' second result is a boolean.
22208 if (Op.getResNo() == 0)
22211 case X86ISD::SETCC:
22212 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
22214 case ISD::INTRINSIC_WO_CHAIN: {
22215 unsigned IntId = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
22216 unsigned NumLoBits = 0;
22219 case Intrinsic::x86_sse_movmsk_ps:
22220 case Intrinsic::x86_avx_movmsk_ps_256:
22221 case Intrinsic::x86_sse2_movmsk_pd:
22222 case Intrinsic::x86_avx_movmsk_pd_256:
22223 case Intrinsic::x86_mmx_pmovmskb:
22224 case Intrinsic::x86_sse2_pmovmskb_128:
22225 case Intrinsic::x86_avx2_pmovmskb: {
22226 // High bits of movmskp{s|d}, pmovmskb are known zero.
22228 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
22229 case Intrinsic::x86_sse_movmsk_ps: NumLoBits = 4; break;
22230 case Intrinsic::x86_avx_movmsk_ps_256: NumLoBits = 8; break;
22231 case Intrinsic::x86_sse2_movmsk_pd: NumLoBits = 2; break;
22232 case Intrinsic::x86_avx_movmsk_pd_256: NumLoBits = 4; break;
22233 case Intrinsic::x86_mmx_pmovmskb: NumLoBits = 8; break;
22234 case Intrinsic::x86_sse2_pmovmskb_128: NumLoBits = 16; break;
22235 case Intrinsic::x86_avx2_pmovmskb: NumLoBits = 32; break;
22237 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - NumLoBits);
22246 unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
22248 const SelectionDAG &,
22249 unsigned Depth) const {
22250 // SETCC_CARRY sets the dest to ~0 for true or 0 for false.
22251 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
22252 return Op.getValueType().getScalarType().getSizeInBits();
22258 /// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
22259 /// node is a GlobalAddress + offset.
22260 bool X86TargetLowering::isGAPlusOffset(SDNode *N,
22261 const GlobalValue* &GA,
22262 int64_t &Offset) const {
22263 if (N->getOpcode() == X86ISD::Wrapper) {
22264 if (isa<GlobalAddressSDNode>(N->getOperand(0))) {
22265 GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
22266 Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset();
22270 return TargetLowering::isGAPlusOffset(N, GA, Offset);
22273 /// isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the
22274 /// same as extracting the high 128-bit part of 256-bit vector and then
22275 /// inserting the result into the low part of a new 256-bit vector
22276 static bool isShuffleHigh128VectorInsertLow(ShuffleVectorSDNode *SVOp) {
22277 EVT VT = SVOp->getValueType(0);
22278 unsigned NumElems = VT.getVectorNumElements();
22280 // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
22281 for (unsigned i = 0, j = NumElems/2; i != NumElems/2; ++i, ++j)
22282 if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
22283 SVOp->getMaskElt(j) >= 0)
22289 /// isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the
22290 /// same as extracting the low 128-bit part of 256-bit vector and then
22291 /// inserting the result into the high part of a new 256-bit vector
22292 static bool isShuffleLow128VectorInsertHigh(ShuffleVectorSDNode *SVOp) {
22293 EVT VT = SVOp->getValueType(0);
22294 unsigned NumElems = VT.getVectorNumElements();
22296 // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
22297 for (unsigned i = NumElems/2, j = 0; i != NumElems; ++i, ++j)
22298 if (!isUndefOrEqual(SVOp->getMaskElt(i), j) ||
22299 SVOp->getMaskElt(j) >= 0)
22305 /// PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors.
22306 static SDValue PerformShuffleCombine256(SDNode *N, SelectionDAG &DAG,
22307 TargetLowering::DAGCombinerInfo &DCI,
22308 const X86Subtarget* Subtarget) {
22310 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
22311 SDValue V1 = SVOp->getOperand(0);
22312 SDValue V2 = SVOp->getOperand(1);
22313 EVT VT = SVOp->getValueType(0);
22314 unsigned NumElems = VT.getVectorNumElements();
22316 if (V1.getOpcode() == ISD::CONCAT_VECTORS &&
22317 V2.getOpcode() == ISD::CONCAT_VECTORS) {
22321 // V UNDEF BUILD_VECTOR UNDEF
22323 // CONCAT_VECTOR CONCAT_VECTOR
22326 // RESULT: V + zero extended
22328 if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR ||
22329 V2.getOperand(1).getOpcode() != ISD::UNDEF ||
22330 V1.getOperand(1).getOpcode() != ISD::UNDEF)
22333 if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()))
22336 // To match the shuffle mask, the first half of the mask should
22337 // be exactly the first vector, and all the rest a splat with the
22338 // first element of the second one.
22339 for (unsigned i = 0; i != NumElems/2; ++i)
22340 if (!isUndefOrEqual(SVOp->getMaskElt(i), i) ||
22341 !isUndefOrEqual(SVOp->getMaskElt(i+NumElems/2), NumElems))
22344 // If V1 is coming from a vector load then just fold to a VZEXT_LOAD.
22345 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(V1.getOperand(0))) {
22346 if (Ld->hasNUsesOfValue(1, 0)) {
22347 SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other);
22348 SDValue Ops[] = { Ld->getChain(), Ld->getBasePtr() };
22350 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
22352 Ld->getPointerInfo(),
22353 Ld->getAlignment(),
22354 false/*isVolatile*/, true/*ReadMem*/,
22355 false/*WriteMem*/);
22357 // Make sure the newly-created LOAD is in the same position as Ld in
22358 // terms of dependency. We create a TokenFactor for Ld and ResNode,
22359 // and update uses of Ld's output chain to use the TokenFactor.
22360 if (Ld->hasAnyUseOfValue(1)) {
22361 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
22362 SDValue(Ld, 1), SDValue(ResNode.getNode(), 1));
22363 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), NewChain);
22364 DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(Ld, 1),
22365 SDValue(ResNode.getNode(), 1));
22368 return DAG.getNode(ISD::BITCAST, dl, VT, ResNode);
22372 // Emit a zeroed vector and insert the desired subvector on its
22374 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
22375 SDValue InsV = Insert128BitVector(Zeros, V1.getOperand(0), 0, DAG, dl);
22376 return DCI.CombineTo(N, InsV);
22379 //===--------------------------------------------------------------------===//
22380 // Combine some shuffles into subvector extracts and inserts:
22383 // vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
22384 if (isShuffleHigh128VectorInsertLow(SVOp)) {
22385 SDValue V = Extract128BitVector(V1, NumElems/2, DAG, dl);
22386 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, 0, DAG, dl);
22387 return DCI.CombineTo(N, InsV);
22390 // vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
22391 if (isShuffleLow128VectorInsertHigh(SVOp)) {
22392 SDValue V = Extract128BitVector(V1, 0, DAG, dl);
22393 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, NumElems/2, DAG, dl);
22394 return DCI.CombineTo(N, InsV);
22400 /// \brief Combine an arbitrary chain of shuffles into a single instruction if
22403 /// This is the leaf of the recursive combinine below. When we have found some
22404 /// chain of single-use x86 shuffle instructions and accumulated the combined
22405 /// shuffle mask represented by them, this will try to pattern match that mask
22406 /// into either a single instruction if there is a special purpose instruction
22407 /// for this operation, or into a PSHUFB instruction which is a fully general
22408 /// instruction but should only be used to replace chains over a certain depth.
22409 static bool combineX86ShuffleChain(SDValue Op, SDValue Root, ArrayRef<int> Mask,
22410 int Depth, bool HasPSHUFB, SelectionDAG &DAG,
22411 TargetLowering::DAGCombinerInfo &DCI,
22412 const X86Subtarget *Subtarget) {
22413 assert(!Mask.empty() && "Cannot combine an empty shuffle mask!");
22415 // Find the operand that enters the chain. Note that multiple uses are OK
22416 // here, we're not going to remove the operand we find.
22417 SDValue Input = Op.getOperand(0);
22418 while (Input.getOpcode() == ISD::BITCAST)
22419 Input = Input.getOperand(0);
22421 MVT VT = Input.getSimpleValueType();
22422 MVT RootVT = Root.getSimpleValueType();
22425 // Just remove no-op shuffle masks.
22426 if (Mask.size() == 1) {
22427 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Input),
22432 // Use the float domain if the operand type is a floating point type.
22433 bool FloatDomain = VT.isFloatingPoint();
22435 // For floating point shuffles, we don't have free copies in the shuffle
22436 // instructions or the ability to load as part of the instruction, so
22437 // canonicalize their shuffles to UNPCK or MOV variants.
22439 // Note that even with AVX we prefer the PSHUFD form of shuffle for integer
22440 // vectors because it can have a load folded into it that UNPCK cannot. This
22441 // doesn't preclude something switching to the shorter encoding post-RA.
22443 if (Mask.equals(0, 0) || Mask.equals(1, 1)) {
22444 bool Lo = Mask.equals(0, 0);
22447 // Check if we have SSE3 which will let us use MOVDDUP. That instruction
22448 // is no slower than UNPCKLPD but has the option to fold the input operand
22449 // into even an unaligned memory load.
22450 if (Lo && Subtarget->hasSSE3()) {
22451 Shuffle = X86ISD::MOVDDUP;
22452 ShuffleVT = MVT::v2f64;
22454 // We have MOVLHPS and MOVHLPS throughout SSE and they encode smaller
22455 // than the UNPCK variants.
22456 Shuffle = Lo ? X86ISD::MOVLHPS : X86ISD::MOVHLPS;
22457 ShuffleVT = MVT::v4f32;
22459 if (Depth == 1 && Root->getOpcode() == Shuffle)
22460 return false; // Nothing to do!
22461 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22462 DCI.AddToWorklist(Op.getNode());
22463 if (Shuffle == X86ISD::MOVDDUP)
22464 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op);
22466 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
22467 DCI.AddToWorklist(Op.getNode());
22468 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22472 if (Subtarget->hasSSE3() &&
22473 (Mask.equals(0, 0, 2, 2) || Mask.equals(1, 1, 3, 3))) {
22474 bool Lo = Mask.equals(0, 0, 2, 2);
22475 unsigned Shuffle = Lo ? X86ISD::MOVSLDUP : X86ISD::MOVSHDUP;
22476 MVT ShuffleVT = MVT::v4f32;
22477 if (Depth == 1 && Root->getOpcode() == Shuffle)
22478 return false; // Nothing to do!
22479 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22480 DCI.AddToWorklist(Op.getNode());
22481 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op);
22482 DCI.AddToWorklist(Op.getNode());
22483 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22487 if (Mask.equals(0, 0, 1, 1) || Mask.equals(2, 2, 3, 3)) {
22488 bool Lo = Mask.equals(0, 0, 1, 1);
22489 unsigned Shuffle = Lo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
22490 MVT ShuffleVT = MVT::v4f32;
22491 if (Depth == 1 && Root->getOpcode() == Shuffle)
22492 return false; // Nothing to do!
22493 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22494 DCI.AddToWorklist(Op.getNode());
22495 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
22496 DCI.AddToWorklist(Op.getNode());
22497 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22503 // We always canonicalize the 8 x i16 and 16 x i8 shuffles into their UNPCK
22504 // variants as none of these have single-instruction variants that are
22505 // superior to the UNPCK formulation.
22506 if (!FloatDomain &&
22507 (Mask.equals(0, 0, 1, 1, 2, 2, 3, 3) ||
22508 Mask.equals(4, 4, 5, 5, 6, 6, 7, 7) ||
22509 Mask.equals(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7) ||
22510 Mask.equals(8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15,
22512 bool Lo = Mask[0] == 0;
22513 unsigned Shuffle = Lo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
22514 if (Depth == 1 && Root->getOpcode() == Shuffle)
22515 return false; // Nothing to do!
22517 switch (Mask.size()) {
22519 ShuffleVT = MVT::v8i16;
22522 ShuffleVT = MVT::v16i8;
22525 llvm_unreachable("Impossible mask size!");
22527 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input);
22528 DCI.AddToWorklist(Op.getNode());
22529 Op = DAG.getNode(Shuffle, DL, ShuffleVT, Op, Op);
22530 DCI.AddToWorklist(Op.getNode());
22531 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22536 // Don't try to re-form single instruction chains under any circumstances now
22537 // that we've done encoding canonicalization for them.
22541 // If we have 3 or more shuffle instructions or a chain involving PSHUFB, we
22542 // can replace them with a single PSHUFB instruction profitably. Intel's
22543 // manuals suggest only using PSHUFB if doing so replacing 5 instructions, but
22544 // in practice PSHUFB tends to be *very* fast so we're more aggressive.
22545 if ((Depth >= 3 || HasPSHUFB) && Subtarget->hasSSSE3()) {
22546 SmallVector<SDValue, 16> PSHUFBMask;
22547 assert(Mask.size() <= 16 && "Can't shuffle elements smaller than bytes!");
22548 int Ratio = 16 / Mask.size();
22549 for (unsigned i = 0; i < 16; ++i) {
22550 if (Mask[i / Ratio] == SM_SentinelUndef) {
22551 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
22554 int M = Mask[i / Ratio] != SM_SentinelZero
22555 ? Ratio * Mask[i / Ratio] + i % Ratio
22557 PSHUFBMask.push_back(DAG.getConstant(M, MVT::i8));
22559 Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Input);
22560 DCI.AddToWorklist(Op.getNode());
22561 SDValue PSHUFBMaskOp =
22562 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, PSHUFBMask);
22563 DCI.AddToWorklist(PSHUFBMaskOp.getNode());
22564 Op = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, Op, PSHUFBMaskOp);
22565 DCI.AddToWorklist(Op.getNode());
22566 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op),
22571 // Failed to find any combines.
22575 /// \brief Fully generic combining of x86 shuffle instructions.
22577 /// This should be the last combine run over the x86 shuffle instructions. Once
22578 /// they have been fully optimized, this will recursively consider all chains
22579 /// of single-use shuffle instructions, build a generic model of the cumulative
22580 /// shuffle operation, and check for simpler instructions which implement this
22581 /// operation. We use this primarily for two purposes:
22583 /// 1) Collapse generic shuffles to specialized single instructions when
22584 /// equivalent. In most cases, this is just an encoding size win, but
22585 /// sometimes we will collapse multiple generic shuffles into a single
22586 /// special-purpose shuffle.
22587 /// 2) Look for sequences of shuffle instructions with 3 or more total
22588 /// instructions, and replace them with the slightly more expensive SSSE3
22589 /// PSHUFB instruction if available. We do this as the last combining step
22590 /// to ensure we avoid using PSHUFB if we can implement the shuffle with
22591 /// a suitable short sequence of other instructions. The PHUFB will either
22592 /// use a register or have to read from memory and so is slightly (but only
22593 /// slightly) more expensive than the other shuffle instructions.
22595 /// Because this is inherently a quadratic operation (for each shuffle in
22596 /// a chain, we recurse up the chain), the depth is limited to 8 instructions.
22597 /// This should never be an issue in practice as the shuffle lowering doesn't
22598 /// produce sequences of more than 8 instructions.
22600 /// FIXME: We will currently miss some cases where the redundant shuffling
22601 /// would simplify under the threshold for PSHUFB formation because of
22602 /// combine-ordering. To fix this, we should do the redundant instruction
22603 /// combining in this recursive walk.
22604 static bool combineX86ShufflesRecursively(SDValue Op, SDValue Root,
22605 ArrayRef<int> RootMask,
22606 int Depth, bool HasPSHUFB,
22608 TargetLowering::DAGCombinerInfo &DCI,
22609 const X86Subtarget *Subtarget) {
22610 // Bound the depth of our recursive combine because this is ultimately
22611 // quadratic in nature.
22615 // Directly rip through bitcasts to find the underlying operand.
22616 while (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).hasOneUse())
22617 Op = Op.getOperand(0);
22619 MVT VT = Op.getSimpleValueType();
22620 if (!VT.isVector())
22621 return false; // Bail if we hit a non-vector.
22622 // FIXME: This routine should be taught about 256-bit shuffles, or a 256-bit
22623 // version should be added.
22624 if (VT.getSizeInBits() != 128)
22627 assert(Root.getSimpleValueType().isVector() &&
22628 "Shuffles operate on vector types!");
22629 assert(VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits() &&
22630 "Can only combine shuffles of the same vector register size.");
22632 if (!isTargetShuffle(Op.getOpcode()))
22634 SmallVector<int, 16> OpMask;
22636 bool HaveMask = getTargetShuffleMask(Op.getNode(), VT, OpMask, IsUnary);
22637 // We only can combine unary shuffles which we can decode the mask for.
22638 if (!HaveMask || !IsUnary)
22641 assert(VT.getVectorNumElements() == OpMask.size() &&
22642 "Different mask size from vector size!");
22643 assert(((RootMask.size() > OpMask.size() &&
22644 RootMask.size() % OpMask.size() == 0) ||
22645 (OpMask.size() > RootMask.size() &&
22646 OpMask.size() % RootMask.size() == 0) ||
22647 OpMask.size() == RootMask.size()) &&
22648 "The smaller number of elements must divide the larger.");
22649 int RootRatio = std::max<int>(1, OpMask.size() / RootMask.size());
22650 int OpRatio = std::max<int>(1, RootMask.size() / OpMask.size());
22651 assert(((RootRatio == 1 && OpRatio == 1) ||
22652 (RootRatio == 1) != (OpRatio == 1)) &&
22653 "Must not have a ratio for both incoming and op masks!");
22655 SmallVector<int, 16> Mask;
22656 Mask.reserve(std::max(OpMask.size(), RootMask.size()));
22658 // Merge this shuffle operation's mask into our accumulated mask. Note that
22659 // this shuffle's mask will be the first applied to the input, followed by the
22660 // root mask to get us all the way to the root value arrangement. The reason
22661 // for this order is that we are recursing up the operation chain.
22662 for (int i = 0, e = std::max(OpMask.size(), RootMask.size()); i < e; ++i) {
22663 int RootIdx = i / RootRatio;
22664 if (RootMask[RootIdx] < 0) {
22665 // This is a zero or undef lane, we're done.
22666 Mask.push_back(RootMask[RootIdx]);
22670 int RootMaskedIdx = RootMask[RootIdx] * RootRatio + i % RootRatio;
22671 int OpIdx = RootMaskedIdx / OpRatio;
22672 if (OpMask[OpIdx] < 0) {
22673 // The incoming lanes are zero or undef, it doesn't matter which ones we
22675 Mask.push_back(OpMask[OpIdx]);
22679 // Ok, we have non-zero lanes, map them through.
22680 Mask.push_back(OpMask[OpIdx] * OpRatio +
22681 RootMaskedIdx % OpRatio);
22684 // See if we can recurse into the operand to combine more things.
22685 switch (Op.getOpcode()) {
22686 case X86ISD::PSHUFB:
22688 case X86ISD::PSHUFD:
22689 case X86ISD::PSHUFHW:
22690 case X86ISD::PSHUFLW:
22691 if (Op.getOperand(0).hasOneUse() &&
22692 combineX86ShufflesRecursively(Op.getOperand(0), Root, Mask, Depth + 1,
22693 HasPSHUFB, DAG, DCI, Subtarget))
22697 case X86ISD::UNPCKL:
22698 case X86ISD::UNPCKH:
22699 assert(Op.getOperand(0) == Op.getOperand(1) && "We only combine unary shuffles!");
22700 // We can't check for single use, we have to check that this shuffle is the only user.
22701 if (Op->isOnlyUserOf(Op.getOperand(0).getNode()) &&
22702 combineX86ShufflesRecursively(Op.getOperand(0), Root, Mask, Depth + 1,
22703 HasPSHUFB, DAG, DCI, Subtarget))
22708 // Minor canonicalization of the accumulated shuffle mask to make it easier
22709 // to match below. All this does is detect masks with squential pairs of
22710 // elements, and shrink them to the half-width mask. It does this in a loop
22711 // so it will reduce the size of the mask to the minimal width mask which
22712 // performs an equivalent shuffle.
22713 SmallVector<int, 16> WidenedMask;
22714 while (Mask.size() > 1 && canWidenShuffleElements(Mask, WidenedMask)) {
22715 Mask = std::move(WidenedMask);
22716 WidenedMask.clear();
22719 return combineX86ShuffleChain(Op, Root, Mask, Depth, HasPSHUFB, DAG, DCI,
22723 /// \brief Get the PSHUF-style mask from PSHUF node.
22725 /// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
22726 /// PSHUF-style masks that can be reused with such instructions.
22727 static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
22728 SmallVector<int, 4> Mask;
22730 bool HaveMask = getTargetShuffleMask(N.getNode(), N.getSimpleValueType(), Mask, IsUnary);
22734 switch (N.getOpcode()) {
22735 case X86ISD::PSHUFD:
22737 case X86ISD::PSHUFLW:
22740 case X86ISD::PSHUFHW:
22741 Mask.erase(Mask.begin(), Mask.begin() + 4);
22742 for (int &M : Mask)
22746 llvm_unreachable("No valid shuffle instruction found!");
22750 /// \brief Search for a combinable shuffle across a chain ending in pshufd.
22752 /// We walk up the chain and look for a combinable shuffle, skipping over
22753 /// shuffles that we could hoist this shuffle's transformation past without
22754 /// altering anything.
22756 combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
22758 TargetLowering::DAGCombinerInfo &DCI) {
22759 assert(N.getOpcode() == X86ISD::PSHUFD &&
22760 "Called with something other than an x86 128-bit half shuffle!");
22763 // Walk up a single-use chain looking for a combinable shuffle. Keep a stack
22764 // of the shuffles in the chain so that we can form a fresh chain to replace
22766 SmallVector<SDValue, 8> Chain;
22767 SDValue V = N.getOperand(0);
22768 for (; V.hasOneUse(); V = V.getOperand(0)) {
22769 switch (V.getOpcode()) {
22771 return SDValue(); // Nothing combined!
22774 // Skip bitcasts as we always know the type for the target specific
22778 case X86ISD::PSHUFD:
22779 // Found another dword shuffle.
22782 case X86ISD::PSHUFLW:
22783 // Check that the low words (being shuffled) are the identity in the
22784 // dword shuffle, and the high words are self-contained.
22785 if (Mask[0] != 0 || Mask[1] != 1 ||
22786 !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
22789 Chain.push_back(V);
22792 case X86ISD::PSHUFHW:
22793 // Check that the high words (being shuffled) are the identity in the
22794 // dword shuffle, and the low words are self-contained.
22795 if (Mask[2] != 2 || Mask[3] != 3 ||
22796 !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
22799 Chain.push_back(V);
22802 case X86ISD::UNPCKL:
22803 case X86ISD::UNPCKH:
22804 // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
22805 // shuffle into a preceding word shuffle.
22806 if (V.getValueType() != MVT::v16i8 && V.getValueType() != MVT::v8i16)
22809 // Search for a half-shuffle which we can combine with.
22810 unsigned CombineOp =
22811 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
22812 if (V.getOperand(0) != V.getOperand(1) ||
22813 !V->isOnlyUserOf(V.getOperand(0).getNode()))
22815 Chain.push_back(V);
22816 V = V.getOperand(0);
22818 switch (V.getOpcode()) {
22820 return SDValue(); // Nothing to combine.
22822 case X86ISD::PSHUFLW:
22823 case X86ISD::PSHUFHW:
22824 if (V.getOpcode() == CombineOp)
22827 Chain.push_back(V);
22831 V = V.getOperand(0);
22835 } while (V.hasOneUse());
22838 // Break out of the loop if we break out of the switch.
22842 if (!V.hasOneUse())
22843 // We fell out of the loop without finding a viable combining instruction.
22846 // Merge this node's mask and our incoming mask.
22847 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
22848 for (int &M : Mask)
22850 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
22851 getV4X86ShuffleImm8ForMask(Mask, DAG));
22853 // Rebuild the chain around this new shuffle.
22854 while (!Chain.empty()) {
22855 SDValue W = Chain.pop_back_val();
22857 if (V.getValueType() != W.getOperand(0).getValueType())
22858 V = DAG.getNode(ISD::BITCAST, DL, W.getOperand(0).getValueType(), V);
22860 switch (W.getOpcode()) {
22862 llvm_unreachable("Only PSHUF and UNPCK instructions get here!");
22864 case X86ISD::UNPCKL:
22865 case X86ISD::UNPCKH:
22866 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
22869 case X86ISD::PSHUFD:
22870 case X86ISD::PSHUFLW:
22871 case X86ISD::PSHUFHW:
22872 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
22876 if (V.getValueType() != N.getValueType())
22877 V = DAG.getNode(ISD::BITCAST, DL, N.getValueType(), V);
22879 // Return the new chain to replace N.
22883 /// \brief Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.
22885 /// We walk up the chain, skipping shuffles of the other half and looking
22886 /// through shuffles which switch halves trying to find a shuffle of the same
22887 /// pair of dwords.
22888 static bool combineRedundantHalfShuffle(SDValue N, MutableArrayRef<int> Mask,
22890 TargetLowering::DAGCombinerInfo &DCI) {
22892 (N.getOpcode() == X86ISD::PSHUFLW || N.getOpcode() == X86ISD::PSHUFHW) &&
22893 "Called with something other than an x86 128-bit half shuffle!");
22895 unsigned CombineOpcode = N.getOpcode();
22897 // Walk up a single-use chain looking for a combinable shuffle.
22898 SDValue V = N.getOperand(0);
22899 for (; V.hasOneUse(); V = V.getOperand(0)) {
22900 switch (V.getOpcode()) {
22902 return false; // Nothing combined!
22905 // Skip bitcasts as we always know the type for the target specific
22909 case X86ISD::PSHUFLW:
22910 case X86ISD::PSHUFHW:
22911 if (V.getOpcode() == CombineOpcode)
22914 // Other-half shuffles are no-ops.
22917 // Break out of the loop if we break out of the switch.
22921 if (!V.hasOneUse())
22922 // We fell out of the loop without finding a viable combining instruction.
22925 // Combine away the bottom node as its shuffle will be accumulated into
22926 // a preceding shuffle.
22927 DCI.CombineTo(N.getNode(), N.getOperand(0), /*AddTo*/ true);
22929 // Record the old value.
22932 // Merge this node's mask and our incoming mask (adjusted to account for all
22933 // the pshufd instructions encountered).
22934 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
22935 for (int &M : Mask)
22937 V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0),
22938 getV4X86ShuffleImm8ForMask(Mask, DAG));
22940 // Check that the shuffles didn't cancel each other out. If not, we need to
22941 // combine to the new one.
22943 // Replace the combinable shuffle with the combined one, updating all users
22944 // so that we re-evaluate the chain here.
22945 DCI.CombineTo(Old.getNode(), V, /*AddTo*/ true);
22950 /// \brief Try to combine x86 target specific shuffles.
22951 static SDValue PerformTargetShuffleCombine(SDValue N, SelectionDAG &DAG,
22952 TargetLowering::DAGCombinerInfo &DCI,
22953 const X86Subtarget *Subtarget) {
22955 MVT VT = N.getSimpleValueType();
22956 SmallVector<int, 4> Mask;
22958 switch (N.getOpcode()) {
22959 case X86ISD::PSHUFD:
22960 case X86ISD::PSHUFLW:
22961 case X86ISD::PSHUFHW:
22962 Mask = getPSHUFShuffleMask(N);
22963 assert(Mask.size() == 4);
22969 // Nuke no-op shuffles that show up after combining.
22970 if (isNoopShuffleMask(Mask))
22971 return DCI.CombineTo(N.getNode(), N.getOperand(0), /*AddTo*/ true);
22973 // Look for simplifications involving one or two shuffle instructions.
22974 SDValue V = N.getOperand(0);
22975 switch (N.getOpcode()) {
22978 case X86ISD::PSHUFLW:
22979 case X86ISD::PSHUFHW:
22980 assert(VT == MVT::v8i16);
22983 if (combineRedundantHalfShuffle(N, Mask, DAG, DCI))
22984 return SDValue(); // We combined away this shuffle, so we're done.
22986 // See if this reduces to a PSHUFD which is no more expensive and can
22987 // combine with more operations. Note that it has to at least flip the
22988 // dwords as otherwise it would have been removed as a no-op.
22989 if (Mask[0] == 2 && Mask[1] == 3 && Mask[2] == 0 && Mask[3] == 1) {
22990 int DMask[] = {0, 1, 2, 3};
22991 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
22992 DMask[DOffset + 0] = DOffset + 1;
22993 DMask[DOffset + 1] = DOffset + 0;
22994 V = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V);
22995 DCI.AddToWorklist(V.getNode());
22996 V = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V,
22997 getV4X86ShuffleImm8ForMask(DMask, DAG));
22998 DCI.AddToWorklist(V.getNode());
22999 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V);
23002 // Look for shuffle patterns which can be implemented as a single unpack.
23003 // FIXME: This doesn't handle the location of the PSHUFD generically, and
23004 // only works when we have a PSHUFD followed by two half-shuffles.
23005 if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
23006 (V.getOpcode() == X86ISD::PSHUFLW ||
23007 V.getOpcode() == X86ISD::PSHUFHW) &&
23008 V.getOpcode() != N.getOpcode() &&
23010 SDValue D = V.getOperand(0);
23011 while (D.getOpcode() == ISD::BITCAST && D.hasOneUse())
23012 D = D.getOperand(0);
23013 if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
23014 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
23015 SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
23016 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
23017 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
23019 for (int i = 0; i < 4; ++i) {
23020 WordMask[i + NOffset] = Mask[i] + NOffset;
23021 WordMask[i + VOffset] = VMask[i] + VOffset;
23023 // Map the word mask through the DWord mask.
23025 for (int i = 0; i < 8; ++i)
23026 MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
23027 const int UnpackLoMask[] = {0, 0, 1, 1, 2, 2, 3, 3};
23028 const int UnpackHiMask[] = {4, 4, 5, 5, 6, 6, 7, 7};
23029 if (std::equal(std::begin(MappedMask), std::end(MappedMask),
23030 std::begin(UnpackLoMask)) ||
23031 std::equal(std::begin(MappedMask), std::end(MappedMask),
23032 std::begin(UnpackHiMask))) {
23033 // We can replace all three shuffles with an unpack.
23034 V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, D.getOperand(0));
23035 DCI.AddToWorklist(V.getNode());
23036 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
23038 DL, MVT::v8i16, V, V);
23045 case X86ISD::PSHUFD:
23046 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG, DCI))
23055 /// \brief Try to combine a shuffle into a target-specific add-sub node.
23057 /// We combine this directly on the abstract vector shuffle nodes so it is
23058 /// easier to generically match. We also insert dummy vector shuffle nodes for
23059 /// the operands which explicitly discard the lanes which are unused by this
23060 /// operation to try to flow through the rest of the combiner the fact that
23061 /// they're unused.
23062 static SDValue combineShuffleToAddSub(SDNode *N, SelectionDAG &DAG) {
23064 EVT VT = N->getValueType(0);
23066 // We only handle target-independent shuffles.
23067 // FIXME: It would be easy and harmless to use the target shuffle mask
23068 // extraction tool to support more.
23069 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
23072 auto *SVN = cast<ShuffleVectorSDNode>(N);
23073 ArrayRef<int> Mask = SVN->getMask();
23074 SDValue V1 = N->getOperand(0);
23075 SDValue V2 = N->getOperand(1);
23077 // We require the first shuffle operand to be the SUB node, and the second to
23078 // be the ADD node.
23079 // FIXME: We should support the commuted patterns.
23080 if (V1->getOpcode() != ISD::FSUB || V2->getOpcode() != ISD::FADD)
23083 // If there are other uses of these operations we can't fold them.
23084 if (!V1->hasOneUse() || !V2->hasOneUse())
23087 // Ensure that both operations have the same operands. Note that we can
23088 // commute the FADD operands.
23089 SDValue LHS = V1->getOperand(0), RHS = V1->getOperand(1);
23090 if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
23091 (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
23094 // We're looking for blends between FADD and FSUB nodes. We insist on these
23095 // nodes being lined up in a specific expected pattern.
23096 if (!(isShuffleEquivalent(V1, V2, Mask, 0, 3) ||
23097 isShuffleEquivalent(V1, V2, Mask, 0, 5, 2, 7) ||
23098 isShuffleEquivalent(V1, V2, Mask, 0, 9, 2, 11, 4, 13, 6, 15)))
23101 // Only specific types are legal at this point, assert so we notice if and
23102 // when these change.
23103 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v8f32 ||
23104 VT == MVT::v4f64) &&
23105 "Unknown vector type encountered!");
23107 return DAG.getNode(X86ISD::ADDSUB, DL, VT, LHS, RHS);
23110 /// PerformShuffleCombine - Performs several different shuffle combines.
23111 static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
23112 TargetLowering::DAGCombinerInfo &DCI,
23113 const X86Subtarget *Subtarget) {
23115 SDValue N0 = N->getOperand(0);
23116 SDValue N1 = N->getOperand(1);
23117 EVT VT = N->getValueType(0);
23119 // Don't create instructions with illegal types after legalize types has run.
23120 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23121 if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType()))
23124 // If we have legalized the vector types, look for blends of FADD and FSUB
23125 // nodes that we can fuse into an ADDSUB node.
23126 if (TLI.isTypeLegal(VT) && Subtarget->hasSSE3())
23127 if (SDValue AddSub = combineShuffleToAddSub(N, DAG))
23130 // Combine 256-bit vector shuffles. This is only profitable when in AVX mode
23131 if (Subtarget->hasFp256() && VT.is256BitVector() &&
23132 N->getOpcode() == ISD::VECTOR_SHUFFLE)
23133 return PerformShuffleCombine256(N, DAG, DCI, Subtarget);
23135 // During Type Legalization, when promoting illegal vector types,
23136 // the backend might introduce new shuffle dag nodes and bitcasts.
23138 // This code performs the following transformation:
23139 // fold: (shuffle (bitcast (BINOP A, B)), Undef, <Mask>) ->
23140 // (shuffle (BINOP (bitcast A), (bitcast B)), Undef, <Mask>)
23142 // We do this only if both the bitcast and the BINOP dag nodes have
23143 // one use. Also, perform this transformation only if the new binary
23144 // operation is legal. This is to avoid introducing dag nodes that
23145 // potentially need to be further expanded (or custom lowered) into a
23146 // less optimal sequence of dag nodes.
23147 if (!DCI.isBeforeLegalize() && DCI.isBeforeLegalizeOps() &&
23148 N1.getOpcode() == ISD::UNDEF && N0.hasOneUse() &&
23149 N0.getOpcode() == ISD::BITCAST) {
23150 SDValue BC0 = N0.getOperand(0);
23151 EVT SVT = BC0.getValueType();
23152 unsigned Opcode = BC0.getOpcode();
23153 unsigned NumElts = VT.getVectorNumElements();
23155 if (BC0.hasOneUse() && SVT.isVector() &&
23156 SVT.getVectorNumElements() * 2 == NumElts &&
23157 TLI.isOperationLegal(Opcode, VT)) {
23158 bool CanFold = false;
23170 unsigned SVTNumElts = SVT.getVectorNumElements();
23171 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
23172 for (unsigned i = 0, e = SVTNumElts; i != e && CanFold; ++i)
23173 CanFold = SVOp->getMaskElt(i) == (int)(i * 2);
23174 for (unsigned i = SVTNumElts, e = NumElts; i != e && CanFold; ++i)
23175 CanFold = SVOp->getMaskElt(i) < 0;
23178 SDValue BC00 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(0));
23179 SDValue BC01 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(1));
23180 SDValue NewBinOp = DAG.getNode(BC0.getOpcode(), dl, VT, BC00, BC01);
23181 return DAG.getVectorShuffle(VT, dl, NewBinOp, N1, &SVOp->getMask()[0]);
23186 // Only handle 128 wide vector from here on.
23187 if (!VT.is128BitVector())
23190 // Combine a vector_shuffle that is equal to build_vector load1, load2, load3,
23191 // load4, <0, 1, 2, 3> into a 128-bit load if the load addresses are
23192 // consecutive, non-overlapping, and in the right order.
23193 SmallVector<SDValue, 16> Elts;
23194 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
23195 Elts.push_back(getShuffleScalarElt(N, i, DAG, 0));
23197 SDValue LD = EltsFromConsecutiveLoads(VT, Elts, dl, DAG, true);
23201 if (isTargetShuffle(N->getOpcode())) {
23203 PerformTargetShuffleCombine(SDValue(N, 0), DAG, DCI, Subtarget);
23204 if (Shuffle.getNode())
23207 // Try recursively combining arbitrary sequences of x86 shuffle
23208 // instructions into higher-order shuffles. We do this after combining
23209 // specific PSHUF instruction sequences into their minimal form so that we
23210 // can evaluate how many specialized shuffle instructions are involved in
23211 // a particular chain.
23212 SmallVector<int, 1> NonceMask; // Just a placeholder.
23213 NonceMask.push_back(0);
23214 if (combineX86ShufflesRecursively(SDValue(N, 0), SDValue(N, 0), NonceMask,
23215 /*Depth*/ 1, /*HasPSHUFB*/ false, DAG,
23217 return SDValue(); // This routine will use CombineTo to replace N.
23223 /// PerformTruncateCombine - Converts truncate operation to
23224 /// a sequence of vector shuffle operations.
23225 /// It is possible when we truncate 256-bit vector to 128-bit vector
23226 static SDValue PerformTruncateCombine(SDNode *N, SelectionDAG &DAG,
23227 TargetLowering::DAGCombinerInfo &DCI,
23228 const X86Subtarget *Subtarget) {
23232 /// XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target
23233 /// specific shuffle of a load can be folded into a single element load.
23234 /// Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but
23235 /// shuffles have been custom lowered so we need to handle those here.
23236 static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
23237 TargetLowering::DAGCombinerInfo &DCI) {
23238 if (DCI.isBeforeLegalizeOps())
23241 SDValue InVec = N->getOperand(0);
23242 SDValue EltNo = N->getOperand(1);
23244 if (!isa<ConstantSDNode>(EltNo))
23247 EVT OriginalVT = InVec.getValueType();
23249 if (InVec.getOpcode() == ISD::BITCAST) {
23250 // Don't duplicate a load with other uses.
23251 if (!InVec.hasOneUse())
23253 EVT BCVT = InVec.getOperand(0).getValueType();
23254 if (BCVT.getVectorNumElements() != OriginalVT.getVectorNumElements())
23256 InVec = InVec.getOperand(0);
23259 EVT CurrentVT = InVec.getValueType();
23261 if (!isTargetShuffle(InVec.getOpcode()))
23264 // Don't duplicate a load with other uses.
23265 if (!InVec.hasOneUse())
23268 SmallVector<int, 16> ShuffleMask;
23270 if (!getTargetShuffleMask(InVec.getNode(), CurrentVT.getSimpleVT(),
23271 ShuffleMask, UnaryShuffle))
23274 // Select the input vector, guarding against out of range extract vector.
23275 unsigned NumElems = CurrentVT.getVectorNumElements();
23276 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
23277 int Idx = (Elt > (int)NumElems) ? -1 : ShuffleMask[Elt];
23278 SDValue LdNode = (Idx < (int)NumElems) ? InVec.getOperand(0)
23279 : InVec.getOperand(1);
23281 // If inputs to shuffle are the same for both ops, then allow 2 uses
23282 unsigned AllowedUses = InVec.getNumOperands() > 1 &&
23283 InVec.getOperand(0) == InVec.getOperand(1) ? 2 : 1;
23285 if (LdNode.getOpcode() == ISD::BITCAST) {
23286 // Don't duplicate a load with other uses.
23287 if (!LdNode.getNode()->hasNUsesOfValue(AllowedUses, 0))
23290 AllowedUses = 1; // only allow 1 load use if we have a bitcast
23291 LdNode = LdNode.getOperand(0);
23294 if (!ISD::isNormalLoad(LdNode.getNode()))
23297 LoadSDNode *LN0 = cast<LoadSDNode>(LdNode);
23299 if (!LN0 ||!LN0->hasNUsesOfValue(AllowedUses, 0) || LN0->isVolatile())
23302 EVT EltVT = N->getValueType(0);
23303 // If there's a bitcast before the shuffle, check if the load type and
23304 // alignment is valid.
23305 unsigned Align = LN0->getAlignment();
23306 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23307 unsigned NewAlign = TLI.getDataLayout()->getABITypeAlignment(
23308 EltVT.getTypeForEVT(*DAG.getContext()));
23310 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, EltVT))
23313 // All checks match so transform back to vector_shuffle so that DAG combiner
23314 // can finish the job
23317 // Create shuffle node taking into account the case that its a unary shuffle
23318 SDValue Shuffle = (UnaryShuffle) ? DAG.getUNDEF(CurrentVT)
23319 : InVec.getOperand(1);
23320 Shuffle = DAG.getVectorShuffle(CurrentVT, dl,
23321 InVec.getOperand(0), Shuffle,
23323 Shuffle = DAG.getNode(ISD::BITCAST, dl, OriginalVT, Shuffle);
23324 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), Shuffle,
23328 /// \brief Detect bitcasts between i32 to x86mmx low word. Since MMX types are
23329 /// special and don't usually play with other vector types, it's better to
23330 /// handle them early to be sure we emit efficient code by avoiding
23331 /// store-load conversions.
23332 static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG) {
23333 if (N->getValueType(0) != MVT::x86mmx ||
23334 N->getOperand(0)->getOpcode() != ISD::BUILD_VECTOR ||
23335 N->getOperand(0)->getValueType(0) != MVT::v2i32)
23338 SDValue V = N->getOperand(0);
23339 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
23340 if (C && C->getZExtValue() == 0 && V.getOperand(0).getValueType() == MVT::i32)
23341 return DAG.getNode(X86ISD::MMX_MOVW2D, SDLoc(V.getOperand(0)),
23342 N->getValueType(0), V.getOperand(0));
23347 /// PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index
23348 /// generation and convert it from being a bunch of shuffles and extracts
23349 /// into a somewhat faster sequence. For i686, the best sequence is apparently
23350 /// storing the value and loading scalars back, while for x64 we should
23351 /// use 64-bit extracts and shifts.
23352 static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
23353 TargetLowering::DAGCombinerInfo &DCI) {
23354 SDValue NewOp = XFormVExtractWithShuffleIntoLoad(N, DAG, DCI);
23355 if (NewOp.getNode())
23358 SDValue InputVector = N->getOperand(0);
23360 // Detect mmx to i32 conversion through a v2i32 elt extract.
23361 if (InputVector.getOpcode() == ISD::BITCAST && InputVector.hasOneUse() &&
23362 N->getValueType(0) == MVT::i32 &&
23363 InputVector.getValueType() == MVT::v2i32) {
23365 // The bitcast source is a direct mmx result.
23366 SDValue MMXSrc = InputVector.getNode()->getOperand(0);
23367 if (MMXSrc.getValueType() == MVT::x86mmx)
23368 return DAG.getNode(X86ISD::MMX_MOVD2W, SDLoc(InputVector),
23369 N->getValueType(0),
23370 InputVector.getNode()->getOperand(0));
23372 // The mmx is indirect: (i64 extract_elt (v1i64 bitcast (x86mmx ...))).
23373 SDValue MMXSrcOp = MMXSrc.getOperand(0);
23374 if (MMXSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT && MMXSrc.hasOneUse() &&
23375 MMXSrc.getValueType() == MVT::i64 && MMXSrcOp.hasOneUse() &&
23376 MMXSrcOp.getOpcode() == ISD::BITCAST &&
23377 MMXSrcOp.getValueType() == MVT::v1i64 &&
23378 MMXSrcOp.getOperand(0).getValueType() == MVT::x86mmx)
23379 return DAG.getNode(X86ISD::MMX_MOVD2W, SDLoc(InputVector),
23380 N->getValueType(0),
23381 MMXSrcOp.getOperand(0));
23384 // Only operate on vectors of 4 elements, where the alternative shuffling
23385 // gets to be more expensive.
23386 if (InputVector.getValueType() != MVT::v4i32)
23389 // Check whether every use of InputVector is an EXTRACT_VECTOR_ELT with a
23390 // single use which is a sign-extend or zero-extend, and all elements are
23392 SmallVector<SDNode *, 4> Uses;
23393 unsigned ExtractedElements = 0;
23394 for (SDNode::use_iterator UI = InputVector.getNode()->use_begin(),
23395 UE = InputVector.getNode()->use_end(); UI != UE; ++UI) {
23396 if (UI.getUse().getResNo() != InputVector.getResNo())
23399 SDNode *Extract = *UI;
23400 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
23403 if (Extract->getValueType(0) != MVT::i32)
23405 if (!Extract->hasOneUse())
23407 if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND &&
23408 Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND)
23410 if (!isa<ConstantSDNode>(Extract->getOperand(1)))
23413 // Record which element was extracted.
23414 ExtractedElements |=
23415 1 << cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
23417 Uses.push_back(Extract);
23420 // If not all the elements were used, this may not be worthwhile.
23421 if (ExtractedElements != 15)
23424 // Ok, we've now decided to do the transformation.
23425 // If 64-bit shifts are legal, use the extract-shift sequence,
23426 // otherwise bounce the vector off the cache.
23427 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23429 SDLoc dl(InputVector);
23431 if (TLI.isOperationLegal(ISD::SRA, MVT::i64)) {
23432 SDValue Cst = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, InputVector);
23433 EVT VecIdxTy = DAG.getTargetLoweringInfo().getVectorIdxTy();
23434 SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst,
23435 DAG.getConstant(0, VecIdxTy));
23436 SDValue TopHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst,
23437 DAG.getConstant(1, VecIdxTy));
23439 SDValue ShAmt = DAG.getConstant(32,
23440 DAG.getTargetLoweringInfo().getShiftAmountTy(MVT::i64));
23441 Vals[0] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BottomHalf);
23442 Vals[1] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
23443 DAG.getNode(ISD::SRA, dl, MVT::i64, BottomHalf, ShAmt));
23444 Vals[2] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, TopHalf);
23445 Vals[3] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
23446 DAG.getNode(ISD::SRA, dl, MVT::i64, TopHalf, ShAmt));
23448 // Store the value to a temporary stack slot.
23449 SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType());
23450 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr,
23451 MachinePointerInfo(), false, false, 0);
23453 EVT ElementType = InputVector.getValueType().getVectorElementType();
23454 unsigned EltSize = ElementType.getSizeInBits() / 8;
23456 // Replace each use (extract) with a load of the appropriate element.
23457 for (unsigned i = 0; i < 4; ++i) {
23458 uint64_t Offset = EltSize * i;
23459 SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy());
23461 SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(),
23462 StackPtr, OffsetVal);
23464 // Load the scalar.
23465 Vals[i] = DAG.getLoad(ElementType, dl, Ch,
23466 ScalarAddr, MachinePointerInfo(),
23467 false, false, false, 0);
23472 // Replace the extracts
23473 for (SmallVectorImpl<SDNode *>::iterator UI = Uses.begin(),
23474 UE = Uses.end(); UI != UE; ++UI) {
23475 SDNode *Extract = *UI;
23477 SDValue Idx = Extract->getOperand(1);
23478 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
23479 DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), Vals[IdxVal]);
23482 // The replacement was made in place; don't return anything.
23486 /// \brief Matches a VSELECT onto min/max or return 0 if the node doesn't match.
23487 static std::pair<unsigned, bool>
23488 matchIntegerMINMAX(SDValue Cond, EVT VT, SDValue LHS, SDValue RHS,
23489 SelectionDAG &DAG, const X86Subtarget *Subtarget) {
23490 if (!VT.isVector())
23491 return std::make_pair(0, false);
23493 bool NeedSplit = false;
23494 switch (VT.getSimpleVT().SimpleTy) {
23495 default: return std::make_pair(0, false);
23498 if (!Subtarget->hasVLX())
23499 return std::make_pair(0, false);
23503 if (!Subtarget->hasBWI())
23504 return std::make_pair(0, false);
23508 if (!Subtarget->hasAVX512())
23509 return std::make_pair(0, false);
23514 if (!Subtarget->hasAVX2())
23516 if (!Subtarget->hasAVX())
23517 return std::make_pair(0, false);
23522 if (!Subtarget->hasSSE2())
23523 return std::make_pair(0, false);
23526 // SSE2 has only a small subset of the operations.
23527 bool hasUnsigned = Subtarget->hasSSE41() ||
23528 (Subtarget->hasSSE2() && VT == MVT::v16i8);
23529 bool hasSigned = Subtarget->hasSSE41() ||
23530 (Subtarget->hasSSE2() && VT == MVT::v8i16);
23532 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23535 // Check for x CC y ? x : y.
23536 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
23537 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
23542 Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
23545 Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
23548 Opc = hasSigned ? X86ISD::SMIN : 0; break;
23551 Opc = hasSigned ? X86ISD::SMAX : 0; break;
23553 // Check for x CC y ? y : x -- a min/max with reversed arms.
23554 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
23555 DAG.isEqualTo(RHS, Cond.getOperand(0))) {
23560 Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
23563 Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
23566 Opc = hasSigned ? X86ISD::SMAX : 0; break;
23569 Opc = hasSigned ? X86ISD::SMIN : 0; break;
23573 return std::make_pair(Opc, NeedSplit);
23577 transformVSELECTtoBlendVECTOR_SHUFFLE(SDNode *N, SelectionDAG &DAG,
23578 const X86Subtarget *Subtarget) {
23580 SDValue Cond = N->getOperand(0);
23581 SDValue LHS = N->getOperand(1);
23582 SDValue RHS = N->getOperand(2);
23584 if (Cond.getOpcode() == ISD::SIGN_EXTEND) {
23585 SDValue CondSrc = Cond->getOperand(0);
23586 if (CondSrc->getOpcode() == ISD::SIGN_EXTEND_INREG)
23587 Cond = CondSrc->getOperand(0);
23590 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
23593 // A vselect where all conditions and data are constants can be optimized into
23594 // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
23595 if (ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) &&
23596 ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()))
23599 unsigned MaskValue = 0;
23600 if (!BUILD_VECTORtoBlendMask(cast<BuildVectorSDNode>(Cond), MaskValue))
23603 MVT VT = N->getSimpleValueType(0);
23604 unsigned NumElems = VT.getVectorNumElements();
23605 SmallVector<int, 8> ShuffleMask(NumElems, -1);
23606 for (unsigned i = 0; i < NumElems; ++i) {
23607 // Be sure we emit undef where we can.
23608 if (Cond.getOperand(i)->getOpcode() == ISD::UNDEF)
23609 ShuffleMask[i] = -1;
23611 ShuffleMask[i] = i + NumElems * ((MaskValue >> i) & 1);
23614 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23615 if (!TLI.isShuffleMaskLegal(ShuffleMask, VT))
23617 return DAG.getVectorShuffle(VT, dl, LHS, RHS, &ShuffleMask[0]);
23620 /// PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT
23622 static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
23623 TargetLowering::DAGCombinerInfo &DCI,
23624 const X86Subtarget *Subtarget) {
23626 SDValue Cond = N->getOperand(0);
23627 // Get the LHS/RHS of the select.
23628 SDValue LHS = N->getOperand(1);
23629 SDValue RHS = N->getOperand(2);
23630 EVT VT = LHS.getValueType();
23631 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23633 // If we have SSE[12] support, try to form min/max nodes. SSE min/max
23634 // instructions match the semantics of the common C idiom x<y?x:y but not
23635 // x<=y?x:y, because of how they handle negative zero (which can be
23636 // ignored in unsafe-math mode).
23637 // We also try to create v2f32 min/max nodes, which we later widen to v4f32.
23638 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
23639 VT != MVT::f80 && (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
23640 (Subtarget->hasSSE2() ||
23641 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) {
23642 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23644 unsigned Opcode = 0;
23645 // Check for x CC y ? x : y.
23646 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
23647 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
23651 // Converting this to a min would handle NaNs incorrectly, and swapping
23652 // the operands would cause it to handle comparisons between positive
23653 // and negative zero incorrectly.
23654 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
23655 if (!DAG.getTarget().Options.UnsafeFPMath &&
23656 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
23658 std::swap(LHS, RHS);
23660 Opcode = X86ISD::FMIN;
23663 // Converting this to a min would handle comparisons between positive
23664 // and negative zero incorrectly.
23665 if (!DAG.getTarget().Options.UnsafeFPMath &&
23666 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
23668 Opcode = X86ISD::FMIN;
23671 // Converting this to a min would handle both negative zeros and NaNs
23672 // incorrectly, but we can swap the operands to fix both.
23673 std::swap(LHS, RHS);
23677 Opcode = X86ISD::FMIN;
23681 // Converting this to a max would handle comparisons between positive
23682 // and negative zero incorrectly.
23683 if (!DAG.getTarget().Options.UnsafeFPMath &&
23684 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
23686 Opcode = X86ISD::FMAX;
23689 // Converting this to a max would handle NaNs incorrectly, and swapping
23690 // the operands would cause it to handle comparisons between positive
23691 // and negative zero incorrectly.
23692 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
23693 if (!DAG.getTarget().Options.UnsafeFPMath &&
23694 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
23696 std::swap(LHS, RHS);
23698 Opcode = X86ISD::FMAX;
23701 // Converting this to a max would handle both negative zeros and NaNs
23702 // incorrectly, but we can swap the operands to fix both.
23703 std::swap(LHS, RHS);
23707 Opcode = X86ISD::FMAX;
23710 // Check for x CC y ? y : x -- a min/max with reversed arms.
23711 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
23712 DAG.isEqualTo(RHS, Cond.getOperand(0))) {
23716 // Converting this to a min would handle comparisons between positive
23717 // and negative zero incorrectly, and swapping the operands would
23718 // cause it to handle NaNs incorrectly.
23719 if (!DAG.getTarget().Options.UnsafeFPMath &&
23720 !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) {
23721 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
23723 std::swap(LHS, RHS);
23725 Opcode = X86ISD::FMIN;
23728 // Converting this to a min would handle NaNs incorrectly.
23729 if (!DAG.getTarget().Options.UnsafeFPMath &&
23730 (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
23732 Opcode = X86ISD::FMIN;
23735 // Converting this to a min would handle both negative zeros and NaNs
23736 // incorrectly, but we can swap the operands to fix both.
23737 std::swap(LHS, RHS);
23741 Opcode = X86ISD::FMIN;
23745 // Converting this to a max would handle NaNs incorrectly.
23746 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
23748 Opcode = X86ISD::FMAX;
23751 // Converting this to a max would handle comparisons between positive
23752 // and negative zero incorrectly, and swapping the operands would
23753 // cause it to handle NaNs incorrectly.
23754 if (!DAG.getTarget().Options.UnsafeFPMath &&
23755 !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) {
23756 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
23758 std::swap(LHS, RHS);
23760 Opcode = X86ISD::FMAX;
23763 // Converting this to a max would handle both negative zeros and NaNs
23764 // incorrectly, but we can swap the operands to fix both.
23765 std::swap(LHS, RHS);
23769 Opcode = X86ISD::FMAX;
23775 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
23778 EVT CondVT = Cond.getValueType();
23779 if (Subtarget->hasAVX512() && VT.isVector() && CondVT.isVector() &&
23780 CondVT.getVectorElementType() == MVT::i1) {
23781 // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
23782 // lowering on KNL. In this case we convert it to
23783 // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
23784 // The same situation for all 128 and 256-bit vectors of i8 and i16.
23785 // Since SKX these selects have a proper lowering.
23786 EVT OpVT = LHS.getValueType();
23787 if ((OpVT.is128BitVector() || OpVT.is256BitVector()) &&
23788 (OpVT.getVectorElementType() == MVT::i8 ||
23789 OpVT.getVectorElementType() == MVT::i16) &&
23790 !(Subtarget->hasBWI() && Subtarget->hasVLX())) {
23791 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, OpVT, Cond);
23792 DCI.AddToWorklist(Cond.getNode());
23793 return DAG.getNode(N->getOpcode(), DL, OpVT, Cond, LHS, RHS);
23796 // If this is a select between two integer constants, try to do some
23798 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) {
23799 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS))
23800 // Don't do this for crazy integer types.
23801 if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) {
23802 // If this is efficiently invertible, canonicalize the LHSC/RHSC values
23803 // so that TrueC (the true value) is larger than FalseC.
23804 bool NeedsCondInvert = false;
23806 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) &&
23807 // Efficiently invertible.
23808 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible.
23809 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible.
23810 isa<ConstantSDNode>(Cond.getOperand(1))))) {
23811 NeedsCondInvert = true;
23812 std::swap(TrueC, FalseC);
23815 // Optimize C ? 8 : 0 -> zext(C) << 3. Likewise for any pow2/0.
23816 if (FalseC->getAPIntValue() == 0 &&
23817 TrueC->getAPIntValue().isPowerOf2()) {
23818 if (NeedsCondInvert) // Invert the condition if needed.
23819 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
23820 DAG.getConstant(1, Cond.getValueType()));
23822 // Zero extend the condition if needed.
23823 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond);
23825 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
23826 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond,
23827 DAG.getConstant(ShAmt, MVT::i8));
23830 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.
23831 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
23832 if (NeedsCondInvert) // Invert the condition if needed.
23833 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
23834 DAG.getConstant(1, Cond.getValueType()));
23836 // Zero extend the condition if needed.
23837 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
23838 FalseC->getValueType(0), Cond);
23839 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
23840 SDValue(FalseC, 0));
23843 // Optimize cases that will turn into an LEA instruction. This requires
23844 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
23845 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
23846 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
23847 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
23849 bool isFastMultiplier = false;
23851 switch ((unsigned char)Diff) {
23853 case 1: // result = add base, cond
23854 case 2: // result = lea base( , cond*2)
23855 case 3: // result = lea base(cond, cond*2)
23856 case 4: // result = lea base( , cond*4)
23857 case 5: // result = lea base(cond, cond*4)
23858 case 8: // result = lea base( , cond*8)
23859 case 9: // result = lea base(cond, cond*8)
23860 isFastMultiplier = true;
23865 if (isFastMultiplier) {
23866 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
23867 if (NeedsCondInvert) // Invert the condition if needed.
23868 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
23869 DAG.getConstant(1, Cond.getValueType()));
23871 // Zero extend the condition if needed.
23872 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
23874 // Scale the condition by the difference.
23876 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
23877 DAG.getConstant(Diff, Cond.getValueType()));
23879 // Add the base if non-zero.
23880 if (FalseC->getAPIntValue() != 0)
23881 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
23882 SDValue(FalseC, 0));
23889 // Canonicalize max and min:
23890 // (x > y) ? x : y -> (x >= y) ? x : y
23891 // (x < y) ? x : y -> (x <= y) ? x : y
23892 // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
23893 // the need for an extra compare
23894 // against zero. e.g.
23895 // (x - y) > 0 : (x - y) ? 0 -> (x - y) >= 0 : (x - y) ? 0
23897 // testl %edi, %edi
23899 // cmovgl %edi, %eax
23903 // cmovsl %eax, %edi
23904 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
23905 DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
23906 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
23907 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23912 ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE;
23913 Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(),
23914 Cond.getOperand(0), Cond.getOperand(1), NewCC);
23915 return DAG.getNode(ISD::SELECT, DL, VT, Cond, LHS, RHS);
23920 // Early exit check
23921 if (!TLI.isTypeLegal(VT))
23924 // Match VSELECTs into subs with unsigned saturation.
23925 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
23926 // psubus is available in SSE2 and AVX2 for i8 and i16 vectors.
23927 ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) ||
23928 (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) {
23929 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
23931 // Check if one of the arms of the VSELECT is a zero vector. If it's on the
23932 // left side invert the predicate to simplify logic below.
23934 if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
23936 CC = ISD::getSetCCInverse(CC, true);
23937 } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) {
23941 if (Other.getNode() && Other->getNumOperands() == 2 &&
23942 DAG.isEqualTo(Other->getOperand(0), Cond.getOperand(0))) {
23943 SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
23944 SDValue CondRHS = Cond->getOperand(1);
23946 // Look for a general sub with unsigned saturation first.
23947 // x >= y ? x-y : 0 --> subus x, y
23948 // x > y ? x-y : 0 --> subus x, y
23949 if ((CC == ISD::SETUGE || CC == ISD::SETUGT) &&
23950 Other->getOpcode() == ISD::SUB && DAG.isEqualTo(OpRHS, CondRHS))
23951 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
23953 if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS))
23954 if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
23955 if (auto *CondRHSBV = dyn_cast<BuildVectorSDNode>(CondRHS))
23956 if (auto *CondRHSConst = CondRHSBV->getConstantSplatNode())
23957 // If the RHS is a constant we have to reverse the const
23958 // canonicalization.
23959 // x > C-1 ? x+-C : 0 --> subus x, C
23960 if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD &&
23961 CondRHSConst->getAPIntValue() ==
23962 (-OpRHSConst->getAPIntValue() - 1))
23963 return DAG.getNode(
23964 X86ISD::SUBUS, DL, VT, OpLHS,
23965 DAG.getConstant(-OpRHSConst->getAPIntValue(), VT));
23967 // Another special case: If C was a sign bit, the sub has been
23968 // canonicalized into a xor.
23969 // FIXME: Would it be better to use computeKnownBits to determine
23970 // whether it's safe to decanonicalize the xor?
23971 // x s< 0 ? x^C : 0 --> subus x, C
23972 if (CC == ISD::SETLT && Other->getOpcode() == ISD::XOR &&
23973 ISD::isBuildVectorAllZeros(CondRHS.getNode()) &&
23974 OpRHSConst->getAPIntValue().isSignBit())
23975 // Note that we have to rebuild the RHS constant here to ensure we
23976 // don't rely on particular values of undef lanes.
23977 return DAG.getNode(
23978 X86ISD::SUBUS, DL, VT, OpLHS,
23979 DAG.getConstant(OpRHSConst->getAPIntValue(), VT));
23984 // Try to match a min/max vector operation.
23985 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC) {
23986 std::pair<unsigned, bool> ret = matchIntegerMINMAX(Cond, VT, LHS, RHS, DAG, Subtarget);
23987 unsigned Opc = ret.first;
23988 bool NeedSplit = ret.second;
23990 if (Opc && NeedSplit) {
23991 unsigned NumElems = VT.getVectorNumElements();
23992 // Extract the LHS vectors
23993 SDValue LHS1 = Extract128BitVector(LHS, 0, DAG, DL);
23994 SDValue LHS2 = Extract128BitVector(LHS, NumElems/2, DAG, DL);
23996 // Extract the RHS vectors
23997 SDValue RHS1 = Extract128BitVector(RHS, 0, DAG, DL);
23998 SDValue RHS2 = Extract128BitVector(RHS, NumElems/2, DAG, DL);
24000 // Create min/max for each subvector
24001 LHS = DAG.getNode(Opc, DL, LHS1.getValueType(), LHS1, RHS1);
24002 RHS = DAG.getNode(Opc, DL, LHS2.getValueType(), LHS2, RHS2);
24004 // Merge the result
24005 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS, RHS);
24007 return DAG.getNode(Opc, DL, VT, LHS, RHS);
24010 // Simplify vector selection if condition value type matches vselect
24012 if (N->getOpcode() == ISD::VSELECT && CondVT == VT) {
24013 assert(Cond.getValueType().isVector() &&
24014 "vector select expects a vector selector!");
24016 bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
24017 bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
24019 // Try invert the condition if true value is not all 1s and false value
24021 if (!TValIsAllOnes && !FValIsAllZeros &&
24022 // Check if the selector will be produced by CMPP*/PCMP*
24023 Cond.getOpcode() == ISD::SETCC &&
24024 // Check if SETCC has already been promoted
24025 TLI.getSetCCResultType(*DAG.getContext(), VT) == CondVT) {
24026 bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
24027 bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
24029 if (TValIsAllZeros || FValIsAllOnes) {
24030 SDValue CC = Cond.getOperand(2);
24031 ISD::CondCode NewCC =
24032 ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
24033 Cond.getOperand(0).getValueType().isInteger());
24034 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1), NewCC);
24035 std::swap(LHS, RHS);
24036 TValIsAllOnes = FValIsAllOnes;
24037 FValIsAllZeros = TValIsAllZeros;
24041 if (TValIsAllOnes || FValIsAllZeros) {
24044 if (TValIsAllOnes && FValIsAllZeros)
24046 else if (TValIsAllOnes)
24047 Ret = DAG.getNode(ISD::OR, DL, CondVT, Cond,
24048 DAG.getNode(ISD::BITCAST, DL, CondVT, RHS));
24049 else if (FValIsAllZeros)
24050 Ret = DAG.getNode(ISD::AND, DL, CondVT, Cond,
24051 DAG.getNode(ISD::BITCAST, DL, CondVT, LHS));
24053 return DAG.getNode(ISD::BITCAST, DL, VT, Ret);
24057 // If we know that this node is legal then we know that it is going to be
24058 // matched by one of the SSE/AVX BLEND instructions. These instructions only
24059 // depend on the highest bit in each word. Try to use SimplifyDemandedBits
24060 // to simplify previous instructions.
24061 if (N->getOpcode() == ISD::VSELECT && DCI.isBeforeLegalizeOps() &&
24062 !DCI.isBeforeLegalize() &&
24063 // We explicitly check against v8i16 and v16i16 because, although
24064 // they're marked as Custom, they might only be legal when Cond is a
24065 // build_vector of constants. This will be taken care in a later
24067 (TLI.isOperationLegalOrCustom(ISD::VSELECT, VT) && VT != MVT::v16i16 &&
24068 VT != MVT::v8i16) &&
24069 // Don't optimize vector of constants. Those are handled by
24070 // the generic code and all the bits must be properly set for
24071 // the generic optimizer.
24072 !ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) {
24073 unsigned BitWidth = Cond.getValueType().getScalarType().getSizeInBits();
24075 // Don't optimize vector selects that map to mask-registers.
24079 assert(BitWidth >= 8 && BitWidth <= 64 && "Invalid mask size");
24080 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 1);
24082 APInt KnownZero, KnownOne;
24083 TargetLowering::TargetLoweringOpt TLO(DAG, DCI.isBeforeLegalize(),
24084 DCI.isBeforeLegalizeOps());
24085 if (TLO.ShrinkDemandedConstant(Cond, DemandedMask) ||
24086 TLI.SimplifyDemandedBits(Cond, DemandedMask, KnownZero, KnownOne,
24088 // If we changed the computation somewhere in the DAG, this change
24089 // will affect all users of Cond.
24090 // Make sure it is fine and update all the nodes so that we do not
24091 // use the generic VSELECT anymore. Otherwise, we may perform
24092 // wrong optimizations as we messed up with the actual expectation
24093 // for the vector boolean values.
24094 if (Cond != TLO.Old) {
24095 // Check all uses of that condition operand to check whether it will be
24096 // consumed by non-BLEND instructions, which may depend on all bits are
24098 for (SDNode::use_iterator I = Cond->use_begin(), E = Cond->use_end();
24100 if (I->getOpcode() != ISD::VSELECT)
24101 // TODO: Add other opcodes eventually lowered into BLEND.
24104 // Update all the users of the condition, before committing the change,
24105 // so that the VSELECT optimizations that expect the correct vector
24106 // boolean value will not be triggered.
24107 for (SDNode::use_iterator I = Cond->use_begin(), E = Cond->use_end();
24109 DAG.ReplaceAllUsesOfValueWith(
24111 DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(*I), I->getValueType(0),
24112 Cond, I->getOperand(1), I->getOperand(2)));
24113 DCI.CommitTargetLoweringOpt(TLO);
24116 // At this point, only Cond is changed. Change the condition
24117 // just for N to keep the opportunity to optimize all other
24118 // users their own way.
24119 DAG.ReplaceAllUsesOfValueWith(
24121 DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(N), N->getValueType(0),
24122 TLO.New, N->getOperand(1), N->getOperand(2)));
24127 // We should generate an X86ISD::BLENDI from a vselect if its argument
24128 // is a sign_extend_inreg of an any_extend of a BUILD_VECTOR of
24129 // constants. This specific pattern gets generated when we split a
24130 // selector for a 512 bit vector in a machine without AVX512 (but with
24131 // 256-bit vectors), during legalization:
24133 // (vselect (sign_extend (any_extend (BUILD_VECTOR)) i1) LHS RHS)
24135 // Iff we find this pattern and the build_vectors are built from
24136 // constants, we translate the vselect into a shuffle_vector that we
24137 // know will be matched by LowerVECTOR_SHUFFLEtoBlend.
24138 if ((N->getOpcode() == ISD::VSELECT ||
24139 N->getOpcode() == X86ISD::SHRUNKBLEND) &&
24140 !DCI.isBeforeLegalize()) {
24141 SDValue Shuffle = transformVSELECTtoBlendVECTOR_SHUFFLE(N, DAG, Subtarget);
24142 if (Shuffle.getNode())
24149 // Check whether a boolean test is testing a boolean value generated by
24150 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
24153 // Simplify the following patterns:
24154 // (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
24155 // (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
24156 // to (Op EFLAGS Cond)
24158 // (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
24159 // (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
24160 // to (Op EFLAGS !Cond)
24162 // where Op could be BRCOND or CMOV.
24164 static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
24165 // Quit if not CMP and SUB with its value result used.
24166 if (Cmp.getOpcode() != X86ISD::CMP &&
24167 (Cmp.getOpcode() != X86ISD::SUB || Cmp.getNode()->hasAnyUseOfValue(0)))
24170 // Quit if not used as a boolean value.
24171 if (CC != X86::COND_E && CC != X86::COND_NE)
24174 // Check CMP operands. One of them should be 0 or 1 and the other should be
24175 // an SetCC or extended from it.
24176 SDValue Op1 = Cmp.getOperand(0);
24177 SDValue Op2 = Cmp.getOperand(1);
24180 const ConstantSDNode* C = nullptr;
24181 bool needOppositeCond = (CC == X86::COND_E);
24182 bool checkAgainstTrue = false; // Is it a comparison against 1?
24184 if ((C = dyn_cast<ConstantSDNode>(Op1)))
24186 else if ((C = dyn_cast<ConstantSDNode>(Op2)))
24188 else // Quit if all operands are not constants.
24191 if (C->getZExtValue() == 1) {
24192 needOppositeCond = !needOppositeCond;
24193 checkAgainstTrue = true;
24194 } else if (C->getZExtValue() != 0)
24195 // Quit if the constant is neither 0 or 1.
24198 bool truncatedToBoolWithAnd = false;
24199 // Skip (zext $x), (trunc $x), or (and $x, 1) node.
24200 while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
24201 SetCC.getOpcode() == ISD::TRUNCATE ||
24202 SetCC.getOpcode() == ISD::AND) {
24203 if (SetCC.getOpcode() == ISD::AND) {
24205 ConstantSDNode *CS;
24206 if ((CS = dyn_cast<ConstantSDNode>(SetCC.getOperand(0))) &&
24207 CS->getZExtValue() == 1)
24209 if ((CS = dyn_cast<ConstantSDNode>(SetCC.getOperand(1))) &&
24210 CS->getZExtValue() == 1)
24214 SetCC = SetCC.getOperand(OpIdx);
24215 truncatedToBoolWithAnd = true;
24217 SetCC = SetCC.getOperand(0);
24220 switch (SetCC.getOpcode()) {
24221 case X86ISD::SETCC_CARRY:
24222 // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
24223 // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
24224 // i.e. it's a comparison against true but the result of SETCC_CARRY is not
24225 // truncated to i1 using 'and'.
24226 if (checkAgainstTrue && !truncatedToBoolWithAnd)
24228 assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&
24229 "Invalid use of SETCC_CARRY!");
24231 case X86ISD::SETCC:
24232 // Set the condition code or opposite one if necessary.
24233 CC = X86::CondCode(SetCC.getConstantOperandVal(0));
24234 if (needOppositeCond)
24235 CC = X86::GetOppositeBranchCondition(CC);
24236 return SetCC.getOperand(1);
24237 case X86ISD::CMOV: {
24238 // Check whether false/true value has canonical one, i.e. 0 or 1.
24239 ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
24240 ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
24241 // Quit if true value is not a constant.
24244 // Quit if false value is not a constant.
24246 SDValue Op = SetCC.getOperand(0);
24247 // Skip 'zext' or 'trunc' node.
24248 if (Op.getOpcode() == ISD::ZERO_EXTEND ||
24249 Op.getOpcode() == ISD::TRUNCATE)
24250 Op = Op.getOperand(0);
24251 // A special case for rdrand/rdseed, where 0 is set if false cond is
24253 if ((Op.getOpcode() != X86ISD::RDRAND &&
24254 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
24257 // Quit if false value is not the constant 0 or 1.
24258 bool FValIsFalse = true;
24259 if (FVal && FVal->getZExtValue() != 0) {
24260 if (FVal->getZExtValue() != 1)
24262 // If FVal is 1, opposite cond is needed.
24263 needOppositeCond = !needOppositeCond;
24264 FValIsFalse = false;
24266 // Quit if TVal is not the constant opposite of FVal.
24267 if (FValIsFalse && TVal->getZExtValue() != 1)
24269 if (!FValIsFalse && TVal->getZExtValue() != 0)
24271 CC = X86::CondCode(SetCC.getConstantOperandVal(2));
24272 if (needOppositeCond)
24273 CC = X86::GetOppositeBranchCondition(CC);
24274 return SetCC.getOperand(3);
24281 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
24282 static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
24283 TargetLowering::DAGCombinerInfo &DCI,
24284 const X86Subtarget *Subtarget) {
24287 // If the flag operand isn't dead, don't touch this CMOV.
24288 if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
24291 SDValue FalseOp = N->getOperand(0);
24292 SDValue TrueOp = N->getOperand(1);
24293 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
24294 SDValue Cond = N->getOperand(3);
24296 if (CC == X86::COND_E || CC == X86::COND_NE) {
24297 switch (Cond.getOpcode()) {
24301 // If operand of BSR / BSF are proven never zero, then ZF cannot be set.
24302 if (DAG.isKnownNeverZero(Cond.getOperand(0)))
24303 return (CC == X86::COND_E) ? FalseOp : TrueOp;
24309 Flags = checkBoolTestSetCCCombine(Cond, CC);
24310 if (Flags.getNode() &&
24311 // Extra check as FCMOV only supports a subset of X86 cond.
24312 (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) {
24313 SDValue Ops[] = { FalseOp, TrueOp,
24314 DAG.getConstant(CC, MVT::i8), Flags };
24315 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
24318 // If this is a select between two integer constants, try to do some
24319 // optimizations. Note that the operands are ordered the opposite of SELECT
24321 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
24322 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
24323 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
24324 // larger than FalseC (the false value).
24325 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
24326 CC = X86::GetOppositeBranchCondition(CC);
24327 std::swap(TrueC, FalseC);
24328 std::swap(TrueOp, FalseOp);
24331 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0.
24332 // This is efficient for any integer data type (including i8/i16) and
24334 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
24335 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
24336 DAG.getConstant(CC, MVT::i8), Cond);
24338 // Zero extend the condition if needed.
24339 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
24341 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
24342 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
24343 DAG.getConstant(ShAmt, MVT::i8));
24344 if (N->getNumValues() == 2) // Dead flag value?
24345 return DCI.CombineTo(N, Cond, SDValue());
24349 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
24350 // for any integer data type, including i8/i16.
24351 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
24352 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
24353 DAG.getConstant(CC, MVT::i8), Cond);
24355 // Zero extend the condition if needed.
24356 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
24357 FalseC->getValueType(0), Cond);
24358 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
24359 SDValue(FalseC, 0));
24361 if (N->getNumValues() == 2) // Dead flag value?
24362 return DCI.CombineTo(N, Cond, SDValue());
24366 // Optimize cases that will turn into an LEA instruction. This requires
24367 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
24368 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
24369 uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
24370 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
24372 bool isFastMultiplier = false;
24374 switch ((unsigned char)Diff) {
24376 case 1: // result = add base, cond
24377 case 2: // result = lea base( , cond*2)
24378 case 3: // result = lea base(cond, cond*2)
24379 case 4: // result = lea base( , cond*4)
24380 case 5: // result = lea base(cond, cond*4)
24381 case 8: // result = lea base( , cond*8)
24382 case 9: // result = lea base(cond, cond*8)
24383 isFastMultiplier = true;
24388 if (isFastMultiplier) {
24389 APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
24390 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
24391 DAG.getConstant(CC, MVT::i8), Cond);
24392 // Zero extend the condition if needed.
24393 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
24395 // Scale the condition by the difference.
24397 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
24398 DAG.getConstant(Diff, Cond.getValueType()));
24400 // Add the base if non-zero.
24401 if (FalseC->getAPIntValue() != 0)
24402 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
24403 SDValue(FalseC, 0));
24404 if (N->getNumValues() == 2) // Dead flag value?
24405 return DCI.CombineTo(N, Cond, SDValue());
24412 // Handle these cases:
24413 // (select (x != c), e, c) -> select (x != c), e, x),
24414 // (select (x == c), c, e) -> select (x == c), x, e)
24415 // where the c is an integer constant, and the "select" is the combination
24416 // of CMOV and CMP.
24418 // The rationale for this change is that the conditional-move from a constant
24419 // needs two instructions, however, conditional-move from a register needs
24420 // only one instruction.
24422 // CAVEAT: By replacing a constant with a symbolic value, it may obscure
24423 // some instruction-combining opportunities. This opt needs to be
24424 // postponed as late as possible.
24426 if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
24427 // the DCI.xxxx conditions are provided to postpone the optimization as
24428 // late as possible.
24430 ConstantSDNode *CmpAgainst = nullptr;
24431 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
24432 (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
24433 !isa<ConstantSDNode>(Cond.getOperand(0))) {
24435 if (CC == X86::COND_NE &&
24436 CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
24437 CC = X86::GetOppositeBranchCondition(CC);
24438 std::swap(TrueOp, FalseOp);
24441 if (CC == X86::COND_E &&
24442 CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
24443 SDValue Ops[] = { FalseOp, Cond.getOperand(0),
24444 DAG.getConstant(CC, MVT::i8), Cond };
24445 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops);
24453 static SDValue PerformINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
24454 const X86Subtarget *Subtarget) {
24455 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
24457 default: return SDValue();
24458 // SSE/AVX/AVX2 blend intrinsics.
24459 case Intrinsic::x86_avx2_pblendvb:
24460 case Intrinsic::x86_avx2_pblendw:
24461 case Intrinsic::x86_avx2_pblendd_128:
24462 case Intrinsic::x86_avx2_pblendd_256:
24463 // Don't try to simplify this intrinsic if we don't have AVX2.
24464 if (!Subtarget->hasAVX2())
24467 case Intrinsic::x86_avx_blend_pd_256:
24468 case Intrinsic::x86_avx_blend_ps_256:
24469 case Intrinsic::x86_avx_blendv_pd_256:
24470 case Intrinsic::x86_avx_blendv_ps_256:
24471 // Don't try to simplify this intrinsic if we don't have AVX.
24472 if (!Subtarget->hasAVX())
24475 case Intrinsic::x86_sse41_pblendw:
24476 case Intrinsic::x86_sse41_blendpd:
24477 case Intrinsic::x86_sse41_blendps:
24478 case Intrinsic::x86_sse41_blendvps:
24479 case Intrinsic::x86_sse41_blendvpd:
24480 case Intrinsic::x86_sse41_pblendvb: {
24481 SDValue Op0 = N->getOperand(1);
24482 SDValue Op1 = N->getOperand(2);
24483 SDValue Mask = N->getOperand(3);
24485 // Don't try to simplify this intrinsic if we don't have SSE4.1.
24486 if (!Subtarget->hasSSE41())
24489 // fold (blend A, A, Mask) -> A
24492 // fold (blend A, B, allZeros) -> A
24493 if (ISD::isBuildVectorAllZeros(Mask.getNode()))
24495 // fold (blend A, B, allOnes) -> B
24496 if (ISD::isBuildVectorAllOnes(Mask.getNode()))
24499 // Simplify the case where the mask is a constant i32 value.
24500 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Mask)) {
24501 if (C->isNullValue())
24503 if (C->isAllOnesValue())
24510 // Packed SSE2/AVX2 arithmetic shift immediate intrinsics.
24511 case Intrinsic::x86_sse2_psrai_w:
24512 case Intrinsic::x86_sse2_psrai_d:
24513 case Intrinsic::x86_avx2_psrai_w:
24514 case Intrinsic::x86_avx2_psrai_d:
24515 case Intrinsic::x86_sse2_psra_w:
24516 case Intrinsic::x86_sse2_psra_d:
24517 case Intrinsic::x86_avx2_psra_w:
24518 case Intrinsic::x86_avx2_psra_d: {
24519 SDValue Op0 = N->getOperand(1);
24520 SDValue Op1 = N->getOperand(2);
24521 EVT VT = Op0.getValueType();
24522 assert(VT.isVector() && "Expected a vector type!");
24524 if (isa<BuildVectorSDNode>(Op1))
24525 Op1 = Op1.getOperand(0);
24527 if (!isa<ConstantSDNode>(Op1))
24530 EVT SVT = VT.getVectorElementType();
24531 unsigned SVTBits = SVT.getSizeInBits();
24533 ConstantSDNode *CND = cast<ConstantSDNode>(Op1);
24534 const APInt &C = APInt(SVTBits, CND->getAPIntValue().getZExtValue());
24535 uint64_t ShAmt = C.getZExtValue();
24537 // Don't try to convert this shift into a ISD::SRA if the shift
24538 // count is bigger than or equal to the element size.
24539 if (ShAmt >= SVTBits)
24542 // Trivial case: if the shift count is zero, then fold this
24543 // into the first operand.
24547 // Replace this packed shift intrinsic with a target independent
24549 SDValue Splat = DAG.getConstant(C, VT);
24550 return DAG.getNode(ISD::SRA, SDLoc(N), VT, Op0, Splat);
24555 /// PerformMulCombine - Optimize a single multiply with constant into two
24556 /// in order to implement it with two cheaper instructions, e.g.
24557 /// LEA + SHL, LEA + LEA.
24558 static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG,
24559 TargetLowering::DAGCombinerInfo &DCI) {
24560 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
24563 EVT VT = N->getValueType(0);
24564 if (VT != MVT::i64 && VT != MVT::i32)
24567 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
24570 uint64_t MulAmt = C->getZExtValue();
24571 if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9)
24574 uint64_t MulAmt1 = 0;
24575 uint64_t MulAmt2 = 0;
24576 if ((MulAmt % 9) == 0) {
24578 MulAmt2 = MulAmt / 9;
24579 } else if ((MulAmt % 5) == 0) {
24581 MulAmt2 = MulAmt / 5;
24582 } else if ((MulAmt % 3) == 0) {
24584 MulAmt2 = MulAmt / 3;
24587 (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){
24590 if (isPowerOf2_64(MulAmt2) &&
24591 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD))
24592 // If second multiplifer is pow2, issue it first. We want the multiply by
24593 // 3, 5, or 9 to be folded into the addressing mode unless the lone use
24595 std::swap(MulAmt1, MulAmt2);
24598 if (isPowerOf2_64(MulAmt1))
24599 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
24600 DAG.getConstant(Log2_64(MulAmt1), MVT::i8));
24602 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
24603 DAG.getConstant(MulAmt1, VT));
24605 if (isPowerOf2_64(MulAmt2))
24606 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
24607 DAG.getConstant(Log2_64(MulAmt2), MVT::i8));
24609 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
24610 DAG.getConstant(MulAmt2, VT));
24612 // Do not add new nodes to DAG combiner worklist.
24613 DCI.CombineTo(N, NewMul, false);
24618 static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) {
24619 SDValue N0 = N->getOperand(0);
24620 SDValue N1 = N->getOperand(1);
24621 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
24622 EVT VT = N0.getValueType();
24624 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
24625 // since the result of setcc_c is all zero's or all ones.
24626 if (VT.isInteger() && !VT.isVector() &&
24627 N1C && N0.getOpcode() == ISD::AND &&
24628 N0.getOperand(1).getOpcode() == ISD::Constant) {
24629 SDValue N00 = N0.getOperand(0);
24630 if (N00.getOpcode() == X86ISD::SETCC_CARRY ||
24631 ((N00.getOpcode() == ISD::ANY_EXTEND ||
24632 N00.getOpcode() == ISD::ZERO_EXTEND) &&
24633 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) {
24634 APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
24635 APInt ShAmt = N1C->getAPIntValue();
24636 Mask = Mask.shl(ShAmt);
24638 return DAG.getNode(ISD::AND, SDLoc(N), VT,
24639 N00, DAG.getConstant(Mask, VT));
24643 // Hardware support for vector shifts is sparse which makes us scalarize the
24644 // vector operations in many cases. Also, on sandybridge ADD is faster than
24646 // (shl V, 1) -> add V,V
24647 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
24648 if (auto *N1SplatC = N1BV->getConstantSplatNode()) {
24649 assert(N0.getValueType().isVector() && "Invalid vector shift type");
24650 // We shift all of the values by one. In many cases we do not have
24651 // hardware support for this operation. This is better expressed as an ADD
24653 if (N1SplatC->getZExtValue() == 1)
24654 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);
24660 /// \brief Returns a vector of 0s if the node in input is a vector logical
24661 /// shift by a constant amount which is known to be bigger than or equal
24662 /// to the vector element size in bits.
24663 static SDValue performShiftToAllZeros(SDNode *N, SelectionDAG &DAG,
24664 const X86Subtarget *Subtarget) {
24665 EVT VT = N->getValueType(0);
24667 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 &&
24668 (!Subtarget->hasInt256() ||
24669 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16)))
24672 SDValue Amt = N->getOperand(1);
24674 if (auto *AmtBV = dyn_cast<BuildVectorSDNode>(Amt))
24675 if (auto *AmtSplat = AmtBV->getConstantSplatNode()) {
24676 APInt ShiftAmt = AmtSplat->getAPIntValue();
24677 unsigned MaxAmount = VT.getVectorElementType().getSizeInBits();
24679 // SSE2/AVX2 logical shifts always return a vector of 0s
24680 // if the shift amount is bigger than or equal to
24681 // the element size. The constant shift amount will be
24682 // encoded as a 8-bit immediate.
24683 if (ShiftAmt.trunc(8).uge(MaxAmount))
24684 return getZeroVector(VT, Subtarget, DAG, DL);
24690 /// PerformShiftCombine - Combine shifts.
24691 static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG,
24692 TargetLowering::DAGCombinerInfo &DCI,
24693 const X86Subtarget *Subtarget) {
24694 if (N->getOpcode() == ISD::SHL) {
24695 SDValue V = PerformSHLCombine(N, DAG);
24696 if (V.getNode()) return V;
24699 if (N->getOpcode() != ISD::SRA) {
24700 // Try to fold this logical shift into a zero vector.
24701 SDValue V = performShiftToAllZeros(N, DAG, Subtarget);
24702 if (V.getNode()) return V;
24708 // CMPEQCombine - Recognize the distinctive (AND (setcc ...) (setcc ..))
24709 // where both setccs reference the same FP CMP, and rewrite for CMPEQSS
24710 // and friends. Likewise for OR -> CMPNEQSS.
24711 static SDValue CMPEQCombine(SDNode *N, SelectionDAG &DAG,
24712 TargetLowering::DAGCombinerInfo &DCI,
24713 const X86Subtarget *Subtarget) {
24716 // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
24717 // we're requiring SSE2 for both.
24718 if (Subtarget->hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
24719 SDValue N0 = N->getOperand(0);
24720 SDValue N1 = N->getOperand(1);
24721 SDValue CMP0 = N0->getOperand(1);
24722 SDValue CMP1 = N1->getOperand(1);
24725 // The SETCCs should both refer to the same CMP.
24726 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
24729 SDValue CMP00 = CMP0->getOperand(0);
24730 SDValue CMP01 = CMP0->getOperand(1);
24731 EVT VT = CMP00.getValueType();
24733 if (VT == MVT::f32 || VT == MVT::f64) {
24734 bool ExpectingFlags = false;
24735 // Check for any users that want flags:
24736 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
24737 !ExpectingFlags && UI != UE; ++UI)
24738 switch (UI->getOpcode()) {
24743 ExpectingFlags = true;
24745 case ISD::CopyToReg:
24746 case ISD::SIGN_EXTEND:
24747 case ISD::ZERO_EXTEND:
24748 case ISD::ANY_EXTEND:
24752 if (!ExpectingFlags) {
24753 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
24754 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
24756 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
24757 X86::CondCode tmp = cc0;
24762 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
24763 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
24764 // FIXME: need symbolic constants for these magic numbers.
24765 // See X86ATTInstPrinter.cpp:printSSECC().
24766 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
24767 if (Subtarget->hasAVX512()) {
24768 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00,
24769 CMP01, DAG.getConstant(x86cc, MVT::i8));
24770 if (N->getValueType(0) != MVT::i1)
24771 return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0),
24775 SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
24776 CMP00.getValueType(), CMP00, CMP01,
24777 DAG.getConstant(x86cc, MVT::i8));
24779 bool is64BitFP = (CMP00.getValueType() == MVT::f64);
24780 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
24782 if (is64BitFP && !Subtarget->is64Bit()) {
24783 // On a 32-bit target, we cannot bitcast the 64-bit float to a
24784 // 64-bit integer, since that's not a legal type. Since
24785 // OnesOrZeroesF is all ones of all zeroes, we don't need all the
24786 // bits, but can do this little dance to extract the lowest 32 bits
24787 // and work with those going forward.
24788 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
24790 SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32,
24792 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
24793 Vector32, DAG.getIntPtrConstant(0));
24797 SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, IntVT, OnesOrZeroesF);
24798 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
24799 DAG.getConstant(1, IntVT));
24800 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed);
24801 return OneBitOfTruth;
24809 /// CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector
24810 /// so it can be folded inside ANDNP.
24811 static bool CanFoldXORWithAllOnes(const SDNode *N) {
24812 EVT VT = N->getValueType(0);
24814 // Match direct AllOnes for 128 and 256-bit vectors
24815 if (ISD::isBuildVectorAllOnes(N))
24818 // Look through a bit convert.
24819 if (N->getOpcode() == ISD::BITCAST)
24820 N = N->getOperand(0).getNode();
24822 // Sometimes the operand may come from a insert_subvector building a 256-bit
24824 if (VT.is256BitVector() &&
24825 N->getOpcode() == ISD::INSERT_SUBVECTOR) {
24826 SDValue V1 = N->getOperand(0);
24827 SDValue V2 = N->getOperand(1);
24829 if (V1.getOpcode() == ISD::INSERT_SUBVECTOR &&
24830 V1.getOperand(0).getOpcode() == ISD::UNDEF &&
24831 ISD::isBuildVectorAllOnes(V1.getOperand(1).getNode()) &&
24832 ISD::isBuildVectorAllOnes(V2.getNode()))
24839 // On AVX/AVX2 the type v8i1 is legalized to v8i16, which is an XMM sized
24840 // register. In most cases we actually compare or select YMM-sized registers
24841 // and mixing the two types creates horrible code. This method optimizes
24842 // some of the transition sequences.
24843 static SDValue WidenMaskArithmetic(SDNode *N, SelectionDAG &DAG,
24844 TargetLowering::DAGCombinerInfo &DCI,
24845 const X86Subtarget *Subtarget) {
24846 EVT VT = N->getValueType(0);
24847 if (!VT.is256BitVector())
24850 assert((N->getOpcode() == ISD::ANY_EXTEND ||
24851 N->getOpcode() == ISD::ZERO_EXTEND ||
24852 N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node");
24854 SDValue Narrow = N->getOperand(0);
24855 EVT NarrowVT = Narrow->getValueType(0);
24856 if (!NarrowVT.is128BitVector())
24859 if (Narrow->getOpcode() != ISD::XOR &&
24860 Narrow->getOpcode() != ISD::AND &&
24861 Narrow->getOpcode() != ISD::OR)
24864 SDValue N0 = Narrow->getOperand(0);
24865 SDValue N1 = Narrow->getOperand(1);
24868 // The Left side has to be a trunc.
24869 if (N0.getOpcode() != ISD::TRUNCATE)
24872 // The type of the truncated inputs.
24873 EVT WideVT = N0->getOperand(0)->getValueType(0);
24877 // The right side has to be a 'trunc' or a constant vector.
24878 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE;
24879 ConstantSDNode *RHSConstSplat = nullptr;
24880 if (auto *RHSBV = dyn_cast<BuildVectorSDNode>(N1))
24881 RHSConstSplat = RHSBV->getConstantSplatNode();
24882 if (!RHSTrunc && !RHSConstSplat)
24885 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24887 if (!TLI.isOperationLegalOrPromote(Narrow->getOpcode(), WideVT))
24890 // Set N0 and N1 to hold the inputs to the new wide operation.
24891 N0 = N0->getOperand(0);
24892 if (RHSConstSplat) {
24893 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT.getScalarType(),
24894 SDValue(RHSConstSplat, 0));
24895 SmallVector<SDValue, 8> C(WideVT.getVectorNumElements(), N1);
24896 N1 = DAG.getNode(ISD::BUILD_VECTOR, DL, WideVT, C);
24897 } else if (RHSTrunc) {
24898 N1 = N1->getOperand(0);
24901 // Generate the wide operation.
24902 SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, WideVT, N0, N1);
24903 unsigned Opcode = N->getOpcode();
24905 case ISD::ANY_EXTEND:
24907 case ISD::ZERO_EXTEND: {
24908 unsigned InBits = NarrowVT.getScalarType().getSizeInBits();
24909 APInt Mask = APInt::getAllOnesValue(InBits);
24910 Mask = Mask.zext(VT.getScalarType().getSizeInBits());
24911 return DAG.getNode(ISD::AND, DL, VT,
24912 Op, DAG.getConstant(Mask, VT));
24914 case ISD::SIGN_EXTEND:
24915 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
24916 Op, DAG.getValueType(NarrowVT));
24918 llvm_unreachable("Unexpected opcode");
24922 static SDValue VectorZextCombine(SDNode *N, SelectionDAG &DAG,
24923 TargetLowering::DAGCombinerInfo &DCI,
24924 const X86Subtarget *Subtarget) {
24925 SDValue N0 = N->getOperand(0);
24926 SDValue N1 = N->getOperand(1);
24929 // A vector zext_in_reg may be represented as a shuffle,
24930 // feeding into a bitcast (this represents anyext) feeding into
24931 // an and with a mask.
24932 // We'd like to try to combine that into a shuffle with zero
24933 // plus a bitcast, removing the and.
24934 if (N0.getOpcode() != ISD::BITCAST ||
24935 N0.getOperand(0).getOpcode() != ISD::VECTOR_SHUFFLE)
24938 // The other side of the AND should be a splat of 2^C, where C
24939 // is the number of bits in the source type.
24940 if (N1.getOpcode() == ISD::BITCAST)
24941 N1 = N1.getOperand(0);
24942 if (N1.getOpcode() != ISD::BUILD_VECTOR)
24944 BuildVectorSDNode *Vector = cast<BuildVectorSDNode>(N1);
24946 ShuffleVectorSDNode *Shuffle = cast<ShuffleVectorSDNode>(N0.getOperand(0));
24947 EVT SrcType = Shuffle->getValueType(0);
24949 // We expect a single-source shuffle
24950 if (Shuffle->getOperand(1)->getOpcode() != ISD::UNDEF)
24953 unsigned SrcSize = SrcType.getScalarSizeInBits();
24955 APInt SplatValue, SplatUndef;
24956 unsigned SplatBitSize;
24958 if (!Vector->isConstantSplat(SplatValue, SplatUndef,
24959 SplatBitSize, HasAnyUndefs))
24962 unsigned ResSize = N1.getValueType().getScalarSizeInBits();
24963 // Make sure the splat matches the mask we expect
24964 if (SplatBitSize > ResSize ||
24965 (SplatValue + 1).exactLogBase2() != (int)SrcSize)
24968 // Make sure the input and output size make sense
24969 if (SrcSize >= ResSize || ResSize % SrcSize)
24972 // We expect a shuffle of the form <0, u, u, u, 1, u, u, u...>
24973 // The number of u's between each two values depends on the ratio between
24974 // the source and dest type.
24975 unsigned ZextRatio = ResSize / SrcSize;
24976 bool IsZext = true;
24977 for (unsigned i = 0; i < SrcType.getVectorNumElements(); ++i) {
24978 if (i % ZextRatio) {
24979 if (Shuffle->getMaskElt(i) > 0) {
24985 if (Shuffle->getMaskElt(i) != (int)(i / ZextRatio)) {
24986 // Expected element number
24996 // Ok, perform the transformation - replace the shuffle with
24997 // a shuffle of the form <0, k, k, k, 1, k, k, k> with zero
24998 // (instead of undef) where the k elements come from the zero vector.
24999 SmallVector<int, 8> Mask;
25000 unsigned NumElems = SrcType.getVectorNumElements();
25001 for (unsigned i = 0; i < NumElems; ++i)
25003 Mask.push_back(NumElems);
25005 Mask.push_back(i / ZextRatio);
25007 SDValue NewShuffle = DAG.getVectorShuffle(Shuffle->getValueType(0), DL,
25008 Shuffle->getOperand(0), DAG.getConstant(0, SrcType), Mask);
25009 return DAG.getNode(ISD::BITCAST, DL, N0.getValueType(), NewShuffle);
25012 static SDValue PerformAndCombine(SDNode *N, SelectionDAG &DAG,
25013 TargetLowering::DAGCombinerInfo &DCI,
25014 const X86Subtarget *Subtarget) {
25015 if (DCI.isBeforeLegalizeOps())
25018 SDValue Zext = VectorZextCombine(N, DAG, DCI, Subtarget);
25019 if (Zext.getNode())
25022 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
25026 EVT VT = N->getValueType(0);
25027 SDValue N0 = N->getOperand(0);
25028 SDValue N1 = N->getOperand(1);
25031 // Create BEXTR instructions
25032 // BEXTR is ((X >> imm) & (2**size-1))
25033 if (VT == MVT::i32 || VT == MVT::i64) {
25034 // Check for BEXTR.
25035 if ((Subtarget->hasBMI() || Subtarget->hasTBM()) &&
25036 (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL)) {
25037 ConstantSDNode *MaskNode = dyn_cast<ConstantSDNode>(N1);
25038 ConstantSDNode *ShiftNode = dyn_cast<ConstantSDNode>(N0.getOperand(1));
25039 if (MaskNode && ShiftNode) {
25040 uint64_t Mask = MaskNode->getZExtValue();
25041 uint64_t Shift = ShiftNode->getZExtValue();
25042 if (isMask_64(Mask)) {
25043 uint64_t MaskSize = countPopulation(Mask);
25044 if (Shift + MaskSize <= VT.getSizeInBits())
25045 return DAG.getNode(X86ISD::BEXTR, DL, VT, N0.getOperand(0),
25046 DAG.getConstant(Shift | (MaskSize << 8), VT));
25054 // Want to form ANDNP nodes:
25055 // 1) In the hopes of then easily combining them with OR and AND nodes
25056 // to form PBLEND/PSIGN.
25057 // 2) To match ANDN packed intrinsics
25058 if (VT != MVT::v2i64 && VT != MVT::v4i64)
25061 // Check LHS for vnot
25062 if (N0.getOpcode() == ISD::XOR &&
25063 //ISD::isBuildVectorAllOnes(N0.getOperand(1).getNode()))
25064 CanFoldXORWithAllOnes(N0.getOperand(1).getNode()))
25065 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
25067 // Check RHS for vnot
25068 if (N1.getOpcode() == ISD::XOR &&
25069 //ISD::isBuildVectorAllOnes(N1.getOperand(1).getNode()))
25070 CanFoldXORWithAllOnes(N1.getOperand(1).getNode()))
25071 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
25076 static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
25077 TargetLowering::DAGCombinerInfo &DCI,
25078 const X86Subtarget *Subtarget) {
25079 if (DCI.isBeforeLegalizeOps())
25082 SDValue R = CMPEQCombine(N, DAG, DCI, Subtarget);
25086 SDValue N0 = N->getOperand(0);
25087 SDValue N1 = N->getOperand(1);
25088 EVT VT = N->getValueType(0);
25090 // look for psign/blend
25091 if (VT == MVT::v2i64 || VT == MVT::v4i64) {
25092 if (!Subtarget->hasSSSE3() ||
25093 (VT == MVT::v4i64 && !Subtarget->hasInt256()))
25096 // Canonicalize pandn to RHS
25097 if (N0.getOpcode() == X86ISD::ANDNP)
25099 // or (and (m, y), (pandn m, x))
25100 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) {
25101 SDValue Mask = N1.getOperand(0);
25102 SDValue X = N1.getOperand(1);
25104 if (N0.getOperand(0) == Mask)
25105 Y = N0.getOperand(1);
25106 if (N0.getOperand(1) == Mask)
25107 Y = N0.getOperand(0);
25109 // Check to see if the mask appeared in both the AND and ANDNP and
25113 // Validate that X, Y, and Mask are BIT_CONVERTS, and see through them.
25114 // Look through mask bitcast.
25115 if (Mask.getOpcode() == ISD::BITCAST)
25116 Mask = Mask.getOperand(0);
25117 if (X.getOpcode() == ISD::BITCAST)
25118 X = X.getOperand(0);
25119 if (Y.getOpcode() == ISD::BITCAST)
25120 Y = Y.getOperand(0);
25122 EVT MaskVT = Mask.getValueType();
25124 // Validate that the Mask operand is a vector sra node.
25125 // FIXME: what to do for bytes, since there is a psignb/pblendvb, but
25126 // there is no psrai.b
25127 unsigned EltBits = MaskVT.getVectorElementType().getSizeInBits();
25128 unsigned SraAmt = ~0;
25129 if (Mask.getOpcode() == ISD::SRA) {
25130 if (auto *AmtBV = dyn_cast<BuildVectorSDNode>(Mask.getOperand(1)))
25131 if (auto *AmtConst = AmtBV->getConstantSplatNode())
25132 SraAmt = AmtConst->getZExtValue();
25133 } else if (Mask.getOpcode() == X86ISD::VSRAI) {
25134 SDValue SraC = Mask.getOperand(1);
25135 SraAmt = cast<ConstantSDNode>(SraC)->getZExtValue();
25137 if ((SraAmt + 1) != EltBits)
25142 // Now we know we at least have a plendvb with the mask val. See if
25143 // we can form a psignb/w/d.
25144 // psign = x.type == y.type == mask.type && y = sub(0, x);
25145 if (Y.getOpcode() == ISD::SUB && Y.getOperand(1) == X &&
25146 ISD::isBuildVectorAllZeros(Y.getOperand(0).getNode()) &&
25147 X.getValueType() == MaskVT && Y.getValueType() == MaskVT) {
25148 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
25149 "Unsupported VT for PSIGN");
25150 Mask = DAG.getNode(X86ISD::PSIGN, DL, MaskVT, X, Mask.getOperand(0));
25151 return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
25153 // PBLENDVB only available on SSE 4.1
25154 if (!Subtarget->hasSSE41())
25157 EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8;
25159 X = DAG.getNode(ISD::BITCAST, DL, BlendVT, X);
25160 Y = DAG.getNode(ISD::BITCAST, DL, BlendVT, Y);
25161 Mask = DAG.getNode(ISD::BITCAST, DL, BlendVT, Mask);
25162 Mask = DAG.getNode(ISD::VSELECT, DL, BlendVT, Mask, Y, X);
25163 return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
25167 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
25170 // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c)
25171 MachineFunction &MF = DAG.getMachineFunction();
25173 MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
25175 // SHLD/SHRD instructions have lower register pressure, but on some
25176 // platforms they have higher latency than the equivalent
25177 // series of shifts/or that would otherwise be generated.
25178 // Don't fold (or (x << c) | (y >> (64 - c))) if SHLD/SHRD instructions
25179 // have higher latencies and we are not optimizing for size.
25180 if (!OptForSize && Subtarget->isSHLDSlow())
25183 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
25185 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
25187 if (!N0.hasOneUse() || !N1.hasOneUse())
25190 SDValue ShAmt0 = N0.getOperand(1);
25191 if (ShAmt0.getValueType() != MVT::i8)
25193 SDValue ShAmt1 = N1.getOperand(1);
25194 if (ShAmt1.getValueType() != MVT::i8)
25196 if (ShAmt0.getOpcode() == ISD::TRUNCATE)
25197 ShAmt0 = ShAmt0.getOperand(0);
25198 if (ShAmt1.getOpcode() == ISD::TRUNCATE)
25199 ShAmt1 = ShAmt1.getOperand(0);
25202 unsigned Opc = X86ISD::SHLD;
25203 SDValue Op0 = N0.getOperand(0);
25204 SDValue Op1 = N1.getOperand(0);
25205 if (ShAmt0.getOpcode() == ISD::SUB) {
25206 Opc = X86ISD::SHRD;
25207 std::swap(Op0, Op1);
25208 std::swap(ShAmt0, ShAmt1);
25211 unsigned Bits = VT.getSizeInBits();
25212 if (ShAmt1.getOpcode() == ISD::SUB) {
25213 SDValue Sum = ShAmt1.getOperand(0);
25214 if (ConstantSDNode *SumC = dyn_cast<ConstantSDNode>(Sum)) {
25215 SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
25216 if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE)
25217 ShAmt1Op1 = ShAmt1Op1.getOperand(0);
25218 if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0)
25219 return DAG.getNode(Opc, DL, VT,
25221 DAG.getNode(ISD::TRUNCATE, DL,
25224 } else if (ConstantSDNode *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) {
25225 ConstantSDNode *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0);
25227 ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue() == Bits)
25228 return DAG.getNode(Opc, DL, VT,
25229 N0.getOperand(0), N1.getOperand(0),
25230 DAG.getNode(ISD::TRUNCATE, DL,
25237 // Generate NEG and CMOV for integer abs.
25238 static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
25239 EVT VT = N->getValueType(0);
25241 // Since X86 does not have CMOV for 8-bit integer, we don't convert
25242 // 8-bit integer abs to NEG and CMOV.
25243 if (VT.isInteger() && VT.getSizeInBits() == 8)
25246 SDValue N0 = N->getOperand(0);
25247 SDValue N1 = N->getOperand(1);
25250 // Check pattern of XOR(ADD(X,Y), Y) where Y is SRA(X, size(X)-1)
25251 // and change it to SUB and CMOV.
25252 if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
25253 N0.getOpcode() == ISD::ADD &&
25254 N0.getOperand(1) == N1 &&
25255 N1.getOpcode() == ISD::SRA &&
25256 N1.getOperand(0) == N0.getOperand(0))
25257 if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1)))
25258 if (Y1C->getAPIntValue() == VT.getSizeInBits()-1) {
25259 // Generate SUB & CMOV.
25260 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
25261 DAG.getConstant(0, VT), N0.getOperand(0));
25263 SDValue Ops[] = { N0.getOperand(0), Neg,
25264 DAG.getConstant(X86::COND_GE, MVT::i8),
25265 SDValue(Neg.getNode(), 1) };
25266 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops);
25271 // PerformXorCombine - Attempts to turn XOR nodes into BLSMSK nodes
25272 static SDValue PerformXorCombine(SDNode *N, SelectionDAG &DAG,
25273 TargetLowering::DAGCombinerInfo &DCI,
25274 const X86Subtarget *Subtarget) {
25275 if (DCI.isBeforeLegalizeOps())
25278 if (Subtarget->hasCMov()) {
25279 SDValue RV = performIntegerAbsCombine(N, DAG);
25287 /// PerformLOADCombine - Do target-specific dag combines on LOAD nodes.
25288 static SDValue PerformLOADCombine(SDNode *N, SelectionDAG &DAG,
25289 TargetLowering::DAGCombinerInfo &DCI,
25290 const X86Subtarget *Subtarget) {
25291 LoadSDNode *Ld = cast<LoadSDNode>(N);
25292 EVT RegVT = Ld->getValueType(0);
25293 EVT MemVT = Ld->getMemoryVT();
25295 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25297 // For chips with slow 32-byte unaligned loads, break the 32-byte operation
25298 // into two 16-byte operations.
25299 ISD::LoadExtType Ext = Ld->getExtensionType();
25300 unsigned Alignment = Ld->getAlignment();
25301 bool IsAligned = Alignment == 0 || Alignment >= MemVT.getSizeInBits()/8;
25302 if (RegVT.is256BitVector() && Subtarget->isUnalignedMem32Slow() &&
25303 !DCI.isBeforeLegalizeOps() && !IsAligned && Ext == ISD::NON_EXTLOAD) {
25304 unsigned NumElems = RegVT.getVectorNumElements();
25308 SDValue Ptr = Ld->getBasePtr();
25309 SDValue Increment = DAG.getConstant(16, TLI.getPointerTy());
25311 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
25313 SDValue Load1 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr,
25314 Ld->getPointerInfo(), Ld->isVolatile(),
25315 Ld->isNonTemporal(), Ld->isInvariant(),
25317 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
25318 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr,
25319 Ld->getPointerInfo(), Ld->isVolatile(),
25320 Ld->isNonTemporal(), Ld->isInvariant(),
25321 std::min(16U, Alignment));
25322 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
25324 Load2.getValue(1));
25326 SDValue NewVec = DAG.getUNDEF(RegVT);
25327 NewVec = Insert128BitVector(NewVec, Load1, 0, DAG, dl);
25328 NewVec = Insert128BitVector(NewVec, Load2, NumElems/2, DAG, dl);
25329 return DCI.CombineTo(N, NewVec, TF, true);
25335 /// PerformMLOADCombine - Resolve extending loads
25336 static SDValue PerformMLOADCombine(SDNode *N, SelectionDAG &DAG,
25337 TargetLowering::DAGCombinerInfo &DCI,
25338 const X86Subtarget *Subtarget) {
25339 MaskedLoadSDNode *Mld = cast<MaskedLoadSDNode>(N);
25340 if (Mld->getExtensionType() != ISD::SEXTLOAD)
25343 EVT VT = Mld->getValueType(0);
25344 unsigned NumElems = VT.getVectorNumElements();
25345 EVT LdVT = Mld->getMemoryVT();
25348 assert(LdVT != VT && "Cannot extend to the same type");
25349 unsigned ToSz = VT.getVectorElementType().getSizeInBits();
25350 unsigned FromSz = LdVT.getVectorElementType().getSizeInBits();
25351 // From, To sizes and ElemCount must be pow of two
25352 assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
25353 "Unexpected size for extending masked load");
25355 unsigned SizeRatio = ToSz / FromSz;
25356 assert(SizeRatio * NumElems * FromSz == VT.getSizeInBits());
25358 // Create a type on which we perform the shuffle
25359 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
25360 LdVT.getScalarType(), NumElems*SizeRatio);
25361 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
25363 // Convert Src0 value
25364 SDValue WideSrc0 = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mld->getSrc0());
25365 if (Mld->getSrc0().getOpcode() != ISD::UNDEF) {
25366 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
25367 for (unsigned i = 0; i != NumElems; ++i)
25368 ShuffleVec[i] = i * SizeRatio;
25370 // Can't shuffle using an illegal type.
25371 assert (DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT)
25372 && "WideVecVT should be legal");
25373 WideSrc0 = DAG.getVectorShuffle(WideVecVT, dl, WideSrc0,
25374 DAG.getUNDEF(WideVecVT), &ShuffleVec[0]);
25376 // Prepare the new mask
25378 SDValue Mask = Mld->getMask();
25379 if (Mask.getValueType() == VT) {
25380 // Mask and original value have the same type
25381 NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask);
25382 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
25383 for (unsigned i = 0; i != NumElems; ++i)
25384 ShuffleVec[i] = i * SizeRatio;
25385 for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
25386 ShuffleVec[i] = NumElems*SizeRatio;
25387 NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
25388 DAG.getConstant(0, WideVecVT),
25392 assert(Mask.getValueType().getVectorElementType() == MVT::i1);
25393 unsigned WidenNumElts = NumElems*SizeRatio;
25394 unsigned MaskNumElts = VT.getVectorNumElements();
25395 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
25398 unsigned NumConcat = WidenNumElts / MaskNumElts;
25399 SmallVector<SDValue, 16> Ops(NumConcat);
25400 SDValue ZeroVal = DAG.getConstant(0, Mask.getValueType());
25402 for (unsigned i = 1; i != NumConcat; ++i)
25405 NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops);
25408 SDValue WideLd = DAG.getMaskedLoad(WideVecVT, dl, Mld->getChain(),
25409 Mld->getBasePtr(), NewMask, WideSrc0,
25410 Mld->getMemoryVT(), Mld->getMemOperand(),
25412 SDValue NewVec = DAG.getNode(X86ISD::VSEXT, dl, VT, WideLd);
25413 return DCI.CombineTo(N, NewVec, WideLd.getValue(1), true);
25416 /// PerformMSTORECombine - Resolve truncating stores
25417 static SDValue PerformMSTORECombine(SDNode *N, SelectionDAG &DAG,
25418 const X86Subtarget *Subtarget) {
25419 MaskedStoreSDNode *Mst = cast<MaskedStoreSDNode>(N);
25420 if (!Mst->isTruncatingStore())
25423 EVT VT = Mst->getValue().getValueType();
25424 unsigned NumElems = VT.getVectorNumElements();
25425 EVT StVT = Mst->getMemoryVT();
25428 assert(StVT != VT && "Cannot truncate to the same type");
25429 unsigned FromSz = VT.getVectorElementType().getSizeInBits();
25430 unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
25432 // From, To sizes and ElemCount must be pow of two
25433 assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
25434 "Unexpected size for truncating masked store");
25435 // We are going to use the original vector elt for storing.
25436 // Accumulated smaller vector elements must be a multiple of the store size.
25437 assert (((NumElems * FromSz) % ToSz) == 0 &&
25438 "Unexpected ratio for truncating masked store");
25440 unsigned SizeRatio = FromSz / ToSz;
25441 assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
25443 // Create a type on which we perform the shuffle
25444 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
25445 StVT.getScalarType(), NumElems*SizeRatio);
25447 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
25449 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mst->getValue());
25450 SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
25451 for (unsigned i = 0; i != NumElems; ++i)
25452 ShuffleVec[i] = i * SizeRatio;
25454 // Can't shuffle using an illegal type.
25455 assert (DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT)
25456 && "WideVecVT should be legal");
25458 SDValue TruncatedVal = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
25459 DAG.getUNDEF(WideVecVT),
25463 SDValue Mask = Mst->getMask();
25464 if (Mask.getValueType() == VT) {
25465 // Mask and original value have the same type
25466 NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask);
25467 for (unsigned i = 0; i != NumElems; ++i)
25468 ShuffleVec[i] = i * SizeRatio;
25469 for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
25470 ShuffleVec[i] = NumElems*SizeRatio;
25471 NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
25472 DAG.getConstant(0, WideVecVT),
25476 assert(Mask.getValueType().getVectorElementType() == MVT::i1);
25477 unsigned WidenNumElts = NumElems*SizeRatio;
25478 unsigned MaskNumElts = VT.getVectorNumElements();
25479 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
25482 unsigned NumConcat = WidenNumElts / MaskNumElts;
25483 SmallVector<SDValue, 16> Ops(NumConcat);
25484 SDValue ZeroVal = DAG.getConstant(0, Mask.getValueType());
25486 for (unsigned i = 1; i != NumConcat; ++i)
25489 NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops);
25492 return DAG.getMaskedStore(Mst->getChain(), dl, TruncatedVal, Mst->getBasePtr(),
25493 NewMask, StVT, Mst->getMemOperand(), false);
25495 /// PerformSTORECombine - Do target-specific dag combines on STORE nodes.
25496 static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
25497 const X86Subtarget *Subtarget) {
25498 StoreSDNode *St = cast<StoreSDNode>(N);
25499 EVT VT = St->getValue().getValueType();
25500 EVT StVT = St->getMemoryVT();
25502 SDValue StoredVal = St->getOperand(1);
25503 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25505 // If we are saving a concatenation of two XMM registers and 32-byte stores
25506 // are slow, such as on Sandy Bridge, perform two 16-byte stores.
25507 unsigned Alignment = St->getAlignment();
25508 bool IsAligned = Alignment == 0 || Alignment >= VT.getSizeInBits()/8;
25509 if (VT.is256BitVector() && Subtarget->isUnalignedMem32Slow() &&
25510 StVT == VT && !IsAligned) {
25511 unsigned NumElems = VT.getVectorNumElements();
25515 SDValue Value0 = Extract128BitVector(StoredVal, 0, DAG, dl);
25516 SDValue Value1 = Extract128BitVector(StoredVal, NumElems/2, DAG, dl);
25518 SDValue Stride = DAG.getConstant(16, TLI.getPointerTy());
25519 SDValue Ptr0 = St->getBasePtr();
25520 SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride);
25522 SDValue Ch0 = DAG.getStore(St->getChain(), dl, Value0, Ptr0,
25523 St->getPointerInfo(), St->isVolatile(),
25524 St->isNonTemporal(), Alignment);
25525 SDValue Ch1 = DAG.getStore(St->getChain(), dl, Value1, Ptr1,
25526 St->getPointerInfo(), St->isVolatile(),
25527 St->isNonTemporal(),
25528 std::min(16U, Alignment));
25529 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
25532 // Optimize trunc store (of multiple scalars) to shuffle and store.
25533 // First, pack all of the elements in one place. Next, store to memory
25534 // in fewer chunks.
25535 if (St->isTruncatingStore() && VT.isVector()) {
25536 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25537 unsigned NumElems = VT.getVectorNumElements();
25538 assert(StVT != VT && "Cannot truncate to the same type");
25539 unsigned FromSz = VT.getVectorElementType().getSizeInBits();
25540 unsigned ToSz = StVT.getVectorElementType().getSizeInBits();
25542 // From, To sizes and ElemCount must be pow of two
25543 if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue();
25544 // We are going to use the original vector elt for storing.
25545 // Accumulated smaller vector elements must be a multiple of the store size.
25546 if (0 != (NumElems * FromSz) % ToSz) return SDValue();
25548 unsigned SizeRatio = FromSz / ToSz;
25550 assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
25552 // Create a type on which we perform the shuffle
25553 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
25554 StVT.getScalarType(), NumElems*SizeRatio);
25556 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
25558 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue());
25559 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
25560 for (unsigned i = 0; i != NumElems; ++i)
25561 ShuffleVec[i] = i * SizeRatio;
25563 // Can't shuffle using an illegal type.
25564 if (!TLI.isTypeLegal(WideVecVT))
25567 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
25568 DAG.getUNDEF(WideVecVT),
25570 // At this point all of the data is stored at the bottom of the
25571 // register. We now need to save it to mem.
25573 // Find the largest store unit
25574 MVT StoreType = MVT::i8;
25575 for (MVT Tp : MVT::integer_valuetypes()) {
25576 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToSz)
25580 // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
25581 if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 &&
25582 (64 <= NumElems * ToSz))
25583 StoreType = MVT::f64;
25585 // Bitcast the original vector into a vector of store-size units
25586 EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
25587 StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());
25588 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
25589 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff);
25590 SmallVector<SDValue, 8> Chains;
25591 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits()/8,
25592 TLI.getPointerTy());
25593 SDValue Ptr = St->getBasePtr();
25595 // Perform one or more big stores into memory.
25596 for (unsigned i=0, e=(ToSz*NumElems)/StoreType.getSizeInBits(); i!=e; ++i) {
25597 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
25598 StoreType, ShuffWide,
25599 DAG.getIntPtrConstant(i));
25600 SDValue Ch = DAG.getStore(St->getChain(), dl, SubVec, Ptr,
25601 St->getPointerInfo(), St->isVolatile(),
25602 St->isNonTemporal(), St->getAlignment());
25603 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
25604 Chains.push_back(Ch);
25607 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
25610 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering
25611 // the FP state in cases where an emms may be missing.
25612 // A preferable solution to the general problem is to figure out the right
25613 // places to insert EMMS. This qualifies as a quick hack.
25615 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
25616 if (VT.getSizeInBits() != 64)
25619 const Function *F = DAG.getMachineFunction().getFunction();
25620 bool NoImplicitFloatOps = F->hasFnAttribute(Attribute::NoImplicitFloat);
25621 bool F64IsLegal = !DAG.getTarget().Options.UseSoftFloat && !NoImplicitFloatOps
25622 && Subtarget->hasSSE2();
25623 if ((VT.isVector() ||
25624 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
25625 isa<LoadSDNode>(St->getValue()) &&
25626 !cast<LoadSDNode>(St->getValue())->isVolatile() &&
25627 St->getChain().hasOneUse() && !St->isVolatile()) {
25628 SDNode* LdVal = St->getValue().getNode();
25629 LoadSDNode *Ld = nullptr;
25630 int TokenFactorIndex = -1;
25631 SmallVector<SDValue, 8> Ops;
25632 SDNode* ChainVal = St->getChain().getNode();
25633 // Must be a store of a load. We currently handle two cases: the load
25634 // is a direct child, and it's under an intervening TokenFactor. It is
25635 // possible to dig deeper under nested TokenFactors.
25636 if (ChainVal == LdVal)
25637 Ld = cast<LoadSDNode>(St->getChain());
25638 else if (St->getValue().hasOneUse() &&
25639 ChainVal->getOpcode() == ISD::TokenFactor) {
25640 for (unsigned i = 0, e = ChainVal->getNumOperands(); i != e; ++i) {
25641 if (ChainVal->getOperand(i).getNode() == LdVal) {
25642 TokenFactorIndex = i;
25643 Ld = cast<LoadSDNode>(St->getValue());
25645 Ops.push_back(ChainVal->getOperand(i));
25649 if (!Ld || !ISD::isNormalLoad(Ld))
25652 // If this is not the MMX case, i.e. we are just turning i64 load/store
25653 // into f64 load/store, avoid the transformation if there are multiple
25654 // uses of the loaded value.
25655 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))
25660 // If we are a 64-bit capable x86, lower to a single movq load/store pair.
25661 // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store
25663 if (Subtarget->is64Bit() || F64IsLegal) {
25664 EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64;
25665 SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(), Ld->getBasePtr(),
25666 Ld->getPointerInfo(), Ld->isVolatile(),
25667 Ld->isNonTemporal(), Ld->isInvariant(),
25668 Ld->getAlignment());
25669 SDValue NewChain = NewLd.getValue(1);
25670 if (TokenFactorIndex != -1) {
25671 Ops.push_back(NewChain);
25672 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops);
25674 return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(),
25675 St->getPointerInfo(),
25676 St->isVolatile(), St->isNonTemporal(),
25677 St->getAlignment());
25680 // Otherwise, lower to two pairs of 32-bit loads / stores.
25681 SDValue LoAddr = Ld->getBasePtr();
25682 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr,
25683 DAG.getConstant(4, MVT::i32));
25685 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
25686 Ld->getPointerInfo(),
25687 Ld->isVolatile(), Ld->isNonTemporal(),
25688 Ld->isInvariant(), Ld->getAlignment());
25689 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr,
25690 Ld->getPointerInfo().getWithOffset(4),
25691 Ld->isVolatile(), Ld->isNonTemporal(),
25693 MinAlign(Ld->getAlignment(), 4));
25695 SDValue NewChain = LoLd.getValue(1);
25696 if (TokenFactorIndex != -1) {
25697 Ops.push_back(LoLd);
25698 Ops.push_back(HiLd);
25699 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops);
25702 LoAddr = St->getBasePtr();
25703 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr,
25704 DAG.getConstant(4, MVT::i32));
25706 SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr,
25707 St->getPointerInfo(),
25708 St->isVolatile(), St->isNonTemporal(),
25709 St->getAlignment());
25710 SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr,
25711 St->getPointerInfo().getWithOffset(4),
25713 St->isNonTemporal(),
25714 MinAlign(St->getAlignment(), 4));
25715 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt);
25720 /// Return 'true' if this vector operation is "horizontal"
25721 /// and return the operands for the horizontal operation in LHS and RHS. A
25722 /// horizontal operation performs the binary operation on successive elements
25723 /// of its first operand, then on successive elements of its second operand,
25724 /// returning the resulting values in a vector. For example, if
25725 /// A = < float a0, float a1, float a2, float a3 >
25727 /// B = < float b0, float b1, float b2, float b3 >
25728 /// then the result of doing a horizontal operation on A and B is
25729 /// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
25730 /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
25731 /// A horizontal-op B, for some already available A and B, and if so then LHS is
25732 /// set to A, RHS to B, and the routine returns 'true'.
25733 /// Note that the binary operation should have the property that if one of the
25734 /// operands is UNDEF then the result is UNDEF.
25735 static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, bool IsCommutative) {
25736 // Look for the following pattern: if
25737 // A = < float a0, float a1, float a2, float a3 >
25738 // B = < float b0, float b1, float b2, float b3 >
25740 // LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
25741 // RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
25742 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
25743 // which is A horizontal-op B.
25745 // At least one of the operands should be a vector shuffle.
25746 if (LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
25747 RHS.getOpcode() != ISD::VECTOR_SHUFFLE)
25750 MVT VT = LHS.getSimpleValueType();
25752 assert((VT.is128BitVector() || VT.is256BitVector()) &&
25753 "Unsupported vector type for horizontal add/sub");
25755 // Handle 128 and 256-bit vector lengths. AVX defines horizontal add/sub to
25756 // operate independently on 128-bit lanes.
25757 unsigned NumElts = VT.getVectorNumElements();
25758 unsigned NumLanes = VT.getSizeInBits()/128;
25759 unsigned NumLaneElts = NumElts / NumLanes;
25760 assert((NumLaneElts % 2 == 0) &&
25761 "Vector type should have an even number of elements in each lane");
25762 unsigned HalfLaneElts = NumLaneElts/2;
25764 // View LHS in the form
25765 // LHS = VECTOR_SHUFFLE A, B, LMask
25766 // If LHS is not a shuffle then pretend it is the shuffle
25767 // LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
25768 // NOTE: in what follows a default initialized SDValue represents an UNDEF of
25771 SmallVector<int, 16> LMask(NumElts);
25772 if (LHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
25773 if (LHS.getOperand(0).getOpcode() != ISD::UNDEF)
25774 A = LHS.getOperand(0);
25775 if (LHS.getOperand(1).getOpcode() != ISD::UNDEF)
25776 B = LHS.getOperand(1);
25777 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(LHS.getNode())->getMask();
25778 std::copy(Mask.begin(), Mask.end(), LMask.begin());
25780 if (LHS.getOpcode() != ISD::UNDEF)
25782 for (unsigned i = 0; i != NumElts; ++i)
25786 // Likewise, view RHS in the form
25787 // RHS = VECTOR_SHUFFLE C, D, RMask
25789 SmallVector<int, 16> RMask(NumElts);
25790 if (RHS.getOpcode() == ISD::VECTOR_SHUFFLE) {
25791 if (RHS.getOperand(0).getOpcode() != ISD::UNDEF)
25792 C = RHS.getOperand(0);
25793 if (RHS.getOperand(1).getOpcode() != ISD::UNDEF)
25794 D = RHS.getOperand(1);
25795 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(RHS.getNode())->getMask();
25796 std::copy(Mask.begin(), Mask.end(), RMask.begin());
25798 if (RHS.getOpcode() != ISD::UNDEF)
25800 for (unsigned i = 0; i != NumElts; ++i)
25804 // Check that the shuffles are both shuffling the same vectors.
25805 if (!(A == C && B == D) && !(A == D && B == C))
25808 // If everything is UNDEF then bail out: it would be better to fold to UNDEF.
25809 if (!A.getNode() && !B.getNode())
25812 // If A and B occur in reverse order in RHS, then "swap" them (which means
25813 // rewriting the mask).
25815 CommuteVectorShuffleMask(RMask, NumElts);
25817 // At this point LHS and RHS are equivalent to
25818 // LHS = VECTOR_SHUFFLE A, B, LMask
25819 // RHS = VECTOR_SHUFFLE A, B, RMask
25820 // Check that the masks correspond to performing a horizontal operation.
25821 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {
25822 for (unsigned i = 0; i != NumLaneElts; ++i) {
25823 int LIdx = LMask[i+l], RIdx = RMask[i+l];
25825 // Ignore any UNDEF components.
25826 if (LIdx < 0 || RIdx < 0 ||
25827 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
25828 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
25831 // Check that successive elements are being operated on. If not, this is
25832 // not a horizontal operation.
25833 unsigned Src = (i/HalfLaneElts); // each lane is split between srcs
25834 int Index = 2*(i%HalfLaneElts) + NumElts*Src + l;
25835 if (!(LIdx == Index && RIdx == Index + 1) &&
25836 !(IsCommutative && LIdx == Index + 1 && RIdx == Index))
25841 LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
25842 RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
25846 /// Do target-specific dag combines on floating point adds.
25847 static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG,
25848 const X86Subtarget *Subtarget) {
25849 EVT VT = N->getValueType(0);
25850 SDValue LHS = N->getOperand(0);
25851 SDValue RHS = N->getOperand(1);
25853 // Try to synthesize horizontal adds from adds of shuffles.
25854 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
25855 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
25856 isHorizontalBinOp(LHS, RHS, true))
25857 return DAG.getNode(X86ISD::FHADD, SDLoc(N), VT, LHS, RHS);
25861 /// Do target-specific dag combines on floating point subs.
25862 static SDValue PerformFSUBCombine(SDNode *N, SelectionDAG &DAG,
25863 const X86Subtarget *Subtarget) {
25864 EVT VT = N->getValueType(0);
25865 SDValue LHS = N->getOperand(0);
25866 SDValue RHS = N->getOperand(1);
25868 // Try to synthesize horizontal subs from subs of shuffles.
25869 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
25870 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
25871 isHorizontalBinOp(LHS, RHS, false))
25872 return DAG.getNode(X86ISD::FHSUB, SDLoc(N), VT, LHS, RHS);
25876 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
25877 static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) {
25878 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
25880 // F[X]OR(0.0, x) -> x
25881 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
25882 if (C->getValueAPF().isPosZero())
25883 return N->getOperand(1);
25885 // F[X]OR(x, 0.0) -> x
25886 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
25887 if (C->getValueAPF().isPosZero())
25888 return N->getOperand(0);
25892 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
25893 static SDValue PerformFMinFMaxCombine(SDNode *N, SelectionDAG &DAG) {
25894 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
25896 // Only perform optimizations if UnsafeMath is used.
25897 if (!DAG.getTarget().Options.UnsafeFPMath)
25900 // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
25901 // into FMINC and FMAXC, which are Commutative operations.
25902 unsigned NewOp = 0;
25903 switch (N->getOpcode()) {
25904 default: llvm_unreachable("unknown opcode");
25905 case X86ISD::FMIN: NewOp = X86ISD::FMINC; break;
25906 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
25909 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
25910 N->getOperand(0), N->getOperand(1));
25913 /// Do target-specific dag combines on X86ISD::FAND nodes.
25914 static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) {
25915 // FAND(0.0, x) -> 0.0
25916 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
25917 if (C->getValueAPF().isPosZero())
25918 return N->getOperand(0);
25920 // FAND(x, 0.0) -> 0.0
25921 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
25922 if (C->getValueAPF().isPosZero())
25923 return N->getOperand(1);
25928 /// Do target-specific dag combines on X86ISD::FANDN nodes
25929 static SDValue PerformFANDNCombine(SDNode *N, SelectionDAG &DAG) {
25930 // FANDN(0.0, x) -> x
25931 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
25932 if (C->getValueAPF().isPosZero())
25933 return N->getOperand(1);
25935 // FANDN(x, 0.0) -> 0.0
25936 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
25937 if (C->getValueAPF().isPosZero())
25938 return N->getOperand(1);
25943 static SDValue PerformBTCombine(SDNode *N,
25945 TargetLowering::DAGCombinerInfo &DCI) {
25946 // BT ignores high bits in the bit index operand.
25947 SDValue Op1 = N->getOperand(1);
25948 if (Op1.hasOneUse()) {
25949 unsigned BitWidth = Op1.getValueSizeInBits();
25950 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
25951 APInt KnownZero, KnownOne;
25952 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
25953 !DCI.isBeforeLegalizeOps());
25954 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25955 if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) ||
25956 TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO))
25957 DCI.CommitTargetLoweringOpt(TLO);
25962 static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) {
25963 SDValue Op = N->getOperand(0);
25964 if (Op.getOpcode() == ISD::BITCAST)
25965 Op = Op.getOperand(0);
25966 EVT VT = N->getValueType(0), OpVT = Op.getValueType();
25967 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
25968 VT.getVectorElementType().getSizeInBits() ==
25969 OpVT.getVectorElementType().getSizeInBits()) {
25970 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
25975 static SDValue PerformSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG,
25976 const X86Subtarget *Subtarget) {
25977 EVT VT = N->getValueType(0);
25978 if (!VT.isVector())
25981 SDValue N0 = N->getOperand(0);
25982 SDValue N1 = N->getOperand(1);
25983 EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
25986 // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
25987 // both SSE and AVX2 since there is no sign-extended shift right
25988 // operation on a vector with 64-bit elements.
25989 //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
25990 // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
25991 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
25992 N0.getOpcode() == ISD::SIGN_EXTEND)) {
25993 SDValue N00 = N0.getOperand(0);
25995 // EXTLOAD has a better solution on AVX2,
25996 // it may be replaced with X86ISD::VSEXT node.
25997 if (N00.getOpcode() == ISD::LOAD && Subtarget->hasInt256())
25998 if (!ISD::isNormalLoad(N00.getNode()))
26001 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
26002 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32,
26004 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
26010 static SDValue PerformSExtCombine(SDNode *N, SelectionDAG &DAG,
26011 TargetLowering::DAGCombinerInfo &DCI,
26012 const X86Subtarget *Subtarget) {
26013 SDValue N0 = N->getOperand(0);
26014 EVT VT = N->getValueType(0);
26016 // (i8,i32 sext (sdivrem (i8 x, i8 y)) ->
26017 // (i8,i32 (sdivrem_sext_hreg (i8 x, i8 y)
26018 // This exposes the sext to the sdivrem lowering, so that it directly extends
26019 // from AH (which we otherwise need to do contortions to access).
26020 if (N0.getOpcode() == ISD::SDIVREM && N0.getResNo() == 1 &&
26021 N0.getValueType() == MVT::i8 && VT == MVT::i32) {
26023 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT);
26024 SDValue R = DAG.getNode(X86ISD::SDIVREM8_SEXT_HREG, dl, NodeTys,
26025 N0.getOperand(0), N0.getOperand(1));
26026 DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0));
26027 return R.getValue(1);
26030 if (!DCI.isBeforeLegalizeOps())
26033 if (!Subtarget->hasFp256())
26036 if (VT.isVector() && VT.getSizeInBits() == 256) {
26037 SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
26045 static SDValue PerformFMACombine(SDNode *N, SelectionDAG &DAG,
26046 const X86Subtarget* Subtarget) {
26048 EVT VT = N->getValueType(0);
26050 // Let legalize expand this if it isn't a legal type yet.
26051 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
26054 EVT ScalarVT = VT.getScalarType();
26055 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
26056 (!Subtarget->hasFMA() && !Subtarget->hasFMA4()))
26059 SDValue A = N->getOperand(0);
26060 SDValue B = N->getOperand(1);
26061 SDValue C = N->getOperand(2);
26063 bool NegA = (A.getOpcode() == ISD::FNEG);
26064 bool NegB = (B.getOpcode() == ISD::FNEG);
26065 bool NegC = (C.getOpcode() == ISD::FNEG);
26067 // Negative multiplication when NegA xor NegB
26068 bool NegMul = (NegA != NegB);
26070 A = A.getOperand(0);
26072 B = B.getOperand(0);
26074 C = C.getOperand(0);
26078 Opcode = (!NegC) ? X86ISD::FMADD : X86ISD::FMSUB;
26080 Opcode = (!NegC) ? X86ISD::FNMADD : X86ISD::FNMSUB;
26082 return DAG.getNode(Opcode, dl, VT, A, B, C);
26085 static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG,
26086 TargetLowering::DAGCombinerInfo &DCI,
26087 const X86Subtarget *Subtarget) {
26088 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
26089 // (and (i32 x86isd::setcc_carry), 1)
26090 // This eliminates the zext. This transformation is necessary because
26091 // ISD::SETCC is always legalized to i8.
26093 SDValue N0 = N->getOperand(0);
26094 EVT VT = N->getValueType(0);
26096 if (N0.getOpcode() == ISD::AND &&
26098 N0.getOperand(0).hasOneUse()) {
26099 SDValue N00 = N0.getOperand(0);
26100 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
26101 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
26102 if (!C || C->getZExtValue() != 1)
26104 return DAG.getNode(ISD::AND, dl, VT,
26105 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
26106 N00.getOperand(0), N00.getOperand(1)),
26107 DAG.getConstant(1, VT));
26111 if (N0.getOpcode() == ISD::TRUNCATE &&
26113 N0.getOperand(0).hasOneUse()) {
26114 SDValue N00 = N0.getOperand(0);
26115 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
26116 return DAG.getNode(ISD::AND, dl, VT,
26117 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
26118 N00.getOperand(0), N00.getOperand(1)),
26119 DAG.getConstant(1, VT));
26122 if (VT.is256BitVector()) {
26123 SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget);
26128 // (i8,i32 zext (udivrem (i8 x, i8 y)) ->
26129 // (i8,i32 (udivrem_zext_hreg (i8 x, i8 y)
26130 // This exposes the zext to the udivrem lowering, so that it directly extends
26131 // from AH (which we otherwise need to do contortions to access).
26132 if (N0.getOpcode() == ISD::UDIVREM &&
26133 N0.getResNo() == 1 && N0.getValueType() == MVT::i8 &&
26134 (VT == MVT::i32 || VT == MVT::i64)) {
26135 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT);
26136 SDValue R = DAG.getNode(X86ISD::UDIVREM8_ZEXT_HREG, dl, NodeTys,
26137 N0.getOperand(0), N0.getOperand(1));
26138 DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0));
26139 return R.getValue(1);
26145 // Optimize x == -y --> x+y == 0
26146 // x != -y --> x+y != 0
26147 static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG,
26148 const X86Subtarget* Subtarget) {
26149 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
26150 SDValue LHS = N->getOperand(0);
26151 SDValue RHS = N->getOperand(1);
26152 EVT VT = N->getValueType(0);
26155 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB)
26156 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS.getOperand(0)))
26157 if (C->getAPIntValue() == 0 && LHS.hasOneUse()) {
26158 SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N),
26159 LHS.getValueType(), RHS, LHS.getOperand(1));
26160 return DAG.getSetCC(SDLoc(N), N->getValueType(0),
26161 addV, DAG.getConstant(0, addV.getValueType()), CC);
26163 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && RHS.getOpcode() == ISD::SUB)
26164 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getOperand(0)))
26165 if (C->getAPIntValue() == 0 && RHS.hasOneUse()) {
26166 SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N),
26167 RHS.getValueType(), LHS, RHS.getOperand(1));
26168 return DAG.getSetCC(SDLoc(N), N->getValueType(0),
26169 addV, DAG.getConstant(0, addV.getValueType()), CC);
26172 if (VT.getScalarType() == MVT::i1) {
26173 bool IsSEXT0 = (LHS.getOpcode() == ISD::SIGN_EXTEND) &&
26174 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1);
26175 bool IsVZero0 = ISD::isBuildVectorAllZeros(LHS.getNode());
26176 if (!IsSEXT0 && !IsVZero0)
26178 bool IsSEXT1 = (RHS.getOpcode() == ISD::SIGN_EXTEND) &&
26179 (RHS.getOperand(0).getValueType().getScalarType() == MVT::i1);
26180 bool IsVZero1 = ISD::isBuildVectorAllZeros(RHS.getNode());
26182 if (!IsSEXT1 && !IsVZero1)
26185 if (IsSEXT0 && IsVZero1) {
26186 assert(VT == LHS.getOperand(0).getValueType() && "Uexpected operand type");
26187 if (CC == ISD::SETEQ)
26188 return DAG.getNOT(DL, LHS.getOperand(0), VT);
26189 return LHS.getOperand(0);
26191 if (IsSEXT1 && IsVZero0) {
26192 assert(VT == RHS.getOperand(0).getValueType() && "Uexpected operand type");
26193 if (CC == ISD::SETEQ)
26194 return DAG.getNOT(DL, RHS.getOperand(0), VT);
26195 return RHS.getOperand(0);
26202 static SDValue PerformINSERTPSCombine(SDNode *N, SelectionDAG &DAG,
26203 const X86Subtarget *Subtarget) {
26205 MVT VT = N->getOperand(1)->getSimpleValueType(0);
26206 assert((VT == MVT::v4f32 || VT == MVT::v4i32) &&
26207 "X86insertps is only defined for v4x32");
26209 SDValue Ld = N->getOperand(1);
26210 if (MayFoldLoad(Ld)) {
26211 // Extract the countS bits from the immediate so we can get the proper
26212 // address when narrowing the vector load to a specific element.
26213 // When the second source op is a memory address, interps doesn't use
26214 // countS and just gets an f32 from that address.
26215 unsigned DestIndex =
26216 cast<ConstantSDNode>(N->getOperand(2))->getZExtValue() >> 6;
26217 Ld = NarrowVectorLoadToElement(cast<LoadSDNode>(Ld), DestIndex, DAG);
26221 // Create this as a scalar to vector to match the instruction pattern.
26222 SDValue LoadScalarToVector = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Ld);
26223 // countS bits are ignored when loading from memory on insertps, which
26224 // means we don't need to explicitly set them to 0.
26225 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N->getOperand(0),
26226 LoadScalarToVector, N->getOperand(2));
26229 // Helper function of PerformSETCCCombine. It is to materialize "setb reg"
26230 // as "sbb reg,reg", since it can be extended without zext and produces
26231 // an all-ones bit which is more useful than 0/1 in some cases.
26232 static SDValue MaterializeSETB(SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG,
26235 return DAG.getNode(ISD::AND, DL, VT,
26236 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
26237 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS),
26238 DAG.getConstant(1, VT));
26239 assert (VT == MVT::i1 && "Unexpected type for SECCC node");
26240 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1,
26241 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
26242 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS));
26245 // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
26246 static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG,
26247 TargetLowering::DAGCombinerInfo &DCI,
26248 const X86Subtarget *Subtarget) {
26250 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
26251 SDValue EFLAGS = N->getOperand(1);
26253 if (CC == X86::COND_A) {
26254 // Try to convert COND_A into COND_B in an attempt to facilitate
26255 // materializing "setb reg".
26257 // Do not flip "e > c", where "c" is a constant, because Cmp instruction
26258 // cannot take an immediate as its first operand.
26260 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
26261 EFLAGS.getValueType().isInteger() &&
26262 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
26263 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
26264 EFLAGS.getNode()->getVTList(),
26265 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
26266 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
26267 return MaterializeSETB(DL, NewEFLAGS, DAG, N->getSimpleValueType(0));
26271 // Materialize "setb reg" as "sbb reg,reg", since it can be extended without
26272 // a zext and produces an all-ones bit which is more useful than 0/1 in some
26274 if (CC == X86::COND_B)
26275 return MaterializeSETB(DL, EFLAGS, DAG, N->getSimpleValueType(0));
26279 Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
26280 if (Flags.getNode()) {
26281 SDValue Cond = DAG.getConstant(CC, MVT::i8);
26282 return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
26288 // Optimize branch condition evaluation.
26290 static SDValue PerformBrCondCombine(SDNode *N, SelectionDAG &DAG,
26291 TargetLowering::DAGCombinerInfo &DCI,
26292 const X86Subtarget *Subtarget) {
26294 SDValue Chain = N->getOperand(0);
26295 SDValue Dest = N->getOperand(1);
26296 SDValue EFLAGS = N->getOperand(3);
26297 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
26301 Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
26302 if (Flags.getNode()) {
26303 SDValue Cond = DAG.getConstant(CC, MVT::i8);
26304 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), Chain, Dest, Cond,
26311 static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
26312 SelectionDAG &DAG) {
26313 // Take advantage of vector comparisons producing 0 or -1 in each lane to
26314 // optimize away operation when it's from a constant.
26316 // The general transformation is:
26317 // UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
26318 // AND(VECTOR_CMP(x,y), constant2)
26319 // constant2 = UNARYOP(constant)
26321 // Early exit if this isn't a vector operation, the operand of the
26322 // unary operation isn't a bitwise AND, or if the sizes of the operations
26323 // aren't the same.
26324 EVT VT = N->getValueType(0);
26325 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
26326 N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC ||
26327 VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
26330 // Now check that the other operand of the AND is a constant. We could
26331 // make the transformation for non-constant splats as well, but it's unclear
26332 // that would be a benefit as it would not eliminate any operations, just
26333 // perform one more step in scalar code before moving to the vector unit.
26334 if (BuildVectorSDNode *BV =
26335 dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
26336 // Bail out if the vector isn't a constant.
26337 if (!BV->isConstant())
26340 // Everything checks out. Build up the new and improved node.
26342 EVT IntVT = BV->getValueType(0);
26343 // Create a new constant of the appropriate type for the transformed
26345 SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
26346 // The AND node needs bitcasts to/from an integer vector type around it.
26347 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
26348 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
26349 N->getOperand(0)->getOperand(0), MaskConst);
26350 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
26357 static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
26358 const X86Subtarget *Subtarget) {
26359 // First try to optimize away the conversion entirely when it's
26360 // conditionally from a constant. Vectors only.
26361 SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG);
26362 if (Res != SDValue())
26365 // Now move on to more general possibilities.
26366 SDValue Op0 = N->getOperand(0);
26367 EVT InVT = Op0->getValueType(0);
26369 // SINT_TO_FP(v4i8) -> SINT_TO_FP(SEXT(v4i8 to v4i32))
26370 if (InVT == MVT::v8i8 || InVT == MVT::v4i8) {
26372 MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32;
26373 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
26374 return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P);
26377 // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
26378 // a 32-bit target where SSE doesn't support i64->FP operations.
26379 if (Op0.getOpcode() == ISD::LOAD) {
26380 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
26381 EVT VT = Ld->getValueType(0);
26382 if (!Ld->isVolatile() && !N->getValueType(0).isVector() &&
26383 ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() &&
26384 !Subtarget->is64Bit() && VT == MVT::i64) {
26385 SDValue FILDChain = Subtarget->getTargetLowering()->BuildFILD(
26386 SDValue(N, 0), Ld->getValueType(0), Ld->getChain(), Op0, DAG);
26387 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), FILDChain.getValue(1));
26394 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
26395 static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG,
26396 X86TargetLowering::DAGCombinerInfo &DCI) {
26397 // If the LHS and RHS of the ADC node are zero, then it can't overflow and
26398 // the result is either zero or one (depending on the input carry bit).
26399 // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
26400 if (X86::isZeroNode(N->getOperand(0)) &&
26401 X86::isZeroNode(N->getOperand(1)) &&
26402 // We don't have a good way to replace an EFLAGS use, so only do this when
26404 SDValue(N, 1).use_empty()) {
26406 EVT VT = N->getValueType(0);
26407 SDValue CarryOut = DAG.getConstant(0, N->getValueType(1));
26408 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,
26409 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
26410 DAG.getConstant(X86::COND_B,MVT::i8),
26412 DAG.getConstant(1, VT));
26413 return DCI.CombineTo(N, Res1, CarryOut);
26419 // fold (add Y, (sete X, 0)) -> adc 0, Y
26420 // (add Y, (setne X, 0)) -> sbb -1, Y
26421 // (sub (sete X, 0), Y) -> sbb 0, Y
26422 // (sub (setne X, 0), Y) -> adc -1, Y
26423 static SDValue OptimizeConditionalInDecrement(SDNode *N, SelectionDAG &DAG) {
26426 // Look through ZExts.
26427 SDValue Ext = N->getOperand(N->getOpcode() == ISD::SUB ? 1 : 0);
26428 if (Ext.getOpcode() != ISD::ZERO_EXTEND || !Ext.hasOneUse())
26431 SDValue SetCC = Ext.getOperand(0);
26432 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse())
26435 X86::CondCode CC = (X86::CondCode)SetCC.getConstantOperandVal(0);
26436 if (CC != X86::COND_E && CC != X86::COND_NE)
26439 SDValue Cmp = SetCC.getOperand(1);
26440 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
26441 !X86::isZeroNode(Cmp.getOperand(1)) ||
26442 !Cmp.getOperand(0).getValueType().isInteger())
26445 SDValue CmpOp0 = Cmp.getOperand(0);
26446 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
26447 DAG.getConstant(1, CmpOp0.getValueType()));
26449 SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1);
26450 if (CC == X86::COND_NE)
26451 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
26452 DL, OtherVal.getValueType(), OtherVal,
26453 DAG.getConstant(-1ULL, OtherVal.getValueType()), NewCmp);
26454 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
26455 DL, OtherVal.getValueType(), OtherVal,
26456 DAG.getConstant(0, OtherVal.getValueType()), NewCmp);
26459 /// PerformADDCombine - Do target-specific dag combines on integer adds.
26460 static SDValue PerformAddCombine(SDNode *N, SelectionDAG &DAG,
26461 const X86Subtarget *Subtarget) {
26462 EVT VT = N->getValueType(0);
26463 SDValue Op0 = N->getOperand(0);
26464 SDValue Op1 = N->getOperand(1);
26466 // Try to synthesize horizontal adds from adds of shuffles.
26467 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
26468 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
26469 isHorizontalBinOp(Op0, Op1, true))
26470 return DAG.getNode(X86ISD::HADD, SDLoc(N), VT, Op0, Op1);
26472 return OptimizeConditionalInDecrement(N, DAG);
26475 static SDValue PerformSubCombine(SDNode *N, SelectionDAG &DAG,
26476 const X86Subtarget *Subtarget) {
26477 SDValue Op0 = N->getOperand(0);
26478 SDValue Op1 = N->getOperand(1);
26480 // X86 can't encode an immediate LHS of a sub. See if we can push the
26481 // negation into a preceding instruction.
26482 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op0)) {
26483 // If the RHS of the sub is a XOR with one use and a constant, invert the
26484 // immediate. Then add one to the LHS of the sub so we can turn
26485 // X-Y -> X+~Y+1, saving one register.
26486 if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR &&
26487 isa<ConstantSDNode>(Op1.getOperand(1))) {
26488 APInt XorC = cast<ConstantSDNode>(Op1.getOperand(1))->getAPIntValue();
26489 EVT VT = Op0.getValueType();
26490 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
26492 DAG.getConstant(~XorC, VT));
26493 return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor,
26494 DAG.getConstant(C->getAPIntValue()+1, VT));
26498 // Try to synthesize horizontal adds from adds of shuffles.
26499 EVT VT = N->getValueType(0);
26500 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
26501 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
26502 isHorizontalBinOp(Op0, Op1, true))
26503 return DAG.getNode(X86ISD::HSUB, SDLoc(N), VT, Op0, Op1);
26505 return OptimizeConditionalInDecrement(N, DAG);
26508 /// performVZEXTCombine - Performs build vector combines
26509 static SDValue performVZEXTCombine(SDNode *N, SelectionDAG &DAG,
26510 TargetLowering::DAGCombinerInfo &DCI,
26511 const X86Subtarget *Subtarget) {
26513 MVT VT = N->getSimpleValueType(0);
26514 SDValue Op = N->getOperand(0);
26515 MVT OpVT = Op.getSimpleValueType();
26516 MVT OpEltVT = OpVT.getVectorElementType();
26517 unsigned InputBits = OpEltVT.getSizeInBits() * VT.getVectorNumElements();
26519 // (vzext (bitcast (vzext (x)) -> (vzext x)
26521 while (V.getOpcode() == ISD::BITCAST)
26522 V = V.getOperand(0);
26524 if (V != Op && V.getOpcode() == X86ISD::VZEXT) {
26525 MVT InnerVT = V.getSimpleValueType();
26526 MVT InnerEltVT = InnerVT.getVectorElementType();
26528 // If the element sizes match exactly, we can just do one larger vzext. This
26529 // is always an exact type match as vzext operates on integer types.
26530 if (OpEltVT == InnerEltVT) {
26531 assert(OpVT == InnerVT && "Types must match for vzext!");
26532 return DAG.getNode(X86ISD::VZEXT, DL, VT, V.getOperand(0));
26535 // The only other way we can combine them is if only a single element of the
26536 // inner vzext is used in the input to the outer vzext.
26537 if (InnerEltVT.getSizeInBits() < InputBits)
26540 // In this case, the inner vzext is completely dead because we're going to
26541 // only look at bits inside of the low element. Just do the outer vzext on
26542 // a bitcast of the input to the inner.
26543 return DAG.getNode(X86ISD::VZEXT, DL, VT,
26544 DAG.getNode(ISD::BITCAST, DL, OpVT, V));
26547 // Check if we can bypass extracting and re-inserting an element of an input
26548 // vector. Essentialy:
26549 // (bitcast (sclr2vec (ext_vec_elt x))) -> (bitcast x)
26550 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
26551 V.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
26552 V.getOperand(0).getSimpleValueType().getSizeInBits() == InputBits) {
26553 SDValue ExtractedV = V.getOperand(0);
26554 SDValue OrigV = ExtractedV.getOperand(0);
26555 if (auto *ExtractIdx = dyn_cast<ConstantSDNode>(ExtractedV.getOperand(1)))
26556 if (ExtractIdx->getZExtValue() == 0) {
26557 MVT OrigVT = OrigV.getSimpleValueType();
26558 // Extract a subvector if necessary...
26559 if (OrigVT.getSizeInBits() > OpVT.getSizeInBits()) {
26560 int Ratio = OrigVT.getSizeInBits() / OpVT.getSizeInBits();
26561 OrigVT = MVT::getVectorVT(OrigVT.getVectorElementType(),
26562 OrigVT.getVectorNumElements() / Ratio);
26563 OrigV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigVT, OrigV,
26564 DAG.getIntPtrConstant(0));
26566 Op = DAG.getNode(ISD::BITCAST, DL, OpVT, OrigV);
26567 return DAG.getNode(X86ISD::VZEXT, DL, VT, Op);
26574 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
26575 DAGCombinerInfo &DCI) const {
26576 SelectionDAG &DAG = DCI.DAG;
26577 switch (N->getOpcode()) {
26579 case ISD::EXTRACT_VECTOR_ELT:
26580 return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, DCI);
26583 case X86ISD::SHRUNKBLEND:
26584 return PerformSELECTCombine(N, DAG, DCI, Subtarget);
26585 case ISD::BITCAST: return PerformBITCASTCombine(N, DAG);
26586 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI, Subtarget);
26587 case ISD::ADD: return PerformAddCombine(N, DAG, Subtarget);
26588 case ISD::SUB: return PerformSubCombine(N, DAG, Subtarget);
26589 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI);
26590 case ISD::MUL: return PerformMulCombine(N, DAG, DCI);
26593 case ISD::SRL: return PerformShiftCombine(N, DAG, DCI, Subtarget);
26594 case ISD::AND: return PerformAndCombine(N, DAG, DCI, Subtarget);
26595 case ISD::OR: return PerformOrCombine(N, DAG, DCI, Subtarget);
26596 case ISD::XOR: return PerformXorCombine(N, DAG, DCI, Subtarget);
26597 case ISD::LOAD: return PerformLOADCombine(N, DAG, DCI, Subtarget);
26598 case ISD::MLOAD: return PerformMLOADCombine(N, DAG, DCI, Subtarget);
26599 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget);
26600 case ISD::MSTORE: return PerformMSTORECombine(N, DAG, Subtarget);
26601 case ISD::SINT_TO_FP: return PerformSINT_TO_FPCombine(N, DAG, Subtarget);
26602 case ISD::FADD: return PerformFADDCombine(N, DAG, Subtarget);
26603 case ISD::FSUB: return PerformFSUBCombine(N, DAG, Subtarget);
26605 case X86ISD::FOR: return PerformFORCombine(N, DAG);
26607 case X86ISD::FMAX: return PerformFMinFMaxCombine(N, DAG);
26608 case X86ISD::FAND: return PerformFANDCombine(N, DAG);
26609 case X86ISD::FANDN: return PerformFANDNCombine(N, DAG);
26610 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI);
26611 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG);
26612 case ISD::ANY_EXTEND:
26613 case ISD::ZERO_EXTEND: return PerformZExtCombine(N, DAG, DCI, Subtarget);
26614 case ISD::SIGN_EXTEND: return PerformSExtCombine(N, DAG, DCI, Subtarget);
26615 case ISD::SIGN_EXTEND_INREG:
26616 return PerformSIGN_EXTEND_INREGCombine(N, DAG, Subtarget);
26617 case ISD::TRUNCATE: return PerformTruncateCombine(N, DAG,DCI,Subtarget);
26618 case ISD::SETCC: return PerformISDSETCCCombine(N, DAG, Subtarget);
26619 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG, DCI, Subtarget);
26620 case X86ISD::BRCOND: return PerformBrCondCombine(N, DAG, DCI, Subtarget);
26621 case X86ISD::VZEXT: return performVZEXTCombine(N, DAG, DCI, Subtarget);
26622 case X86ISD::SHUFP: // Handle all target specific shuffles
26623 case X86ISD::PALIGNR:
26624 case X86ISD::UNPCKH:
26625 case X86ISD::UNPCKL:
26626 case X86ISD::MOVHLPS:
26627 case X86ISD::MOVLHPS:
26628 case X86ISD::PSHUFB:
26629 case X86ISD::PSHUFD:
26630 case X86ISD::PSHUFHW:
26631 case X86ISD::PSHUFLW:
26632 case X86ISD::MOVSS:
26633 case X86ISD::MOVSD:
26634 case X86ISD::VPERMILPI:
26635 case X86ISD::VPERM2X128:
26636 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI,Subtarget);
26637 case ISD::FMA: return PerformFMACombine(N, DAG, Subtarget);
26638 case ISD::INTRINSIC_WO_CHAIN:
26639 return PerformINTRINSIC_WO_CHAINCombine(N, DAG, Subtarget);
26640 case X86ISD::INSERTPS: {
26641 if (getTargetMachine().getOptLevel() > CodeGenOpt::None)
26642 return PerformINSERTPSCombine(N, DAG, Subtarget);
26645 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DAG, Subtarget);
26651 /// isTypeDesirableForOp - Return true if the target has native support for
26652 /// the specified value type and it is 'desirable' to use the type for the
26653 /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
26654 /// instruction encodings are longer and some i16 instructions are slow.
26655 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
26656 if (!isTypeLegal(VT))
26658 if (VT != MVT::i16)
26665 case ISD::SIGN_EXTEND:
26666 case ISD::ZERO_EXTEND:
26667 case ISD::ANY_EXTEND:
26680 /// IsDesirableToPromoteOp - This method query the target whether it is
26681 /// beneficial for dag combiner to promote the specified node. If true, it
26682 /// should return the desired promotion type by reference.
26683 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
26684 EVT VT = Op.getValueType();
26685 if (VT != MVT::i16)
26688 bool Promote = false;
26689 bool Commute = false;
26690 switch (Op.getOpcode()) {
26693 LoadSDNode *LD = cast<LoadSDNode>(Op);
26694 // If the non-extending load has a single use and it's not live out, then it
26695 // might be folded.
26696 if (LD->getExtensionType() == ISD::NON_EXTLOAD /*&&
26697 Op.hasOneUse()*/) {
26698 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
26699 UE = Op.getNode()->use_end(); UI != UE; ++UI) {
26700 // The only case where we'd want to promote LOAD (rather then it being
26701 // promoted as an operand is when it's only use is liveout.
26702 if (UI->getOpcode() != ISD::CopyToReg)
26709 case ISD::SIGN_EXTEND:
26710 case ISD::ZERO_EXTEND:
26711 case ISD::ANY_EXTEND:
26716 SDValue N0 = Op.getOperand(0);
26717 // Look out for (store (shl (load), x)).
26718 if (MayFoldLoad(N0) && MayFoldIntoStore(Op))
26731 SDValue N0 = Op.getOperand(0);
26732 SDValue N1 = Op.getOperand(1);
26733 if (!Commute && MayFoldLoad(N1))
26735 // Avoid disabling potential load folding opportunities.
26736 if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op)))
26738 if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op)))
26748 //===----------------------------------------------------------------------===//
26749 // X86 Inline Assembly Support
26750 //===----------------------------------------------------------------------===//
26753 // Helper to match a string separated by whitespace.
26754 bool matchAsmImpl(StringRef s, ArrayRef<const StringRef *> args) {
26755 s = s.substr(s.find_first_not_of(" \t")); // Skip leading whitespace.
26757 for (unsigned i = 0, e = args.size(); i != e; ++i) {
26758 StringRef piece(*args[i]);
26759 if (!s.startswith(piece)) // Check if the piece matches.
26762 s = s.substr(piece.size());
26763 StringRef::size_type pos = s.find_first_not_of(" \t");
26764 if (pos == 0) // We matched a prefix.
26772 const VariadicFunction1<bool, StringRef, StringRef, matchAsmImpl> matchAsm={};
26775 static bool clobbersFlagRegisters(const SmallVector<StringRef, 4> &AsmPieces) {
26777 if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
26778 if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{cc}") &&
26779 std::count(AsmPieces.begin(), AsmPieces.end(), "~{flags}") &&
26780 std::count(AsmPieces.begin(), AsmPieces.end(), "~{fpsr}")) {
26782 if (AsmPieces.size() == 3)
26784 else if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{dirflag}"))
26791 bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
26792 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
26794 std::string AsmStr = IA->getAsmString();
26796 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
26797 if (!Ty || Ty->getBitWidth() % 16 != 0)
26800 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
26801 SmallVector<StringRef, 4> AsmPieces;
26802 SplitString(AsmStr, AsmPieces, ";\n");
26804 switch (AsmPieces.size()) {
26805 default: return false;
26807 // FIXME: this should verify that we are targeting a 486 or better. If not,
26808 // we will turn this bswap into something that will be lowered to logical
26809 // ops instead of emitting the bswap asm. For now, we don't support 486 or
26810 // lower so don't worry about this.
26812 if (matchAsm(AsmPieces[0], "bswap", "$0") ||
26813 matchAsm(AsmPieces[0], "bswapl", "$0") ||
26814 matchAsm(AsmPieces[0], "bswapq", "$0") ||
26815 matchAsm(AsmPieces[0], "bswap", "${0:q}") ||
26816 matchAsm(AsmPieces[0], "bswapl", "${0:q}") ||
26817 matchAsm(AsmPieces[0], "bswapq", "${0:q}")) {
26818 // No need to check constraints, nothing other than the equivalent of
26819 // "=r,0" would be valid here.
26820 return IntrinsicLowering::LowerToByteSwap(CI);
26823 // rorw $$8, ${0:w} --> llvm.bswap.i16
26824 if (CI->getType()->isIntegerTy(16) &&
26825 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
26826 (matchAsm(AsmPieces[0], "rorw", "$$8,", "${0:w}") ||
26827 matchAsm(AsmPieces[0], "rolw", "$$8,", "${0:w}"))) {
26829 const std::string &ConstraintsStr = IA->getConstraintString();
26830 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
26831 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
26832 if (clobbersFlagRegisters(AsmPieces))
26833 return IntrinsicLowering::LowerToByteSwap(CI);
26837 if (CI->getType()->isIntegerTy(32) &&
26838 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
26839 matchAsm(AsmPieces[0], "rorw", "$$8,", "${0:w}") &&
26840 matchAsm(AsmPieces[1], "rorl", "$$16,", "$0") &&
26841 matchAsm(AsmPieces[2], "rorw", "$$8,", "${0:w}")) {
26843 const std::string &ConstraintsStr = IA->getConstraintString();
26844 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
26845 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
26846 if (clobbersFlagRegisters(AsmPieces))
26847 return IntrinsicLowering::LowerToByteSwap(CI);
26850 if (CI->getType()->isIntegerTy(64)) {
26851 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
26852 if (Constraints.size() >= 2 &&
26853 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
26854 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
26855 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64
26856 if (matchAsm(AsmPieces[0], "bswap", "%eax") &&
26857 matchAsm(AsmPieces[1], "bswap", "%edx") &&
26858 matchAsm(AsmPieces[2], "xchgl", "%eax,", "%edx"))
26859 return IntrinsicLowering::LowerToByteSwap(CI);
26867 /// getConstraintType - Given a constraint letter, return the type of
26868 /// constraint it is for this target.
26869 X86TargetLowering::ConstraintType
26870 X86TargetLowering::getConstraintType(const std::string &Constraint) const {
26871 if (Constraint.size() == 1) {
26872 switch (Constraint[0]) {
26883 return C_RegisterClass;
26907 return TargetLowering::getConstraintType(Constraint);
26910 /// Examine constraint type and operand type and determine a weight value.
26911 /// This object must already have been set up with the operand type
26912 /// and the current alternative constraint selected.
26913 TargetLowering::ConstraintWeight
26914 X86TargetLowering::getSingleConstraintMatchWeight(
26915 AsmOperandInfo &info, const char *constraint) const {
26916 ConstraintWeight weight = CW_Invalid;
26917 Value *CallOperandVal = info.CallOperandVal;
26918 // If we don't have a value, we can't do a match,
26919 // but allow it at the lowest weight.
26920 if (!CallOperandVal)
26922 Type *type = CallOperandVal->getType();
26923 // Look at the constraint type.
26924 switch (*constraint) {
26926 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
26937 if (CallOperandVal->getType()->isIntegerTy())
26938 weight = CW_SpecificReg;
26943 if (type->isFloatingPointTy())
26944 weight = CW_SpecificReg;
26947 if (type->isX86_MMXTy() && Subtarget->hasMMX())
26948 weight = CW_SpecificReg;
26952 if (((type->getPrimitiveSizeInBits() == 128) && Subtarget->hasSSE1()) ||
26953 ((type->getPrimitiveSizeInBits() == 256) && Subtarget->hasFp256()))
26954 weight = CW_Register;
26957 if (ConstantInt *C = dyn_cast<ConstantInt>(info.CallOperandVal)) {
26958 if (C->getZExtValue() <= 31)
26959 weight = CW_Constant;
26963 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26964 if (C->getZExtValue() <= 63)
26965 weight = CW_Constant;
26969 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26970 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f))
26971 weight = CW_Constant;
26975 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26976 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff))
26977 weight = CW_Constant;
26981 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26982 if (C->getZExtValue() <= 3)
26983 weight = CW_Constant;
26987 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
26988 if (C->getZExtValue() <= 0xff)
26989 weight = CW_Constant;
26994 if (dyn_cast<ConstantFP>(CallOperandVal)) {
26995 weight = CW_Constant;
26999 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
27000 if ((C->getSExtValue() >= -0x80000000LL) &&
27001 (C->getSExtValue() <= 0x7fffffffLL))
27002 weight = CW_Constant;
27006 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
27007 if (C->getZExtValue() <= 0xffffffff)
27008 weight = CW_Constant;
27015 /// LowerXConstraint - try to replace an X constraint, which matches anything,
27016 /// with another that has more specific requirements based on the type of the
27017 /// corresponding operand.
27018 const char *X86TargetLowering::
27019 LowerXConstraint(EVT ConstraintVT) const {
27020 // FP X constraints get lowered to SSE1/2 registers if available, otherwise
27021 // 'f' like normal targets.
27022 if (ConstraintVT.isFloatingPoint()) {
27023 if (Subtarget->hasSSE2())
27025 if (Subtarget->hasSSE1())
27029 return TargetLowering::LowerXConstraint(ConstraintVT);
27032 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
27033 /// vector. If it is invalid, don't add anything to Ops.
27034 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
27035 std::string &Constraint,
27036 std::vector<SDValue>&Ops,
27037 SelectionDAG &DAG) const {
27040 // Only support length 1 constraints for now.
27041 if (Constraint.length() > 1) return;
27043 char ConstraintLetter = Constraint[0];
27044 switch (ConstraintLetter) {
27047 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27048 if (C->getZExtValue() <= 31) {
27049 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27055 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27056 if (C->getZExtValue() <= 63) {
27057 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27063 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27064 if (isInt<8>(C->getSExtValue())) {
27065 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27071 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27072 if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
27073 (Subtarget->is64Bit() && C->getZExtValue() == 0xffffffff)) {
27074 Result = DAG.getTargetConstant(C->getSExtValue(), Op.getValueType());
27080 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27081 if (C->getZExtValue() <= 3) {
27082 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27088 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27089 if (C->getZExtValue() <= 255) {
27090 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27096 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27097 if (C->getZExtValue() <= 127) {
27098 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27104 // 32-bit signed value
27105 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27106 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
27107 C->getSExtValue())) {
27108 // Widen to 64 bits here to get it sign extended.
27109 Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64);
27112 // FIXME gcc accepts some relocatable values here too, but only in certain
27113 // memory models; it's complicated.
27118 // 32-bit unsigned value
27119 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
27120 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
27121 C->getZExtValue())) {
27122 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
27126 // FIXME gcc accepts some relocatable values here too, but only in certain
27127 // memory models; it's complicated.
27131 // Literal immediates are always ok.
27132 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
27133 // Widen to 64 bits here to get it sign extended.
27134 Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64);
27138 // In any sort of PIC mode addresses need to be computed at runtime by
27139 // adding in a register or some sort of table lookup. These can't
27140 // be used as immediates.
27141 if (Subtarget->isPICStyleGOT() || Subtarget->isPICStyleStubPIC())
27144 // If we are in non-pic codegen mode, we allow the address of a global (with
27145 // an optional displacement) to be used with 'i'.
27146 GlobalAddressSDNode *GA = nullptr;
27147 int64_t Offset = 0;
27149 // Match either (GA), (GA+C), (GA+C1+C2), etc.
27151 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) {
27152 Offset += GA->getOffset();
27154 } else if (Op.getOpcode() == ISD::ADD) {
27155 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
27156 Offset += C->getZExtValue();
27157 Op = Op.getOperand(0);
27160 } else if (Op.getOpcode() == ISD::SUB) {
27161 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
27162 Offset += -C->getZExtValue();
27163 Op = Op.getOperand(0);
27168 // Otherwise, this isn't something we can handle, reject it.
27172 const GlobalValue *GV = GA->getGlobal();
27173 // If we require an extra load to get this address, as in PIC mode, we
27174 // can't accept it.
27175 if (isGlobalStubReference(
27176 Subtarget->ClassifyGlobalReference(GV, DAG.getTarget())))
27179 Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op),
27180 GA->getValueType(0), Offset);
27185 if (Result.getNode()) {
27186 Ops.push_back(Result);
27189 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
27192 std::pair<unsigned, const TargetRegisterClass*>
27193 X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
27195 // First, see if this is a constraint that directly corresponds to an LLVM
27197 if (Constraint.size() == 1) {
27198 // GCC Constraint Letters
27199 switch (Constraint[0]) {
27201 // TODO: Slight differences here in allocation order and leaving
27202 // RIP in the class. Do they matter any more here than they do
27203 // in the normal allocation?
27204 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
27205 if (Subtarget->is64Bit()) {
27206 if (VT == MVT::i32 || VT == MVT::f32)
27207 return std::make_pair(0U, &X86::GR32RegClass);
27208 if (VT == MVT::i16)
27209 return std::make_pair(0U, &X86::GR16RegClass);
27210 if (VT == MVT::i8 || VT == MVT::i1)
27211 return std::make_pair(0U, &X86::GR8RegClass);
27212 if (VT == MVT::i64 || VT == MVT::f64)
27213 return std::make_pair(0U, &X86::GR64RegClass);
27216 // 32-bit fallthrough
27217 case 'Q': // Q_REGS
27218 if (VT == MVT::i32 || VT == MVT::f32)
27219 return std::make_pair(0U, &X86::GR32_ABCDRegClass);
27220 if (VT == MVT::i16)
27221 return std::make_pair(0U, &X86::GR16_ABCDRegClass);
27222 if (VT == MVT::i8 || VT == MVT::i1)
27223 return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
27224 if (VT == MVT::i64)
27225 return std::make_pair(0U, &X86::GR64_ABCDRegClass);
27227 case 'r': // GENERAL_REGS
27228 case 'l': // INDEX_REGS
27229 if (VT == MVT::i8 || VT == MVT::i1)
27230 return std::make_pair(0U, &X86::GR8RegClass);
27231 if (VT == MVT::i16)
27232 return std::make_pair(0U, &X86::GR16RegClass);
27233 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit())
27234 return std::make_pair(0U, &X86::GR32RegClass);
27235 return std::make_pair(0U, &X86::GR64RegClass);
27236 case 'R': // LEGACY_REGS
27237 if (VT == MVT::i8 || VT == MVT::i1)
27238 return std::make_pair(0U, &X86::GR8_NOREXRegClass);
27239 if (VT == MVT::i16)
27240 return std::make_pair(0U, &X86::GR16_NOREXRegClass);
27241 if (VT == MVT::i32 || !Subtarget->is64Bit())
27242 return std::make_pair(0U, &X86::GR32_NOREXRegClass);
27243 return std::make_pair(0U, &X86::GR64_NOREXRegClass);
27244 case 'f': // FP Stack registers.
27245 // If SSE is enabled for this VT, use f80 to ensure the isel moves the
27246 // value to the correct fpstack register class.
27247 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
27248 return std::make_pair(0U, &X86::RFP32RegClass);
27249 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
27250 return std::make_pair(0U, &X86::RFP64RegClass);
27251 return std::make_pair(0U, &X86::RFP80RegClass);
27252 case 'y': // MMX_REGS if MMX allowed.
27253 if (!Subtarget->hasMMX()) break;
27254 return std::make_pair(0U, &X86::VR64RegClass);
27255 case 'Y': // SSE_REGS if SSE2 allowed
27256 if (!Subtarget->hasSSE2()) break;
27258 case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
27259 if (!Subtarget->hasSSE1()) break;
27261 switch (VT.SimpleTy) {
27263 // Scalar SSE types.
27266 return std::make_pair(0U, &X86::FR32RegClass);
27269 return std::make_pair(0U, &X86::FR64RegClass);
27277 return std::make_pair(0U, &X86::VR128RegClass);
27285 return std::make_pair(0U, &X86::VR256RegClass);
27290 return std::make_pair(0U, &X86::VR512RegClass);
27296 // Use the default implementation in TargetLowering to convert the register
27297 // constraint into a member of a register class.
27298 std::pair<unsigned, const TargetRegisterClass*> Res;
27299 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
27301 // Not found as a standard register?
27303 // Map st(0) -> st(7) -> ST0
27304 if (Constraint.size() == 7 && Constraint[0] == '{' &&
27305 tolower(Constraint[1]) == 's' &&
27306 tolower(Constraint[2]) == 't' &&
27307 Constraint[3] == '(' &&
27308 (Constraint[4] >= '0' && Constraint[4] <= '7') &&
27309 Constraint[5] == ')' &&
27310 Constraint[6] == '}') {
27312 Res.first = X86::FP0+Constraint[4]-'0';
27313 Res.second = &X86::RFP80RegClass;
27317 // GCC allows "st(0)" to be called just plain "st".
27318 if (StringRef("{st}").equals_lower(Constraint)) {
27319 Res.first = X86::FP0;
27320 Res.second = &X86::RFP80RegClass;
27325 if (StringRef("{flags}").equals_lower(Constraint)) {
27326 Res.first = X86::EFLAGS;
27327 Res.second = &X86::CCRRegClass;
27331 // 'A' means EAX + EDX.
27332 if (Constraint == "A") {
27333 Res.first = X86::EAX;
27334 Res.second = &X86::GR32_ADRegClass;
27340 // Otherwise, check to see if this is a register class of the wrong value
27341 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
27342 // turn into {ax},{dx}.
27343 if (Res.second->hasType(VT))
27344 return Res; // Correct type already, nothing to do.
27346 // All of the single-register GCC register classes map their values onto
27347 // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp". If we
27348 // really want an 8-bit or 32-bit register, map to the appropriate register
27349 // class and return the appropriate register.
27350 if (Res.second == &X86::GR16RegClass) {
27351 if (VT == MVT::i8 || VT == MVT::i1) {
27352 unsigned DestReg = 0;
27353 switch (Res.first) {
27355 case X86::AX: DestReg = X86::AL; break;
27356 case X86::DX: DestReg = X86::DL; break;
27357 case X86::CX: DestReg = X86::CL; break;
27358 case X86::BX: DestReg = X86::BL; break;
27361 Res.first = DestReg;
27362 Res.second = &X86::GR8RegClass;
27364 } else if (VT == MVT::i32 || VT == MVT::f32) {
27365 unsigned DestReg = 0;
27366 switch (Res.first) {
27368 case X86::AX: DestReg = X86::EAX; break;
27369 case X86::DX: DestReg = X86::EDX; break;
27370 case X86::CX: DestReg = X86::ECX; break;
27371 case X86::BX: DestReg = X86::EBX; break;
27372 case X86::SI: DestReg = X86::ESI; break;
27373 case X86::DI: DestReg = X86::EDI; break;
27374 case X86::BP: DestReg = X86::EBP; break;
27375 case X86::SP: DestReg = X86::ESP; break;
27378 Res.first = DestReg;
27379 Res.second = &X86::GR32RegClass;
27381 } else if (VT == MVT::i64 || VT == MVT::f64) {
27382 unsigned DestReg = 0;
27383 switch (Res.first) {
27385 case X86::AX: DestReg = X86::RAX; break;
27386 case X86::DX: DestReg = X86::RDX; break;
27387 case X86::CX: DestReg = X86::RCX; break;
27388 case X86::BX: DestReg = X86::RBX; break;
27389 case X86::SI: DestReg = X86::RSI; break;
27390 case X86::DI: DestReg = X86::RDI; break;
27391 case X86::BP: DestReg = X86::RBP; break;
27392 case X86::SP: DestReg = X86::RSP; break;
27395 Res.first = DestReg;
27396 Res.second = &X86::GR64RegClass;
27399 } else if (Res.second == &X86::FR32RegClass ||
27400 Res.second == &X86::FR64RegClass ||
27401 Res.second == &X86::VR128RegClass ||
27402 Res.second == &X86::VR256RegClass ||
27403 Res.second == &X86::FR32XRegClass ||
27404 Res.second == &X86::FR64XRegClass ||
27405 Res.second == &X86::VR128XRegClass ||
27406 Res.second == &X86::VR256XRegClass ||
27407 Res.second == &X86::VR512RegClass) {
27408 // Handle references to XMM physical registers that got mapped into the
27409 // wrong class. This can happen with constraints like {xmm0} where the
27410 // target independent register mapper will just pick the first match it can
27411 // find, ignoring the required type.
27413 if (VT == MVT::f32 || VT == MVT::i32)
27414 Res.second = &X86::FR32RegClass;
27415 else if (VT == MVT::f64 || VT == MVT::i64)
27416 Res.second = &X86::FR64RegClass;
27417 else if (X86::VR128RegClass.hasType(VT))
27418 Res.second = &X86::VR128RegClass;
27419 else if (X86::VR256RegClass.hasType(VT))
27420 Res.second = &X86::VR256RegClass;
27421 else if (X86::VR512RegClass.hasType(VT))
27422 Res.second = &X86::VR512RegClass;
27428 int X86TargetLowering::getScalingFactorCost(const AddrMode &AM,
27430 // Scaling factors are not free at all.
27431 // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
27432 // will take 2 allocations in the out of order engine instead of 1
27433 // for plain addressing mode, i.e. inst (reg1).
27435 // vaddps (%rsi,%drx), %ymm0, %ymm1
27436 // Requires two allocations (one for the load, one for the computation)
27438 // vaddps (%rsi), %ymm0, %ymm1
27439 // Requires just 1 allocation, i.e., freeing allocations for other operations
27440 // and having less micro operations to execute.
27442 // For some X86 architectures, this is even worse because for instance for
27443 // stores, the complex addressing mode forces the instruction to use the
27444 // "load" ports instead of the dedicated "store" port.
27445 // E.g., on Haswell:
27446 // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
27447 // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
27448 if (isLegalAddressingMode(AM, Ty))
27449 // Scale represents reg2 * scale, thus account for 1
27450 // as soon as we use a second register.
27451 return AM.Scale != 0;
27455 bool X86TargetLowering::isTargetFTOL() const {
27456 return Subtarget->isTargetKnownWindowsMSVC() && !Subtarget->is64Bit();