1 //===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file was developed by Chris Lattner and is distributed under
6 // the University of Illinois Open Source License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the interfaces that X86 uses to lower LLVM code into a
13 //===----------------------------------------------------------------------===//
16 #include "X86InstrBuilder.h"
17 #include "X86ISelLowering.h"
18 #include "X86TargetMachine.h"
19 #include "llvm/CallingConv.h"
20 #include "llvm/Constants.h"
21 #include "llvm/Function.h"
22 #include "llvm/ADT/VectorExtras.h"
23 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/SelectionDAG.h"
28 #include "llvm/CodeGen/SSARegMap.h"
29 #include "llvm/Support/MathExtras.h"
30 #include "llvm/Target/TargetOptions.h"
34 #include "llvm/Support/CommandLine.h"
35 static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden,
36 cl::desc("Enable fastcc on X86"));
38 X86TargetLowering::X86TargetLowering(TargetMachine &TM)
39 : TargetLowering(TM) {
40 Subtarget = &TM.getSubtarget<X86Subtarget>();
41 X86ScalarSSE = Subtarget->hasSSE2();
43 // Set up the TargetLowering object.
45 // X86 is weird, it always uses i8 for shift amounts and setcc results.
46 setShiftAmountType(MVT::i8);
47 setSetCCResultType(MVT::i8);
48 setSetCCResultContents(ZeroOrOneSetCCResult);
49 setSchedulingPreference(SchedulingForRegPressure);
50 setShiftAmountFlavor(Mask); // shl X, 32 == shl X, 0
51 setStackPointerRegisterToSaveRestore(X86::ESP);
53 if (!Subtarget->isTargetDarwin())
54 // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
55 setUseUnderscoreSetJmpLongJmp(true);
57 // Add legal addressing mode scale values.
58 addLegalAddressScale(8);
59 addLegalAddressScale(4);
60 addLegalAddressScale(2);
61 // Enter the ones which require both scale + index last. These are more
63 addLegalAddressScale(9);
64 addLegalAddressScale(5);
65 addLegalAddressScale(3);
67 // Set up the register classes.
68 addRegisterClass(MVT::i8, X86::R8RegisterClass);
69 addRegisterClass(MVT::i16, X86::R16RegisterClass);
70 addRegisterClass(MVT::i32, X86::R32RegisterClass);
72 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
74 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
75 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
76 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
79 // No SSE i64 SINT_TO_FP, so expand i32 UINT_TO_FP instead.
80 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand);
82 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
84 // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
86 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
87 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
88 // SSE has no i16 to fp conversion, only i32
90 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
92 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
93 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
96 // We can handle SINT_TO_FP and FP_TO_SINT from/to i64 even though i64
98 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
99 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
101 // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
103 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
104 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
107 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
109 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
110 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
113 // Handle FP_TO_UINT by promoting the destination to a larger signed
115 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
116 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
117 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
119 if (X86ScalarSSE && !Subtarget->hasSSE3())
120 // Expand FP_TO_UINT into a select.
121 // FIXME: We would like to use a Custom expander here eventually to do
122 // the optimal thing for SSE vs. the default expansion in the legalizer.
123 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
125 // With SSE3 we can use fisttpll to convert to a signed i64.
126 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
128 setOperationAction(ISD::BIT_CONVERT , MVT::f32 , Expand);
129 setOperationAction(ISD::BIT_CONVERT , MVT::i32 , Expand);
131 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
132 setOperationAction(ISD::BR_CC , MVT::Other, Expand);
133 setOperationAction(ISD::SELECT_CC , MVT::Other, Expand);
134 setOperationAction(ISD::MEMMOVE , MVT::Other, Expand);
135 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Expand);
136 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Expand);
137 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
138 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
139 setOperationAction(ISD::SEXTLOAD , MVT::i1 , Expand);
140 setOperationAction(ISD::FREM , MVT::f64 , Expand);
141 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
142 setOperationAction(ISD::CTTZ , MVT::i8 , Expand);
143 setOperationAction(ISD::CTLZ , MVT::i8 , Expand);
144 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
145 setOperationAction(ISD::CTTZ , MVT::i16 , Expand);
146 setOperationAction(ISD::CTLZ , MVT::i16 , Expand);
147 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
148 setOperationAction(ISD::CTTZ , MVT::i32 , Expand);
149 setOperationAction(ISD::CTLZ , MVT::i32 , Expand);
150 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
151 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
153 // These should be promoted to a larger select which is supported.
154 setOperationAction(ISD::SELECT , MVT::i1 , Promote);
155 setOperationAction(ISD::SELECT , MVT::i8 , Promote);
157 // X86 wants to expand cmov itself.
158 setOperationAction(ISD::SELECT , MVT::i16 , Custom);
159 setOperationAction(ISD::SELECT , MVT::i32 , Custom);
160 setOperationAction(ISD::SELECT , MVT::f32 , Custom);
161 setOperationAction(ISD::SELECT , MVT::f64 , Custom);
162 setOperationAction(ISD::SETCC , MVT::i8 , Custom);
163 setOperationAction(ISD::SETCC , MVT::i16 , Custom);
164 setOperationAction(ISD::SETCC , MVT::i32 , Custom);
165 setOperationAction(ISD::SETCC , MVT::f32 , Custom);
166 setOperationAction(ISD::SETCC , MVT::f64 , Custom);
167 // X86 ret instruction may pop stack.
168 setOperationAction(ISD::RET , MVT::Other, Custom);
170 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom);
171 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom);
172 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom);
173 // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
174 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom);
175 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom);
176 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom);
177 // X86 wants to expand memset / memcpy itself.
178 setOperationAction(ISD::MEMSET , MVT::Other, Custom);
179 setOperationAction(ISD::MEMCPY , MVT::Other, Custom);
181 // We don't have line number support yet.
182 setOperationAction(ISD::LOCATION, MVT::Other, Expand);
183 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
184 // FIXME - use subtarget debug flags
185 if (!Subtarget->isTargetDarwin())
186 setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand);
188 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
189 setOperationAction(ISD::VASTART , MVT::Other, Custom);
191 // Use the default implementation.
192 setOperationAction(ISD::VAARG , MVT::Other, Expand);
193 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
194 setOperationAction(ISD::VAEND , MVT::Other, Expand);
195 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
196 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
197 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand);
199 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
200 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
203 // Set up the FP register classes.
204 addRegisterClass(MVT::f32, X86::FR32RegisterClass);
205 addRegisterClass(MVT::f64, X86::FR64RegisterClass);
207 // SSE has no load+extend ops
208 setOperationAction(ISD::EXTLOAD, MVT::f32, Expand);
209 setOperationAction(ISD::ZEXTLOAD, MVT::f32, Expand);
211 // Use ANDPD to simulate FABS.
212 setOperationAction(ISD::FABS , MVT::f64, Custom);
213 setOperationAction(ISD::FABS , MVT::f32, Custom);
215 // Use XORP to simulate FNEG.
216 setOperationAction(ISD::FNEG , MVT::f64, Custom);
217 setOperationAction(ISD::FNEG , MVT::f32, Custom);
219 // We don't support sin/cos/fmod
220 setOperationAction(ISD::FSIN , MVT::f64, Expand);
221 setOperationAction(ISD::FCOS , MVT::f64, Expand);
222 setOperationAction(ISD::FREM , MVT::f64, Expand);
223 setOperationAction(ISD::FSIN , MVT::f32, Expand);
224 setOperationAction(ISD::FCOS , MVT::f32, Expand);
225 setOperationAction(ISD::FREM , MVT::f32, Expand);
227 // Expand FP immediates into loads from the stack, except for the special
229 setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
230 setOperationAction(ISD::ConstantFP, MVT::f32, Expand);
231 addLegalFPImmediate(+0.0); // xorps / xorpd
233 // Set up the FP register classes.
234 addRegisterClass(MVT::f64, X86::RFPRegisterClass);
236 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
239 setOperationAction(ISD::FSIN , MVT::f64 , Expand);
240 setOperationAction(ISD::FCOS , MVT::f64 , Expand);
243 setOperationAction(ISD::ConstantFP, MVT::f64, Expand);
244 addLegalFPImmediate(+0.0); // FLD0
245 addLegalFPImmediate(+1.0); // FLD1
246 addLegalFPImmediate(-0.0); // FLD0/FCHS
247 addLegalFPImmediate(-1.0); // FLD1/FCHS
250 // First set operation action for all vector types to expand. Then we
251 // will selectively turn on ones that can be effectively codegen'd.
252 for (unsigned VT = (unsigned)MVT::Vector + 1;
253 VT != (unsigned)MVT::LAST_VALUETYPE; VT++) {
254 setOperationAction(ISD::ADD , (MVT::ValueType)VT, Expand);
255 setOperationAction(ISD::SUB , (MVT::ValueType)VT, Expand);
256 setOperationAction(ISD::MUL , (MVT::ValueType)VT, Expand);
257 setOperationAction(ISD::LOAD, (MVT::ValueType)VT, Expand);
258 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Expand);
259 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Expand);
262 if (Subtarget->hasMMX()) {
263 addRegisterClass(MVT::v8i8, X86::VR64RegisterClass);
264 addRegisterClass(MVT::v4i16, X86::VR64RegisterClass);
265 addRegisterClass(MVT::v2i32, X86::VR64RegisterClass);
267 // FIXME: add MMX packed arithmetics
268 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i8, Expand);
269 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Expand);
270 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i32, Expand);
273 if (Subtarget->hasSSE1()) {
274 addRegisterClass(MVT::v4f32, X86::VR128RegisterClass);
276 setOperationAction(ISD::ADD, MVT::v4f32, Legal);
277 setOperationAction(ISD::SUB, MVT::v4f32, Legal);
278 setOperationAction(ISD::MUL, MVT::v4f32, Legal);
279 setOperationAction(ISD::LOAD, MVT::v4f32, Legal);
280 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Expand);
281 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
284 if (Subtarget->hasSSE2()) {
285 addRegisterClass(MVT::v2f64, X86::VR128RegisterClass);
286 addRegisterClass(MVT::v16i8, X86::VR128RegisterClass);
287 addRegisterClass(MVT::v8i16, X86::VR128RegisterClass);
288 addRegisterClass(MVT::v4i32, X86::VR128RegisterClass);
289 addRegisterClass(MVT::v2i64, X86::VR128RegisterClass);
292 setOperationAction(ISD::ADD, MVT::v2f64, Legal);
293 setOperationAction(ISD::ADD, MVT::v16i8, Legal);
294 setOperationAction(ISD::ADD, MVT::v8i16, Legal);
295 setOperationAction(ISD::ADD, MVT::v4i32, Legal);
296 setOperationAction(ISD::SUB, MVT::v2f64, Legal);
297 setOperationAction(ISD::MUL, MVT::v2f64, Legal);
298 setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
299 setOperationAction(ISD::LOAD, MVT::v16i8, Legal);
300 setOperationAction(ISD::LOAD, MVT::v8i16, Legal);
301 setOperationAction(ISD::LOAD, MVT::v4i32, Legal);
302 setOperationAction(ISD::LOAD, MVT::v2i64, Legal);
303 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Expand);
304 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Expand);
305 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Expand);
306 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Expand);
307 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
308 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
309 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
312 computeRegisterProperties();
314 // FIXME: These should be based on subtarget info. Plus, the values should
315 // be smaller when we are in optimizing for size mode.
316 maxStoresPerMemset = 16; // For %llvm.memset -> sequence of stores
317 maxStoresPerMemcpy = 16; // For %llvm.memcpy -> sequence of stores
318 maxStoresPerMemmove = 16; // For %llvm.memmove -> sequence of stores
319 allowUnalignedMemoryAccesses = true; // x86 supports it!
322 std::vector<SDOperand>
323 X86TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
324 if (F.getCallingConv() == CallingConv::Fast && EnableFastCC)
325 return LowerFastCCArguments(F, DAG);
326 return LowerCCCArguments(F, DAG);
329 std::pair<SDOperand, SDOperand>
330 X86TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
331 bool isVarArg, unsigned CallingConv,
333 SDOperand Callee, ArgListTy &Args,
335 assert((!isVarArg || CallingConv == CallingConv::C) &&
336 "Only C takes varargs!");
338 // If the callee is a GlobalAddress node (quite common, every direct call is)
339 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
340 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
341 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
342 else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
343 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
345 if (CallingConv == CallingConv::Fast && EnableFastCC)
346 return LowerFastCCCallTo(Chain, RetTy, isTailCall, Callee, Args, DAG);
347 return LowerCCCCallTo(Chain, RetTy, isVarArg, isTailCall, Callee, Args, DAG);
350 //===----------------------------------------------------------------------===//
351 // C Calling Convention implementation
352 //===----------------------------------------------------------------------===//
354 std::vector<SDOperand>
355 X86TargetLowering::LowerCCCArguments(Function &F, SelectionDAG &DAG) {
356 std::vector<SDOperand> ArgValues;
358 MachineFunction &MF = DAG.getMachineFunction();
359 MachineFrameInfo *MFI = MF.getFrameInfo();
361 // Add DAG nodes to load the arguments... On entry to a function on the X86,
362 // the stack frame looks like this:
364 // [ESP] -- return address
365 // [ESP + 4] -- first argument (leftmost lexically)
366 // [ESP + 8] -- second argument, if first argument is four bytes in size
369 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot
370 for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
371 MVT::ValueType ObjectVT = getValueType(I->getType());
372 unsigned ArgIncrement = 4;
375 default: assert(0 && "Unhandled argument type!");
377 case MVT::i8: ObjSize = 1; break;
378 case MVT::i16: ObjSize = 2; break;
379 case MVT::i32: ObjSize = 4; break;
380 case MVT::i64: ObjSize = ArgIncrement = 8; break;
381 case MVT::f32: ObjSize = 4; break;
382 case MVT::f64: ObjSize = ArgIncrement = 8; break;
384 // Create the frame index object for this incoming parameter...
385 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
387 // Create the SelectionDAG nodes corresponding to a load from this parameter
388 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
390 // Don't codegen dead arguments. FIXME: remove this check when we can nuke
394 ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
395 DAG.getSrcValue(NULL));
397 if (MVT::isInteger(ObjectVT))
398 ArgValue = DAG.getConstant(0, ObjectVT);
400 ArgValue = DAG.getConstantFP(0, ObjectVT);
402 ArgValues.push_back(ArgValue);
404 ArgOffset += ArgIncrement; // Move on to the next argument...
407 // If the function takes variable number of arguments, make a frame index for
408 // the start of the first vararg value... for expansion of llvm.va_start.
410 VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
411 ReturnAddrIndex = 0; // No return address slot generated yet.
412 BytesToPopOnReturn = 0; // Callee pops nothing.
413 BytesCallerReserves = ArgOffset;
415 // Finally, inform the code generator which regs we return values in.
416 switch (getValueType(F.getReturnType())) {
417 default: assert(0 && "Unknown type!");
418 case MVT::isVoid: break;
423 MF.addLiveOut(X86::EAX);
426 MF.addLiveOut(X86::EAX);
427 MF.addLiveOut(X86::EDX);
431 MF.addLiveOut(X86::ST0);
437 std::pair<SDOperand, SDOperand>
438 X86TargetLowering::LowerCCCCallTo(SDOperand Chain, const Type *RetTy,
439 bool isVarArg, bool isTailCall,
440 SDOperand Callee, ArgListTy &Args,
442 // Count how many bytes are to be pushed on the stack.
443 unsigned NumBytes = 0;
447 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(0, getPointerTy()));
449 for (unsigned i = 0, e = Args.size(); i != e; ++i)
450 switch (getValueType(Args[i].second)) {
451 default: assert(0 && "Unknown value type!");
465 Chain = DAG.getCALLSEQ_START(Chain,
466 DAG.getConstant(NumBytes, getPointerTy()));
468 // Arguments go on the stack in reverse order, as specified by the ABI.
469 unsigned ArgOffset = 0;
470 SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32);
471 std::vector<SDOperand> Stores;
473 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
474 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
475 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
477 switch (getValueType(Args[i].second)) {
478 default: assert(0 && "Unexpected ValueType for argument!");
482 // Promote the integer to 32 bits. If the input type is signed use a
483 // sign extend, otherwise use a zero extend.
484 if (Args[i].second->isSigned())
485 Args[i].first =DAG.getNode(ISD::SIGN_EXTEND, MVT::i32, Args[i].first);
487 Args[i].first =DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Args[i].first);
492 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
493 Args[i].first, PtrOff,
494 DAG.getSrcValue(NULL)));
499 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
500 Args[i].first, PtrOff,
501 DAG.getSrcValue(NULL)));
506 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
509 std::vector<MVT::ValueType> RetVals;
510 MVT::ValueType RetTyVT = getValueType(RetTy);
511 RetVals.push_back(MVT::Other);
513 // The result values produced have to be legal. Promote the result.
515 case MVT::isVoid: break;
517 RetVals.push_back(RetTyVT);
522 RetVals.push_back(MVT::i32);
526 RetVals.push_back(MVT::f32);
528 RetVals.push_back(MVT::f64);
531 RetVals.push_back(MVT::i32);
532 RetVals.push_back(MVT::i32);
536 std::vector<MVT::ValueType> NodeTys;
537 NodeTys.push_back(MVT::Other); // Returns a chain
538 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
539 std::vector<SDOperand> Ops;
540 Ops.push_back(Chain);
541 Ops.push_back(Callee);
543 // FIXME: Do not generate X86ISD::TAILCALL for now.
544 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
545 SDOperand InFlag = Chain.getValue(1);
548 NodeTys.push_back(MVT::Other); // Returns a chain
549 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
551 Ops.push_back(Chain);
552 Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
553 Ops.push_back(DAG.getConstant(0, getPointerTy()));
554 Ops.push_back(InFlag);
555 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, Ops);
556 InFlag = Chain.getValue(1);
559 if (RetTyVT != MVT::isVoid) {
561 default: assert(0 && "Unknown value type to return!");
564 RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
565 Chain = RetVal.getValue(1);
566 if (RetTyVT == MVT::i1)
567 RetVal = DAG.getNode(ISD::TRUNCATE, MVT::i1, RetVal);
570 RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
571 Chain = RetVal.getValue(1);
574 RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
575 Chain = RetVal.getValue(1);
578 SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
579 SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
581 RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
582 Chain = Hi.getValue(1);
587 std::vector<MVT::ValueType> Tys;
588 Tys.push_back(MVT::f64);
589 Tys.push_back(MVT::Other);
590 Tys.push_back(MVT::Flag);
591 std::vector<SDOperand> Ops;
592 Ops.push_back(Chain);
593 Ops.push_back(InFlag);
594 RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
595 Chain = RetVal.getValue(1);
596 InFlag = RetVal.getValue(2);
598 // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This
599 // shouldn't be necessary except that RFP cannot be live across
600 // multiple blocks. When stackifier is fixed, they can be uncoupled.
601 MachineFunction &MF = DAG.getMachineFunction();
602 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
603 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
605 Tys.push_back(MVT::Other);
607 Ops.push_back(Chain);
608 Ops.push_back(RetVal);
609 Ops.push_back(StackSlot);
610 Ops.push_back(DAG.getValueType(RetTyVT));
611 Ops.push_back(InFlag);
612 Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
613 RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
614 DAG.getSrcValue(NULL));
615 Chain = RetVal.getValue(1);
618 if (RetTyVT == MVT::f32 && !X86ScalarSSE)
619 // FIXME: we would really like to remember that this FP_ROUND
620 // operation is okay to eliminate if we allow excess FP precision.
621 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
627 return std::make_pair(RetVal, Chain);
630 //===----------------------------------------------------------------------===//
631 // Fast Calling Convention implementation
632 //===----------------------------------------------------------------------===//
634 // The X86 'fast' calling convention passes up to two integer arguments in
635 // registers (an appropriate portion of EAX/EDX), passes arguments in C order,
636 // and requires that the callee pop its arguments off the stack (allowing proper
637 // tail calls), and has the same return value conventions as C calling convs.
639 // This calling convention always arranges for the callee pop value to be 8n+4
640 // bytes, which is needed for tail recursion elimination and stack alignment
643 // Note that this can be enhanced in the future to pass fp vals in registers
644 // (when we have a global fp allocator) and do other tricks.
647 /// AddLiveIn - This helper function adds the specified physical register to the
648 /// MachineFunction as a live in value. It also creates a corresponding virtual
650 static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg,
651 TargetRegisterClass *RC) {
652 assert(RC->contains(PReg) && "Not the correct regclass!");
653 unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC);
654 MF.addLiveIn(PReg, VReg);
658 // FASTCC_NUM_INT_ARGS_INREGS - This is the max number of integer arguments
659 // to pass in registers. 0 is none, 1 is is "use EAX", 2 is "use EAX and
660 // EDX". Anything more is illegal.
662 // FIXME: The linscan register allocator currently has problem with
663 // coallescing. At the time of this writing, whenever it decides to coallesce
664 // a physreg with a virtreg, this increases the size of the physreg's live
665 // range, and the live range cannot ever be reduced. This causes problems if
666 // too many physregs are coalleced with virtregs, which can cause the register
667 // allocator to wedge itself.
669 // This code triggers this problem more often if we pass args in registers,
670 // so disable it until this is fixed.
672 // NOTE: this isn't marked const, so that GCC doesn't emit annoying warnings
673 // about code being dead.
675 static unsigned FASTCC_NUM_INT_ARGS_INREGS = 0;
678 std::vector<SDOperand>
679 X86TargetLowering::LowerFastCCArguments(Function &F, SelectionDAG &DAG) {
680 std::vector<SDOperand> ArgValues;
682 MachineFunction &MF = DAG.getMachineFunction();
683 MachineFrameInfo *MFI = MF.getFrameInfo();
685 // Add DAG nodes to load the arguments... On entry to a function the stack
686 // frame looks like this:
688 // [ESP] -- return address
689 // [ESP + 4] -- first nonreg argument (leftmost lexically)
690 // [ESP + 8] -- second nonreg argument, if first argument is 4 bytes in size
692 unsigned ArgOffset = 0; // Frame mechanisms handle retaddr slot
694 // Keep track of the number of integer regs passed so far. This can be either
695 // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
697 unsigned NumIntRegs = 0;
699 for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
700 MVT::ValueType ObjectVT = getValueType(I->getType());
701 unsigned ArgIncrement = 4;
702 unsigned ObjSize = 0;
706 default: assert(0 && "Unhandled argument type!");
709 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
710 if (!I->use_empty()) {
711 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL,
712 X86::R8RegisterClass);
713 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i8);
714 DAG.setRoot(ArgValue.getValue(1));
715 if (ObjectVT == MVT::i1)
716 // FIXME: Should insert a assertzext here.
717 ArgValue = DAG.getNode(ISD::TRUNCATE, MVT::i1, ArgValue);
726 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
727 if (!I->use_empty()) {
728 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX,
729 X86::R16RegisterClass);
730 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i16);
731 DAG.setRoot(ArgValue.getValue(1));
739 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
740 if (!I->use_empty()) {
741 unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::EDX : X86::EAX,
742 X86::R32RegisterClass);
743 ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i32);
744 DAG.setRoot(ArgValue.getValue(1));
752 if (NumIntRegs+2 <= FASTCC_NUM_INT_ARGS_INREGS) {
753 if (!I->use_empty()) {
754 unsigned BotReg = AddLiveIn(MF, X86::EAX, X86::R32RegisterClass);
755 unsigned TopReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
757 SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
758 SDOperand Hi = DAG.getCopyFromReg(Low.getValue(1), TopReg, MVT::i32);
759 DAG.setRoot(Hi.getValue(1));
761 ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
765 } else if (NumIntRegs+1 <= FASTCC_NUM_INT_ARGS_INREGS) {
766 if (!I->use_empty()) {
767 unsigned BotReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
768 SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
769 DAG.setRoot(Low.getValue(1));
771 // Load the high part from memory.
772 // Create the frame index object for this incoming parameter...
773 int FI = MFI->CreateFixedObject(4, ArgOffset);
774 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
775 SDOperand Hi = DAG.getLoad(MVT::i32, DAG.getEntryNode(), FIN,
776 DAG.getSrcValue(NULL));
777 ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
780 NumIntRegs = FASTCC_NUM_INT_ARGS_INREGS;
783 ObjSize = ArgIncrement = 8;
785 case MVT::f32: ObjSize = 4; break;
786 case MVT::f64: ObjSize = ArgIncrement = 8; break;
789 // Don't codegen dead arguments. FIXME: remove this check when we can nuke
791 if (ObjSize && !I->use_empty()) {
792 // Create the frame index object for this incoming parameter...
793 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
795 // Create the SelectionDAG nodes corresponding to a load from this
797 SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
799 ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
800 DAG.getSrcValue(NULL));
801 } else if (ArgValue.Val == 0) {
802 if (MVT::isInteger(ObjectVT))
803 ArgValue = DAG.getConstant(0, ObjectVT);
805 ArgValue = DAG.getConstantFP(0, ObjectVT);
807 ArgValues.push_back(ArgValue);
810 ArgOffset += ArgIncrement; // Move on to the next argument.
813 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
814 // arguments and the arguments after the retaddr has been pushed are aligned.
815 if ((ArgOffset & 7) == 0)
818 VarArgsFrameIndex = 0xAAAAAAA; // fastcc functions can't have varargs.
819 ReturnAddrIndex = 0; // No return address slot generated yet.
820 BytesToPopOnReturn = ArgOffset; // Callee pops all stack arguments.
821 BytesCallerReserves = 0;
823 // Finally, inform the code generator which regs we return values in.
824 switch (getValueType(F.getReturnType())) {
825 default: assert(0 && "Unknown type!");
826 case MVT::isVoid: break;
831 MF.addLiveOut(X86::EAX);
834 MF.addLiveOut(X86::EAX);
835 MF.addLiveOut(X86::EDX);
839 MF.addLiveOut(X86::ST0);
845 std::pair<SDOperand, SDOperand>
846 X86TargetLowering::LowerFastCCCallTo(SDOperand Chain, const Type *RetTy,
847 bool isTailCall, SDOperand Callee,
848 ArgListTy &Args, SelectionDAG &DAG) {
849 // Count how many bytes are to be pushed on the stack.
850 unsigned NumBytes = 0;
852 // Keep track of the number of integer regs passed so far. This can be either
853 // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
855 unsigned NumIntRegs = 0;
857 for (unsigned i = 0, e = Args.size(); i != e; ++i)
858 switch (getValueType(Args[i].second)) {
859 default: assert(0 && "Unknown value type!");
864 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
873 if (NumIntRegs+2 <= FASTCC_NUM_INT_ARGS_INREGS) {
876 } else if (NumIntRegs+1 <= FASTCC_NUM_INT_ARGS_INREGS) {
877 NumIntRegs = FASTCC_NUM_INT_ARGS_INREGS;
888 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
889 // arguments and the arguments after the retaddr has been pushed are aligned.
890 if ((NumBytes & 7) == 0)
893 Chain = DAG.getCALLSEQ_START(Chain,DAG.getConstant(NumBytes, getPointerTy()));
895 // Arguments go on the stack in reverse order, as specified by the ABI.
896 unsigned ArgOffset = 0;
897 SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32);
899 std::vector<SDOperand> Stores;
900 std::vector<SDOperand> RegValuesToPass;
901 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
902 switch (getValueType(Args[i].second)) {
903 default: assert(0 && "Unexpected ValueType for argument!");
905 Args[i].first = DAG.getNode(ISD::ANY_EXTEND, MVT::i8, Args[i].first);
910 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
911 RegValuesToPass.push_back(Args[i].first);
917 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
918 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
919 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
920 Args[i].first, PtrOff,
921 DAG.getSrcValue(NULL)));
926 // Can pass (at least) part of it in regs?
927 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
928 SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
929 Args[i].first, DAG.getConstant(1, MVT::i32));
930 SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
931 Args[i].first, DAG.getConstant(0, MVT::i32));
932 RegValuesToPass.push_back(Lo);
935 // Pass both parts in regs?
936 if (NumIntRegs < FASTCC_NUM_INT_ARGS_INREGS) {
937 RegValuesToPass.push_back(Hi);
940 // Pass the high part in memory.
941 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
942 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
943 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
944 Hi, PtrOff, DAG.getSrcValue(NULL)));
951 SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
952 PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
953 Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
954 Args[i].first, PtrOff,
955 DAG.getSrcValue(NULL)));
961 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
963 // Make sure the instruction takes 8n+4 bytes to make sure the start of the
964 // arguments and the arguments after the retaddr has been pushed are aligned.
965 if ((ArgOffset & 7) == 0)
968 std::vector<MVT::ValueType> RetVals;
969 MVT::ValueType RetTyVT = getValueType(RetTy);
971 RetVals.push_back(MVT::Other);
973 // The result values produced have to be legal. Promote the result.
975 case MVT::isVoid: break;
977 RetVals.push_back(RetTyVT);
982 RetVals.push_back(MVT::i32);
986 RetVals.push_back(MVT::f32);
988 RetVals.push_back(MVT::f64);
991 RetVals.push_back(MVT::i32);
992 RetVals.push_back(MVT::i32);
996 // Build a sequence of copy-to-reg nodes chained together with token chain
997 // and flag operands which copy the outgoing args into registers.
999 for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) {
1001 SDOperand RegToPass = RegValuesToPass[i];
1002 switch (RegToPass.getValueType()) {
1003 default: assert(0 && "Bad thing to pass in regs");
1005 CCReg = (i == 0) ? X86::AL : X86::DL;
1008 CCReg = (i == 0) ? X86::AX : X86::DX;
1011 CCReg = (i == 0) ? X86::EAX : X86::EDX;
1015 Chain = DAG.getCopyToReg(Chain, CCReg, RegToPass, InFlag);
1016 InFlag = Chain.getValue(1);
1019 std::vector<MVT::ValueType> NodeTys;
1020 NodeTys.push_back(MVT::Other); // Returns a chain
1021 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
1022 std::vector<SDOperand> Ops;
1023 Ops.push_back(Chain);
1024 Ops.push_back(Callee);
1026 Ops.push_back(InFlag);
1028 // FIXME: Do not generate X86ISD::TAILCALL for now.
1029 Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
1030 InFlag = Chain.getValue(1);
1033 NodeTys.push_back(MVT::Other); // Returns a chain
1034 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
1036 Ops.push_back(Chain);
1037 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1038 Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1039 Ops.push_back(InFlag);
1040 Chain = DAG.getNode(ISD::CALLSEQ_END, NodeTys, Ops);
1041 InFlag = Chain.getValue(1);
1044 if (RetTyVT != MVT::isVoid) {
1046 default: assert(0 && "Unknown value type to return!");
1049 RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
1050 Chain = RetVal.getValue(1);
1051 if (RetTyVT == MVT::i1)
1052 RetVal = DAG.getNode(ISD::TRUNCATE, MVT::i1, RetVal);
1055 RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
1056 Chain = RetVal.getValue(1);
1059 RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1060 Chain = RetVal.getValue(1);
1063 SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1064 SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
1066 RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
1067 Chain = Hi.getValue(1);
1072 std::vector<MVT::ValueType> Tys;
1073 Tys.push_back(MVT::f64);
1074 Tys.push_back(MVT::Other);
1075 Tys.push_back(MVT::Flag);
1076 std::vector<SDOperand> Ops;
1077 Ops.push_back(Chain);
1078 Ops.push_back(InFlag);
1079 RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
1080 Chain = RetVal.getValue(1);
1081 InFlag = RetVal.getValue(2);
1083 // FIXME: Currently the FST is flagged to the FP_GET_RESULT. This
1084 // shouldn't be necessary except that RFP cannot be live across
1085 // multiple blocks. When stackifier is fixed, they can be uncoupled.
1086 MachineFunction &MF = DAG.getMachineFunction();
1087 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
1088 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1090 Tys.push_back(MVT::Other);
1092 Ops.push_back(Chain);
1093 Ops.push_back(RetVal);
1094 Ops.push_back(StackSlot);
1095 Ops.push_back(DAG.getValueType(RetTyVT));
1096 Ops.push_back(InFlag);
1097 Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
1098 RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
1099 DAG.getSrcValue(NULL));
1100 Chain = RetVal.getValue(1);
1103 if (RetTyVT == MVT::f32 && !X86ScalarSSE)
1104 // FIXME: we would really like to remember that this FP_ROUND
1105 // operation is okay to eliminate if we allow excess FP precision.
1106 RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
1112 return std::make_pair(RetVal, Chain);
1115 SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
1116 if (ReturnAddrIndex == 0) {
1117 // Set up a frame object for the return address.
1118 MachineFunction &MF = DAG.getMachineFunction();
1119 ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4);
1122 return DAG.getFrameIndex(ReturnAddrIndex, MVT::i32);
1127 std::pair<SDOperand, SDOperand> X86TargetLowering::
1128 LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth,
1129 SelectionDAG &DAG) {
1131 if (Depth) // Depths > 0 not supported yet!
1132 Result = DAG.getConstant(0, getPointerTy());
1134 SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
1135 if (!isFrameAddress)
1136 // Just load the return address
1137 Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), RetAddrFI,
1138 DAG.getSrcValue(NULL));
1140 Result = DAG.getNode(ISD::SUB, MVT::i32, RetAddrFI,
1141 DAG.getConstant(4, MVT::i32));
1143 return std::make_pair(Result, Chain);
1146 /// getCondBrOpcodeForX86CC - Returns the X86 conditional branch opcode
1147 /// which corresponds to the condition code.
1148 static unsigned getCondBrOpcodeForX86CC(unsigned X86CC) {
1150 default: assert(0 && "Unknown X86 conditional code!");
1151 case X86ISD::COND_A: return X86::JA;
1152 case X86ISD::COND_AE: return X86::JAE;
1153 case X86ISD::COND_B: return X86::JB;
1154 case X86ISD::COND_BE: return X86::JBE;
1155 case X86ISD::COND_E: return X86::JE;
1156 case X86ISD::COND_G: return X86::JG;
1157 case X86ISD::COND_GE: return X86::JGE;
1158 case X86ISD::COND_L: return X86::JL;
1159 case X86ISD::COND_LE: return X86::JLE;
1160 case X86ISD::COND_NE: return X86::JNE;
1161 case X86ISD::COND_NO: return X86::JNO;
1162 case X86ISD::COND_NP: return X86::JNP;
1163 case X86ISD::COND_NS: return X86::JNS;
1164 case X86ISD::COND_O: return X86::JO;
1165 case X86ISD::COND_P: return X86::JP;
1166 case X86ISD::COND_S: return X86::JS;
1170 /// translateX86CC - do a one to one translation of a ISD::CondCode to the X86
1171 /// specific condition code. It returns a false if it cannot do a direct
1172 /// translation. X86CC is the translated CondCode. Flip is set to true if the
1173 /// the order of comparison operands should be flipped.
1174 static bool translateX86CC(SDOperand CC, bool isFP, unsigned &X86CC,
1176 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1178 X86CC = X86ISD::COND_INVALID;
1180 switch (SetCCOpcode) {
1182 case ISD::SETEQ: X86CC = X86ISD::COND_E; break;
1183 case ISD::SETGT: X86CC = X86ISD::COND_G; break;
1184 case ISD::SETGE: X86CC = X86ISD::COND_GE; break;
1185 case ISD::SETLT: X86CC = X86ISD::COND_L; break;
1186 case ISD::SETLE: X86CC = X86ISD::COND_LE; break;
1187 case ISD::SETNE: X86CC = X86ISD::COND_NE; break;
1188 case ISD::SETULT: X86CC = X86ISD::COND_B; break;
1189 case ISD::SETUGT: X86CC = X86ISD::COND_A; break;
1190 case ISD::SETULE: X86CC = X86ISD::COND_BE; break;
1191 case ISD::SETUGE: X86CC = X86ISD::COND_AE; break;
1194 // On a floating point condition, the flags are set as follows:
1196 // 0 | 0 | 0 | X > Y
1197 // 0 | 0 | 1 | X < Y
1198 // 1 | 0 | 0 | X == Y
1199 // 1 | 1 | 1 | unordered
1200 switch (SetCCOpcode) {
1203 case ISD::SETEQ: X86CC = X86ISD::COND_E; break;
1204 case ISD::SETOLE: Flip = true; // Fallthrough
1206 case ISD::SETGT: X86CC = X86ISD::COND_A; break;
1207 case ISD::SETOLT: Flip = true; // Fallthrough
1209 case ISD::SETGE: X86CC = X86ISD::COND_AE; break;
1210 case ISD::SETUGE: Flip = true; // Fallthrough
1212 case ISD::SETLT: X86CC = X86ISD::COND_B; break;
1213 case ISD::SETUGT: Flip = true; // Fallthrough
1215 case ISD::SETLE: X86CC = X86ISD::COND_BE; break;
1217 case ISD::SETNE: X86CC = X86ISD::COND_NE; break;
1218 case ISD::SETUO: X86CC = X86ISD::COND_P; break;
1219 case ISD::SETO: X86CC = X86ISD::COND_NP; break;
1223 return X86CC != X86ISD::COND_INVALID;
1226 /// hasFPCMov - is there a floating point cmov for the specific X86 condition
1227 /// code. Current x86 isa includes the following FP cmov instructions:
1228 /// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
1229 static bool hasFPCMov(unsigned X86CC) {
1233 case X86ISD::COND_B:
1234 case X86ISD::COND_BE:
1235 case X86ISD::COND_E:
1236 case X86ISD::COND_P:
1237 case X86ISD::COND_A:
1238 case X86ISD::COND_AE:
1239 case X86ISD::COND_NE:
1240 case X86ISD::COND_NP:
1246 X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
1247 MachineBasicBlock *BB) {
1248 switch (MI->getOpcode()) {
1249 default: assert(false && "Unexpected instr type to insert");
1250 case X86::CMOV_FR32:
1251 case X86::CMOV_FR64: {
1252 // To "insert" a SELECT_CC instruction, we actually have to insert the
1253 // diamond control-flow pattern. The incoming instruction knows the
1254 // destination vreg to set, the condition code register to branch on, the
1255 // true/false values to select between, and a branch opcode to use.
1256 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1257 ilist<MachineBasicBlock>::iterator It = BB;
1263 // cmpTY ccX, r1, r2
1265 // fallthrough --> copy0MBB
1266 MachineBasicBlock *thisMBB = BB;
1267 MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
1268 MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
1269 unsigned Opc = getCondBrOpcodeForX86CC(MI->getOperand(3).getImmedValue());
1270 BuildMI(BB, Opc, 1).addMBB(sinkMBB);
1271 MachineFunction *F = BB->getParent();
1272 F->getBasicBlockList().insert(It, copy0MBB);
1273 F->getBasicBlockList().insert(It, sinkMBB);
1274 // Update machine-CFG edges
1275 BB->addSuccessor(copy0MBB);
1276 BB->addSuccessor(sinkMBB);
1279 // %FalseValue = ...
1280 // # fallthrough to sinkMBB
1283 // Update machine-CFG edges
1284 BB->addSuccessor(sinkMBB);
1287 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
1290 BuildMI(BB, X86::PHI, 4, MI->getOperand(0).getReg())
1291 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
1292 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
1294 delete MI; // The pseudo instruction is gone now.
1298 case X86::FP_TO_INT16_IN_MEM:
1299 case X86::FP_TO_INT32_IN_MEM:
1300 case X86::FP_TO_INT64_IN_MEM: {
1301 // Change the floating point control register to use "round towards zero"
1302 // mode when truncating to an integer value.
1303 MachineFunction *F = BB->getParent();
1304 int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2);
1305 addFrameReference(BuildMI(BB, X86::FNSTCW16m, 4), CWFrameIdx);
1307 // Load the old value of the high byte of the control word...
1309 F->getSSARegMap()->createVirtualRegister(X86::R16RegisterClass);
1310 addFrameReference(BuildMI(BB, X86::MOV16rm, 4, OldCW), CWFrameIdx);
1312 // Set the high part to be round to zero...
1313 addFrameReference(BuildMI(BB, X86::MOV16mi, 5), CWFrameIdx).addImm(0xC7F);
1315 // Reload the modified control word now...
1316 addFrameReference(BuildMI(BB, X86::FLDCW16m, 4), CWFrameIdx);
1318 // Restore the memory image of control word to original value
1319 addFrameReference(BuildMI(BB, X86::MOV16mr, 5), CWFrameIdx).addReg(OldCW);
1321 // Get the X86 opcode to use.
1323 switch (MI->getOpcode()) {
1324 default: assert(0 && "illegal opcode!");
1325 case X86::FP_TO_INT16_IN_MEM: Opc = X86::FpIST16m; break;
1326 case X86::FP_TO_INT32_IN_MEM: Opc = X86::FpIST32m; break;
1327 case X86::FP_TO_INT64_IN_MEM: Opc = X86::FpIST64m; break;
1331 MachineOperand &Op = MI->getOperand(0);
1332 if (Op.isRegister()) {
1333 AM.BaseType = X86AddressMode::RegBase;
1334 AM.Base.Reg = Op.getReg();
1336 AM.BaseType = X86AddressMode::FrameIndexBase;
1337 AM.Base.FrameIndex = Op.getFrameIndex();
1339 Op = MI->getOperand(1);
1340 if (Op.isImmediate())
1341 AM.Scale = Op.getImmedValue();
1342 Op = MI->getOperand(2);
1343 if (Op.isImmediate())
1344 AM.IndexReg = Op.getImmedValue();
1345 Op = MI->getOperand(3);
1346 if (Op.isGlobalAddress()) {
1347 AM.GV = Op.getGlobal();
1349 AM.Disp = Op.getImmedValue();
1351 addFullAddress(BuildMI(BB, Opc, 5), AM).addReg(MI->getOperand(4).getReg());
1353 // Reload the original control word now.
1354 addFrameReference(BuildMI(BB, X86::FLDCW16m, 4), CWFrameIdx);
1356 delete MI; // The pseudo instruction is gone now.
1363 //===----------------------------------------------------------------------===//
1364 // X86 Custom Lowering Hooks
1365 //===----------------------------------------------------------------------===//
1367 /// DarwinGVRequiresExtraLoad - true if accessing the GV requires an extra
1368 /// load. For Darwin, external and weak symbols are indirect, loading the value
1369 /// at address GV rather then the value of GV itself. This means that the
1370 /// GlobalAddress must be in the base or index register of the address, not the
1371 /// GV offset field.
1372 static bool DarwinGVRequiresExtraLoad(GlobalValue *GV) {
1373 return (GV->hasWeakLinkage() || GV->hasLinkOnceLinkage() ||
1374 (GV->isExternal() && !GV->hasNotBeenReadFromBytecode()));
1377 /// isPSHUFDMask - Return true if the specified VECTOR_SHUFFLE operand
1378 /// specifies a shuffle of elements that is suitable for input to PSHUFD.
1379 bool X86::isPSHUFDMask(SDNode *N) {
1380 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1382 if (N->getNumOperands() != 4)
1385 // Check if the value doesn't reference the second vector.
1386 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) {
1387 assert(isa<ConstantSDNode>(N->getOperand(i)) &&
1388 "Invalid VECTOR_SHUFFLE mask!");
1389 if (cast<ConstantSDNode>(N->getOperand(i))->getValue() >= 4) return false;
1395 /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
1396 /// specifies a shuffle of elements that is suitable for input to SHUFP*.
1397 bool X86::isSHUFPMask(SDNode *N) {
1398 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1400 unsigned NumOperands = N->getNumOperands();
1401 if (NumOperands != 2 && NumOperands != 4)
1404 // Each half must refer to only one of the vector.
1405 SDOperand Elt = N->getOperand(0);
1406 assert(isa<ConstantSDNode>(Elt) && "Invalid VECTOR_SHUFFLE mask!");
1407 for (unsigned i = 1; i != NumOperands / 2; ++i) {
1408 assert(isa<ConstantSDNode>(N->getOperand(i)) &&
1409 "Invalid VECTOR_SHUFFLE mask!");
1410 if (cast<ConstantSDNode>(N->getOperand(i))->getValue() !=
1411 cast<ConstantSDNode>(Elt)->getValue())
1414 Elt = N->getOperand(NumOperands / 2);
1415 assert(isa<ConstantSDNode>(Elt) && "Invalid VECTOR_SHUFFLE mask!");
1416 for (unsigned i = NumOperands / 2; i != NumOperands; ++i) {
1417 assert(isa<ConstantSDNode>(N->getOperand(i)) &&
1418 "Invalid VECTOR_SHUFFLE mask!");
1419 if (cast<ConstantSDNode>(N->getOperand(i))->getValue() !=
1420 cast<ConstantSDNode>(Elt)->getValue())
1427 /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
1428 /// a splat of a single element.
1429 bool X86::isSplatMask(SDNode *N) {
1430 assert(N->getOpcode() == ISD::BUILD_VECTOR);
1432 // We can only splat 64-bit, and 32-bit quantities.
1433 if (N->getNumOperands() != 4 && N->getNumOperands() != 2)
1436 // This is a splat operation if each element of the permute is the same, and
1437 // if the value doesn't reference the second vector.
1438 SDOperand Elt = N->getOperand(0);
1439 assert(isa<ConstantSDNode>(Elt) && "Invalid VECTOR_SHUFFLE mask!");
1440 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) {
1441 assert(isa<ConstantSDNode>(N->getOperand(i)) &&
1442 "Invalid VECTOR_SHUFFLE mask!");
1443 if (N->getOperand(i) != Elt) return false;
1446 // Make sure it is a splat of the first vector operand.
1447 return cast<ConstantSDNode>(Elt)->getValue() < N->getNumOperands();
1450 /// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
1451 /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP*
1453 unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
1454 unsigned NumOperands = N->getNumOperands();
1455 unsigned Shift = (NumOperands == 4) ? 2 : 1;
1457 unsigned i = NumOperands - 1;
1459 unsigned Val = cast<ConstantSDNode>(N->getOperand(i))->getValue();
1460 if (Val >= NumOperands) Val -= NumOperands;
1469 /// LowerOperation - Provide custom lowering hooks for some operations.
1471 SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
1472 switch (Op.getOpcode()) {
1473 default: assert(0 && "Should not custom lower this!");
1474 case ISD::SHL_PARTS:
1475 case ISD::SRA_PARTS:
1476 case ISD::SRL_PARTS: {
1477 assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 &&
1478 "Not an i64 shift!");
1479 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
1480 SDOperand ShOpLo = Op.getOperand(0);
1481 SDOperand ShOpHi = Op.getOperand(1);
1482 SDOperand ShAmt = Op.getOperand(2);
1483 SDOperand Tmp1 = isSRA ? DAG.getNode(ISD::SRA, MVT::i32, ShOpHi,
1484 DAG.getConstant(31, MVT::i8))
1485 : DAG.getConstant(0, MVT::i32);
1487 SDOperand Tmp2, Tmp3;
1488 if (Op.getOpcode() == ISD::SHL_PARTS) {
1489 Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt);
1490 Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt);
1492 Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt);
1493 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, MVT::i32, ShOpHi, ShAmt);
1496 SDOperand InFlag = DAG.getNode(X86ISD::TEST, MVT::Flag,
1497 ShAmt, DAG.getConstant(32, MVT::i8));
1500 SDOperand CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1502 std::vector<MVT::ValueType> Tys;
1503 Tys.push_back(MVT::i32);
1504 Tys.push_back(MVT::Flag);
1505 std::vector<SDOperand> Ops;
1506 if (Op.getOpcode() == ISD::SHL_PARTS) {
1507 Ops.push_back(Tmp2);
1508 Ops.push_back(Tmp3);
1510 Ops.push_back(InFlag);
1511 Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1512 InFlag = Hi.getValue(1);
1515 Ops.push_back(Tmp3);
1516 Ops.push_back(Tmp1);
1518 Ops.push_back(InFlag);
1519 Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1521 Ops.push_back(Tmp2);
1522 Ops.push_back(Tmp3);
1524 Ops.push_back(InFlag);
1525 Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1526 InFlag = Lo.getValue(1);
1529 Ops.push_back(Tmp3);
1530 Ops.push_back(Tmp1);
1532 Ops.push_back(InFlag);
1533 Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1537 Tys.push_back(MVT::i32);
1538 Tys.push_back(MVT::i32);
1542 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1544 case ISD::SINT_TO_FP: {
1545 assert(Op.getOperand(0).getValueType() <= MVT::i64 &&
1546 Op.getOperand(0).getValueType() >= MVT::i16 &&
1547 "Unknown SINT_TO_FP to lower!");
1550 MVT::ValueType SrcVT = Op.getOperand(0).getValueType();
1551 unsigned Size = MVT::getSizeInBits(SrcVT)/8;
1552 MachineFunction &MF = DAG.getMachineFunction();
1553 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
1554 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1555 SDOperand Chain = DAG.getNode(ISD::STORE, MVT::Other,
1556 DAG.getEntryNode(), Op.getOperand(0),
1557 StackSlot, DAG.getSrcValue(NULL));
1560 std::vector<MVT::ValueType> Tys;
1561 Tys.push_back(MVT::f64);
1562 Tys.push_back(MVT::Other);
1563 if (X86ScalarSSE) Tys.push_back(MVT::Flag);
1564 std::vector<SDOperand> Ops;
1565 Ops.push_back(Chain);
1566 Ops.push_back(StackSlot);
1567 Ops.push_back(DAG.getValueType(SrcVT));
1568 Result = DAG.getNode(X86ScalarSSE ? X86ISD::FILD_FLAG :X86ISD::FILD,
1572 Chain = Result.getValue(1);
1573 SDOperand InFlag = Result.getValue(2);
1575 // FIXME: Currently the FST is flagged to the FILD_FLAG. This
1576 // shouldn't be necessary except that RFP cannot be live across
1577 // multiple blocks. When stackifier is fixed, they can be uncoupled.
1578 MachineFunction &MF = DAG.getMachineFunction();
1579 int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
1580 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1581 std::vector<MVT::ValueType> Tys;
1582 Tys.push_back(MVT::Other);
1583 std::vector<SDOperand> Ops;
1584 Ops.push_back(Chain);
1585 Ops.push_back(Result);
1586 Ops.push_back(StackSlot);
1587 Ops.push_back(DAG.getValueType(Op.getValueType()));
1588 Ops.push_back(InFlag);
1589 Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
1590 Result = DAG.getLoad(Op.getValueType(), Chain, StackSlot,
1591 DAG.getSrcValue(NULL));
1596 case ISD::FP_TO_SINT: {
1597 assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 &&
1598 "Unknown FP_TO_SINT to lower!");
1599 // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
1601 MachineFunction &MF = DAG.getMachineFunction();
1602 unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8;
1603 int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
1604 SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1607 switch (Op.getValueType()) {
1608 default: assert(0 && "Invalid FP_TO_SINT to lower!");
1609 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
1610 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
1611 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
1614 SDOperand Chain = DAG.getEntryNode();
1615 SDOperand Value = Op.getOperand(0);
1617 assert(Op.getValueType() == MVT::i64 && "Invalid FP_TO_SINT to lower!");
1618 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Value, StackSlot,
1619 DAG.getSrcValue(0));
1620 std::vector<MVT::ValueType> Tys;
1621 Tys.push_back(MVT::f64);
1622 Tys.push_back(MVT::Other);
1623 std::vector<SDOperand> Ops;
1624 Ops.push_back(Chain);
1625 Ops.push_back(StackSlot);
1626 Ops.push_back(DAG.getValueType(Op.getOperand(0).getValueType()));
1627 Value = DAG.getNode(X86ISD::FLD, Tys, Ops);
1628 Chain = Value.getValue(1);
1629 SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
1630 StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1633 // Build the FP_TO_INT*_IN_MEM
1634 std::vector<SDOperand> Ops;
1635 Ops.push_back(Chain);
1636 Ops.push_back(Value);
1637 Ops.push_back(StackSlot);
1638 SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops);
1641 return DAG.getLoad(Op.getValueType(), FIST, StackSlot,
1642 DAG.getSrcValue(NULL));
1644 case ISD::READCYCLECOUNTER: {
1645 std::vector<MVT::ValueType> Tys;
1646 Tys.push_back(MVT::Other);
1647 Tys.push_back(MVT::Flag);
1648 std::vector<SDOperand> Ops;
1649 Ops.push_back(Op.getOperand(0));
1650 SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, Ops);
1652 Ops.push_back(DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)));
1653 Ops.push_back(DAG.getCopyFromReg(Ops[0].getValue(1), X86::EDX,
1654 MVT::i32, Ops[0].getValue(2)));
1655 Ops.push_back(Ops[1].getValue(1));
1656 Tys[0] = Tys[1] = MVT::i32;
1657 Tys.push_back(MVT::Other);
1658 return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1661 MVT::ValueType VT = Op.getValueType();
1662 const Type *OpNTy = MVT::getTypeForValueType(VT);
1663 std::vector<Constant*> CV;
1664 if (VT == MVT::f64) {
1665 CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(~(1ULL << 63))));
1666 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1668 CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(~(1U << 31))));
1669 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1670 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1671 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1673 Constant *CS = ConstantStruct::get(CV);
1674 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
1676 = DAG.getNode(X86ISD::LOAD_PACK,
1677 VT, DAG.getEntryNode(), CPIdx, DAG.getSrcValue(NULL));
1678 return DAG.getNode(X86ISD::FAND, VT, Op.getOperand(0), Mask);
1681 MVT::ValueType VT = Op.getValueType();
1682 const Type *OpNTy = MVT::getTypeForValueType(VT);
1683 std::vector<Constant*> CV;
1684 if (VT == MVT::f64) {
1685 CV.push_back(ConstantFP::get(OpNTy, BitsToDouble(1ULL << 63)));
1686 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1688 CV.push_back(ConstantFP::get(OpNTy, BitsToFloat(1U << 31)));
1689 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1690 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1691 CV.push_back(ConstantFP::get(OpNTy, 0.0));
1693 Constant *CS = ConstantStruct::get(CV);
1694 SDOperand CPIdx = DAG.getConstantPool(CS, getPointerTy(), 4);
1696 = DAG.getNode(X86ISD::LOAD_PACK,
1697 VT, DAG.getEntryNode(), CPIdx, DAG.getSrcValue(NULL));
1698 return DAG.getNode(X86ISD::FXOR, VT, Op.getOperand(0), Mask);
1701 assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
1703 SDOperand CC = Op.getOperand(2);
1704 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1705 bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType());
1708 if (translateX86CC(CC, isFP, X86CC, Flip)) {
1710 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1711 Op.getOperand(1), Op.getOperand(0));
1713 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1714 Op.getOperand(0), Op.getOperand(1));
1715 return DAG.getNode(X86ISD::SETCC, MVT::i8,
1716 DAG.getConstant(X86CC, MVT::i8), Cond);
1718 assert(isFP && "Illegal integer SetCC!");
1720 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1721 Op.getOperand(0), Op.getOperand(1));
1722 std::vector<MVT::ValueType> Tys;
1723 std::vector<SDOperand> Ops;
1724 switch (SetCCOpcode) {
1725 default: assert(false && "Illegal floating point SetCC!");
1726 case ISD::SETOEQ: { // !PF & ZF
1727 Tys.push_back(MVT::i8);
1728 Tys.push_back(MVT::Flag);
1729 Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1730 Ops.push_back(Cond);
1731 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1732 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1733 DAG.getConstant(X86ISD::COND_E, MVT::i8),
1735 return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1737 case ISD::SETUNE: { // PF | !ZF
1738 Tys.push_back(MVT::i8);
1739 Tys.push_back(MVT::Flag);
1740 Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1741 Ops.push_back(Cond);
1742 SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1743 SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1744 DAG.getConstant(X86ISD::COND_NE, MVT::i8),
1746 return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1752 MVT::ValueType VT = Op.getValueType();
1753 bool isFP = MVT::isFloatingPoint(VT);
1754 bool isFPStack = isFP && !X86ScalarSSE;
1755 bool isFPSSE = isFP && X86ScalarSSE;
1756 bool addTest = false;
1757 SDOperand Op0 = Op.getOperand(0);
1759 if (Op0.getOpcode() == ISD::SETCC)
1760 Op0 = LowerOperation(Op0, DAG);
1762 if (Op0.getOpcode() == X86ISD::SETCC) {
1763 // If condition flag is set by a X86ISD::CMP, then make a copy of it
1764 // (since flag operand cannot be shared). If the X86ISD::SETCC does not
1765 // have another use it will be eliminated.
1766 // If the X86ISD::SETCC has more than one use, then it's probably better
1767 // to use a test instead of duplicating the X86ISD::CMP (for register
1768 // pressure reason).
1769 if (Op0.getOperand(1).getOpcode() == X86ISD::CMP) {
1770 if (!Op0.hasOneUse()) {
1771 std::vector<MVT::ValueType> Tys;
1772 for (unsigned i = 0; i < Op0.Val->getNumValues(); ++i)
1773 Tys.push_back(Op0.Val->getValueType(i));
1774 std::vector<SDOperand> Ops;
1775 for (unsigned i = 0; i < Op0.getNumOperands(); ++i)
1776 Ops.push_back(Op0.getOperand(i));
1777 Op0 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1780 CC = Op0.getOperand(0);
1781 Cond = Op0.getOperand(1);
1782 // Make a copy as flag result cannot be used by more than one.
1783 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1784 Cond.getOperand(0), Cond.getOperand(1));
1786 isFPStack && !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended());
1793 CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1794 Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Op0, Op0);
1797 std::vector<MVT::ValueType> Tys;
1798 Tys.push_back(Op.getValueType());
1799 Tys.push_back(MVT::Flag);
1800 std::vector<SDOperand> Ops;
1801 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
1802 // condition is true.
1803 Ops.push_back(Op.getOperand(2));
1804 Ops.push_back(Op.getOperand(1));
1806 Ops.push_back(Cond);
1807 return DAG.getNode(X86ISD::CMOV, Tys, Ops);
1810 bool addTest = false;
1811 SDOperand Cond = Op.getOperand(1);
1812 SDOperand Dest = Op.getOperand(2);
1814 if (Cond.getOpcode() == ISD::SETCC)
1815 Cond = LowerOperation(Cond, DAG);
1817 if (Cond.getOpcode() == X86ISD::SETCC) {
1818 // If condition flag is set by a X86ISD::CMP, then make a copy of it
1819 // (since flag operand cannot be shared). If the X86ISD::SETCC does not
1820 // have another use it will be eliminated.
1821 // If the X86ISD::SETCC has more than one use, then it's probably better
1822 // to use a test instead of duplicating the X86ISD::CMP (for register
1823 // pressure reason).
1824 if (Cond.getOperand(1).getOpcode() == X86ISD::CMP) {
1825 if (!Cond.hasOneUse()) {
1826 std::vector<MVT::ValueType> Tys;
1827 for (unsigned i = 0; i < Cond.Val->getNumValues(); ++i)
1828 Tys.push_back(Cond.Val->getValueType(i));
1829 std::vector<SDOperand> Ops;
1830 for (unsigned i = 0; i < Cond.getNumOperands(); ++i)
1831 Ops.push_back(Cond.getOperand(i));
1832 Cond = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1835 CC = Cond.getOperand(0);
1836 Cond = Cond.getOperand(1);
1837 // Make a copy as flag result cannot be used by more than one.
1838 Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1839 Cond.getOperand(0), Cond.getOperand(1));
1846 CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1847 Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Cond, Cond);
1849 return DAG.getNode(X86ISD::BRCOND, Op.getValueType(),
1850 Op.getOperand(0), Op.getOperand(2), CC, Cond);
1853 SDOperand InFlag(0, 0);
1854 SDOperand Chain = Op.getOperand(0);
1856 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1857 if (Align == 0) Align = 1;
1859 ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
1860 // If not DWORD aligned, call memset if size is less than the threshold.
1861 // It knows how to align to the right boundary first.
1862 if ((Align & 3) != 0 ||
1863 (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
1864 MVT::ValueType IntPtr = getPointerTy();
1865 const Type *IntPtrTy = getTargetData().getIntPtrType();
1866 std::vector<std::pair<SDOperand, const Type*> > Args;
1867 Args.push_back(std::make_pair(Op.getOperand(1), IntPtrTy));
1868 // Extend the ubyte argument to be an int value for the call.
1869 SDOperand Val = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Op.getOperand(2));
1870 Args.push_back(std::make_pair(Val, IntPtrTy));
1871 Args.push_back(std::make_pair(Op.getOperand(3), IntPtrTy));
1872 std::pair<SDOperand,SDOperand> CallResult =
1873 LowerCallTo(Chain, Type::VoidTy, false, CallingConv::C, false,
1874 DAG.getExternalSymbol("memset", IntPtr), Args, DAG);
1875 return CallResult.second;
1880 ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1881 unsigned BytesLeft = 0;
1882 bool TwoRepStos = false;
1885 unsigned Val = ValC->getValue() & 255;
1887 // If the value is a constant, then we can potentially use larger sets.
1888 switch (Align & 3) {
1889 case 2: // WORD aligned
1891 Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1892 BytesLeft = I->getValue() % 2;
1893 Val = (Val << 8) | Val;
1896 case 0: // DWORD aligned
1899 Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1900 BytesLeft = I->getValue() % 4;
1902 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1903 DAG.getConstant(2, MVT::i8));
1906 Val = (Val << 8) | Val;
1907 Val = (Val << 16) | Val;
1910 default: // Byte aligned
1912 Count = Op.getOperand(3);
1917 Chain = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
1919 InFlag = Chain.getValue(1);
1922 Count = Op.getOperand(3);
1923 Chain = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
1924 InFlag = Chain.getValue(1);
1927 Chain = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1928 InFlag = Chain.getValue(1);
1929 Chain = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1930 InFlag = Chain.getValue(1);
1932 std::vector<MVT::ValueType> Tys;
1933 Tys.push_back(MVT::Other);
1934 Tys.push_back(MVT::Flag);
1935 std::vector<SDOperand> Ops;
1936 Ops.push_back(Chain);
1937 Ops.push_back(DAG.getValueType(AVT));
1938 Ops.push_back(InFlag);
1939 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, Ops);
1942 InFlag = Chain.getValue(1);
1943 Count = Op.getOperand(3);
1944 MVT::ValueType CVT = Count.getValueType();
1945 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
1946 DAG.getConstant(3, CVT));
1947 Chain = DAG.getCopyToReg(Chain, X86::ECX, Left, InFlag);
1948 InFlag = Chain.getValue(1);
1950 Tys.push_back(MVT::Other);
1951 Tys.push_back(MVT::Flag);
1953 Ops.push_back(Chain);
1954 Ops.push_back(DAG.getValueType(MVT::i8));
1955 Ops.push_back(InFlag);
1956 Chain = DAG.getNode(X86ISD::REP_STOS, Tys, Ops);
1957 } else if (BytesLeft) {
1958 // Issue stores for the last 1 - 3 bytes.
1960 unsigned Val = ValC->getValue() & 255;
1961 unsigned Offset = I->getValue() - BytesLeft;
1962 SDOperand DstAddr = Op.getOperand(1);
1963 MVT::ValueType AddrVT = DstAddr.getValueType();
1964 if (BytesLeft >= 2) {
1965 Value = DAG.getConstant((Val << 8) | Val, MVT::i16);
1966 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Value,
1967 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
1968 DAG.getConstant(Offset, AddrVT)),
1969 DAG.getSrcValue(NULL));
1974 if (BytesLeft == 1) {
1975 Value = DAG.getConstant(Val, MVT::i8);
1976 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Value,
1977 DAG.getNode(ISD::ADD, AddrVT, DstAddr,
1978 DAG.getConstant(Offset, AddrVT)),
1979 DAG.getSrcValue(NULL));
1986 SDOperand Chain = Op.getOperand(0);
1988 (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1989 if (Align == 0) Align = 1;
1991 ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3));
1992 // If not DWORD aligned, call memcpy if size is less than the threshold.
1993 // It knows how to align to the right boundary first.
1994 if ((Align & 3) != 0 ||
1995 (I && I->getValue() < Subtarget->getMinRepStrSizeThreshold())) {
1996 MVT::ValueType IntPtr = getPointerTy();
1997 const Type *IntPtrTy = getTargetData().getIntPtrType();
1998 std::vector<std::pair<SDOperand, const Type*> > Args;
1999 Args.push_back(std::make_pair(Op.getOperand(1), IntPtrTy));
2000 Args.push_back(std::make_pair(Op.getOperand(2), IntPtrTy));
2001 Args.push_back(std::make_pair(Op.getOperand(3), IntPtrTy));
2002 std::pair<SDOperand,SDOperand> CallResult =
2003 LowerCallTo(Chain, Type::VoidTy, false, CallingConv::C, false,
2004 DAG.getExternalSymbol("memcpy", IntPtr), Args, DAG);
2005 return CallResult.second;
2010 unsigned BytesLeft = 0;
2011 bool TwoRepMovs = false;
2012 switch (Align & 3) {
2013 case 2: // WORD aligned
2015 Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
2016 BytesLeft = I->getValue() % 2;
2018 case 0: // DWORD aligned
2021 Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
2022 BytesLeft = I->getValue() % 4;
2024 Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
2025 DAG.getConstant(2, MVT::i8));
2029 default: // Byte aligned
2031 Count = Op.getOperand(3);
2035 SDOperand InFlag(0, 0);
2036 Chain = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
2037 InFlag = Chain.getValue(1);
2038 Chain = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
2039 InFlag = Chain.getValue(1);
2040 Chain = DAG.getCopyToReg(Chain, X86::ESI, Op.getOperand(2), InFlag);
2041 InFlag = Chain.getValue(1);
2043 std::vector<MVT::ValueType> Tys;
2044 Tys.push_back(MVT::Other);
2045 Tys.push_back(MVT::Flag);
2046 std::vector<SDOperand> Ops;
2047 Ops.push_back(Chain);
2048 Ops.push_back(DAG.getValueType(AVT));
2049 Ops.push_back(InFlag);
2050 Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, Ops);
2053 InFlag = Chain.getValue(1);
2054 Count = Op.getOperand(3);
2055 MVT::ValueType CVT = Count.getValueType();
2056 SDOperand Left = DAG.getNode(ISD::AND, CVT, Count,
2057 DAG.getConstant(3, CVT));
2058 Chain = DAG.getCopyToReg(Chain, X86::ECX, Left, InFlag);
2059 InFlag = Chain.getValue(1);
2061 Tys.push_back(MVT::Other);
2062 Tys.push_back(MVT::Flag);
2064 Ops.push_back(Chain);
2065 Ops.push_back(DAG.getValueType(MVT::i8));
2066 Ops.push_back(InFlag);
2067 Chain = DAG.getNode(X86ISD::REP_MOVS, Tys, Ops);
2068 } else if (BytesLeft) {
2069 // Issue loads and stores for the last 1 - 3 bytes.
2070 unsigned Offset = I->getValue() - BytesLeft;
2071 SDOperand DstAddr = Op.getOperand(1);
2072 MVT::ValueType DstVT = DstAddr.getValueType();
2073 SDOperand SrcAddr = Op.getOperand(2);
2074 MVT::ValueType SrcVT = SrcAddr.getValueType();
2076 if (BytesLeft >= 2) {
2077 Value = DAG.getLoad(MVT::i16, Chain,
2078 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
2079 DAG.getConstant(Offset, SrcVT)),
2080 DAG.getSrcValue(NULL));
2081 Chain = Value.getValue(1);
2082 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Value,
2083 DAG.getNode(ISD::ADD, DstVT, DstAddr,
2084 DAG.getConstant(Offset, DstVT)),
2085 DAG.getSrcValue(NULL));
2090 if (BytesLeft == 1) {
2091 Value = DAG.getLoad(MVT::i8, Chain,
2092 DAG.getNode(ISD::ADD, SrcVT, SrcAddr,
2093 DAG.getConstant(Offset, SrcVT)),
2094 DAG.getSrcValue(NULL));
2095 Chain = Value.getValue(1);
2096 Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Value,
2097 DAG.getNode(ISD::ADD, DstVT, DstAddr,
2098 DAG.getConstant(Offset, DstVT)),
2099 DAG.getSrcValue(NULL));
2106 // ConstantPool, GlobalAddress, and ExternalSymbol are lowered as their
2107 // target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
2108 // one of the above mentioned nodes. It has to be wrapped because otherwise
2109 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
2110 // be used to form addressing mode. These wrapped nodes will be selected
2112 case ISD::ConstantPool: {
2113 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
2114 SDOperand Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(),
2115 DAG.getTargetConstantPool(CP->get(), getPointerTy(),
2116 CP->getAlignment()));
2117 if (Subtarget->isTargetDarwin()) {
2118 // With PIC, the address is actually $g + Offset.
2119 if (getTargetMachine().getRelocationModel() == Reloc::PIC)
2120 Result = DAG.getNode(ISD::ADD, getPointerTy(),
2121 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), Result);
2126 case ISD::GlobalAddress: {
2127 GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
2128 SDOperand Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(),
2129 DAG.getTargetGlobalAddress(GV, getPointerTy()));
2130 if (Subtarget->isTargetDarwin()) {
2131 // With PIC, the address is actually $g + Offset.
2132 if (getTargetMachine().getRelocationModel() == Reloc::PIC)
2133 Result = DAG.getNode(ISD::ADD, getPointerTy(),
2134 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), Result);
2136 // For Darwin, external and weak symbols are indirect, so we want to load
2137 // the value at address GV, not the value of GV itself. This means that
2138 // the GlobalAddress must be in the base or index register of the address,
2139 // not the GV offset field.
2140 if (getTargetMachine().getRelocationModel() != Reloc::Static &&
2141 DarwinGVRequiresExtraLoad(GV))
2142 Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(),
2143 Result, DAG.getSrcValue(NULL));
2148 case ISD::ExternalSymbol: {
2149 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
2150 SDOperand Result = DAG.getNode(X86ISD::Wrapper, getPointerTy(),
2151 DAG.getTargetExternalSymbol(Sym, getPointerTy()));
2152 if (Subtarget->isTargetDarwin()) {
2153 // With PIC, the address is actually $g + Offset.
2154 if (getTargetMachine().getRelocationModel() == Reloc::PIC)
2155 Result = DAG.getNode(ISD::ADD, getPointerTy(),
2156 DAG.getNode(X86ISD::GlobalBaseReg, getPointerTy()), Result);
2161 case ISD::VASTART: {
2162 // vastart just stores the address of the VarArgsFrameIndex slot into the
2163 // memory location argument.
2164 // FIXME: Replace MVT::i32 with PointerTy
2165 SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i32);
2166 return DAG.getNode(ISD::STORE, MVT::Other, Op.getOperand(0), FR,
2167 Op.getOperand(1), Op.getOperand(2));
2172 switch(Op.getNumOperands()) {
2174 assert(0 && "Do not know how to return this many arguments!");
2177 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0),
2178 DAG.getConstant(getBytesToPopOnReturn(), MVT::i16));
2180 MVT::ValueType ArgVT = Op.getOperand(1).getValueType();
2181 if (MVT::isInteger(ArgVT))
2182 Copy = DAG.getCopyToReg(Op.getOperand(0), X86::EAX, Op.getOperand(1),
2184 else if (!X86ScalarSSE) {
2185 std::vector<MVT::ValueType> Tys;
2186 Tys.push_back(MVT::Other);
2187 Tys.push_back(MVT::Flag);
2188 std::vector<SDOperand> Ops;
2189 Ops.push_back(Op.getOperand(0));
2190 Ops.push_back(Op.getOperand(1));
2191 Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
2194 SDOperand Chain = Op.getOperand(0);
2195 SDOperand Value = Op.getOperand(1);
2197 if (Value.getOpcode() == ISD::LOAD &&
2198 (Chain == Value.getValue(1) || Chain == Value.getOperand(0))) {
2199 Chain = Value.getOperand(0);
2200 MemLoc = Value.getOperand(1);
2202 // Spill the value to memory and reload it into top of stack.
2203 unsigned Size = MVT::getSizeInBits(ArgVT)/8;
2204 MachineFunction &MF = DAG.getMachineFunction();
2205 int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
2206 MemLoc = DAG.getFrameIndex(SSFI, getPointerTy());
2207 Chain = DAG.getNode(ISD::STORE, MVT::Other, Op.getOperand(0),
2208 Value, MemLoc, DAG.getSrcValue(0));
2210 std::vector<MVT::ValueType> Tys;
2211 Tys.push_back(MVT::f64);
2212 Tys.push_back(MVT::Other);
2213 std::vector<SDOperand> Ops;
2214 Ops.push_back(Chain);
2215 Ops.push_back(MemLoc);
2216 Ops.push_back(DAG.getValueType(ArgVT));
2217 Copy = DAG.getNode(X86ISD::FLD, Tys, Ops);
2219 Tys.push_back(MVT::Other);
2220 Tys.push_back(MVT::Flag);
2222 Ops.push_back(Copy.getValue(1));
2223 Ops.push_back(Copy);
2224 Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
2229 Copy = DAG.getCopyToReg(Op.getOperand(0), X86::EDX, Op.getOperand(2),
2231 Copy = DAG.getCopyToReg(Copy, X86::EAX,Op.getOperand(1),Copy.getValue(1));
2234 return DAG.getNode(X86ISD::RET_FLAG, MVT::Other,
2235 Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16),
2238 case ISD::SCALAR_TO_VECTOR: {
2239 SDOperand AnyExt = DAG.getNode(ISD::ANY_EXTEND, MVT::i32, Op.getOperand(0));
2240 return DAG.getNode(X86ISD::SCALAR_TO_VECTOR, Op.getValueType(), AnyExt);
2242 case ISD::VECTOR_SHUFFLE: {
2243 SDOperand V1 = Op.getOperand(0);
2244 SDOperand V2 = Op.getOperand(1);
2245 SDOperand PermMask = Op.getOperand(2);
2246 MVT::ValueType VT = Op.getValueType();
2248 // Handle splat cases.
2249 if (X86::isSplatMask(PermMask.Val)) {
2250 if (V2.getOpcode() == ISD::UNDEF)
2251 // Leave the VECTOR_SHUFFLE alone. It matches SHUFP*.
2254 // Make it match SHUFP* or UNPCKLPD. Second vector is undef since it's
2256 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
2257 DAG.getNode(ISD::UNDEF, V1.getValueType()),
2259 } else if (X86::isPSHUFDMask(PermMask.Val)) {
2260 if (V2.getOpcode() == ISD::UNDEF)
2261 // Leave the VECTOR_SHUFFLE alone. It matches PSHUFD.
2264 // Make it match PSHUFD. Second vector is undef since it's not needed.
2265 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
2266 DAG.getNode(ISD::UNDEF, V1.getValueType()),
2268 } else if (X86::isSHUFPMask(PermMask.Val)) {
2269 unsigned NumElems = PermMask.getNumOperands();
2270 SDOperand Elt = PermMask.getOperand(0);
2271 if (cast<ConstantSDNode>(Elt)->getValue() >= NumElems) {
2272 // Swap the operands and change mask.
2273 std::vector<SDOperand> MaskVec;
2274 for (unsigned i = NumElems / 2; i != NumElems; ++i)
2275 MaskVec.push_back(PermMask.getOperand(i));
2276 for (unsigned i = 0; i != NumElems / 2; ++i)
2277 MaskVec.push_back(PermMask.getOperand(i));
2279 DAG.getNode(ISD::BUILD_VECTOR, PermMask.getValueType(), MaskVec);
2280 return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, PermMask);
2286 assert(0 && "TODO");
2292 const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
2294 default: return NULL;
2295 case X86ISD::SHLD: return "X86ISD::SHLD";
2296 case X86ISD::SHRD: return "X86ISD::SHRD";
2297 case X86ISD::FAND: return "X86ISD::FAND";
2298 case X86ISD::FXOR: return "X86ISD::FXOR";
2299 case X86ISD::FILD: return "X86ISD::FILD";
2300 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
2301 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
2302 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
2303 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
2304 case X86ISD::FLD: return "X86ISD::FLD";
2305 case X86ISD::FST: return "X86ISD::FST";
2306 case X86ISD::FP_GET_RESULT: return "X86ISD::FP_GET_RESULT";
2307 case X86ISD::FP_SET_RESULT: return "X86ISD::FP_SET_RESULT";
2308 case X86ISD::CALL: return "X86ISD::CALL";
2309 case X86ISD::TAILCALL: return "X86ISD::TAILCALL";
2310 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
2311 case X86ISD::CMP: return "X86ISD::CMP";
2312 case X86ISD::TEST: return "X86ISD::TEST";
2313 case X86ISD::SETCC: return "X86ISD::SETCC";
2314 case X86ISD::CMOV: return "X86ISD::CMOV";
2315 case X86ISD::BRCOND: return "X86ISD::BRCOND";
2316 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
2317 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
2318 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
2319 case X86ISD::LOAD_PACK: return "X86ISD::LOAD_PACK";
2320 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
2321 case X86ISD::Wrapper: return "X86ISD::Wrapper";
2322 case X86ISD::SCALAR_TO_VECTOR: return "X86ISD::SCALAR_TO_VECTOR";
2326 void X86TargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op,
2328 uint64_t &KnownZero,
2330 unsigned Depth) const {
2332 unsigned Opc = Op.getOpcode();
2333 KnownZero = KnownOne = 0; // Don't know anything.
2337 assert(Opc >= ISD::BUILTIN_OP_END && "Expected a target specific node");
2340 KnownZero |= (MVT::getIntVTBitMask(Op.getValueType()) ^ 1ULL);
2345 std::vector<unsigned> X86TargetLowering::
2346 getRegClassForInlineAsmConstraint(const std::string &Constraint,
2347 MVT::ValueType VT) const {
2348 if (Constraint.size() == 1) {
2349 // FIXME: not handling fp-stack yet!
2350 // FIXME: not handling MMX registers yet ('y' constraint).
2351 switch (Constraint[0]) { // GCC X86 Constraint Letters
2352 default: break; // Unknown constriant letter
2353 case 'r': // GENERAL_REGS
2354 case 'R': // LEGACY_REGS
2355 return make_vector<unsigned>(X86::EAX, X86::EBX, X86::ECX, X86::EDX,
2356 X86::ESI, X86::EDI, X86::EBP, X86::ESP, 0);
2357 case 'l': // INDEX_REGS
2358 return make_vector<unsigned>(X86::EAX, X86::EBX, X86::ECX, X86::EDX,
2359 X86::ESI, X86::EDI, X86::EBP, 0);
2360 case 'q': // Q_REGS (GENERAL_REGS in 64-bit mode)
2362 return make_vector<unsigned>(X86::EAX, X86::EBX, X86::ECX, X86::EDX, 0);
2363 case 'x': // SSE_REGS if SSE1 allowed
2364 if (Subtarget->hasSSE1())
2365 return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2366 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7,
2368 return std::vector<unsigned>();
2369 case 'Y': // SSE_REGS if SSE2 allowed
2370 if (Subtarget->hasSSE2())
2371 return make_vector<unsigned>(X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
2372 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7,
2374 return std::vector<unsigned>();
2378 return std::vector<unsigned>();
2381 /// isLegalAddressImmediate - Return true if the integer value or
2382 /// GlobalValue can be used as the offset of the target addressing mode.
2383 bool X86TargetLowering::isLegalAddressImmediate(int64_t V) const {
2384 // X86 allows a sign-extended 32-bit immediate field.
2385 return (V > -(1LL << 32) && V < (1LL << 32)-1);
2388 bool X86TargetLowering::isLegalAddressImmediate(GlobalValue *GV) const {
2389 if (Subtarget->isTargetDarwin()) {
2390 Reloc::Model RModel = getTargetMachine().getRelocationModel();
2391 if (RModel == Reloc::Static)
2393 else if (RModel == Reloc::DynamicNoPIC)
2394 return !DarwinGVRequiresExtraLoad(GV);
2401 /// isShuffleMaskLegal - Targets can use this to indicate that they only
2402 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
2403 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
2404 /// are assumed to be legal.
2406 X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const {
2407 // Only do shuffles on 128-bit vector types for now.
2408 if (MVT::getSizeInBits(VT) == 64) return false;
2409 return (X86::isSplatMask(Mask.Val) ||
2410 X86::isPSHUFDMask(Mask.Val) ||
2411 X86::isSHUFPMask(Mask.Val));