1 //===-- SPUISelLowering.cpp - Cell SPU DAG Lowering Implementation --------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the SPUTargetLowering class.
12 //===----------------------------------------------------------------------===//
14 #include "SPURegisterNames.h"
15 #include "SPUISelLowering.h"
16 #include "SPUTargetMachine.h"
17 #include "llvm/ADT/VectorExtras.h"
18 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
19 #include "llvm/CodeGen/CallingConvLower.h"
20 #include "llvm/CodeGen/MachineFrameInfo.h"
21 #include "llvm/CodeGen/MachineFunction.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/CodeGen/SelectionDAG.h"
25 #include "llvm/Constants.h"
26 #include "llvm/Function.h"
27 #include "llvm/Intrinsics.h"
28 #include "llvm/Support/Debug.h"
29 #include "llvm/Support/MathExtras.h"
30 #include "llvm/Target/TargetOptions.h"
36 // Used in getTargetNodeName() below
38 std::map<unsigned, const char *> node_names;
40 //! MVT::ValueType mapping to useful data for Cell SPU
41 struct valtype_map_s {
42 const MVT::ValueType valtype;
43 const int prefslot_byte;
46 const valtype_map_s valtype_map[] = {
57 const size_t n_valtype_map = sizeof(valtype_map) / sizeof(valtype_map[0]);
59 const valtype_map_s *getValueTypeMapEntry(MVT::ValueType VT) {
60 const valtype_map_s *retval = 0;
62 for (size_t i = 0; i < n_valtype_map; ++i) {
63 if (valtype_map[i].valtype == VT) {
64 retval = valtype_map + i;
71 cerr << "getValueTypeMapEntry returns NULL for "
72 << MVT::getValueTypeString(VT)
81 //! Predicate that returns true if operand is a memory target
83 \arg Op Operand to test
84 \return true if the operand is a memory target (i.e., global
85 address, external symbol, constant pool) or an A-form
88 bool isMemoryOperand(const SDOperand &Op)
90 const unsigned Opc = Op.getOpcode();
91 return (Opc == ISD::GlobalAddress
92 || Opc == ISD::GlobalTLSAddress
93 || Opc == ISD::JumpTable
94 || Opc == ISD::ConstantPool
95 || Opc == ISD::ExternalSymbol
96 || Opc == ISD::TargetGlobalAddress
97 || Opc == ISD::TargetGlobalTLSAddress
98 || Opc == ISD::TargetJumpTable
99 || Opc == ISD::TargetConstantPool
100 || Opc == ISD::TargetExternalSymbol
101 || Opc == SPUISD::AFormAddr);
104 //! Predicate that returns true if the operand is an indirect target
105 bool isIndirectOperand(const SDOperand &Op)
107 const unsigned Opc = Op.getOpcode();
108 return (Opc == ISD::Register
109 || Opc == SPUISD::LDRESULT);
113 SPUTargetLowering::SPUTargetLowering(SPUTargetMachine &TM)
114 : TargetLowering(TM),
117 // Fold away setcc operations if possible.
120 // Use _setjmp/_longjmp instead of setjmp/longjmp.
121 setUseUnderscoreSetJmp(true);
122 setUseUnderscoreLongJmp(true);
124 // Set up the SPU's register classes:
125 // NOTE: i8 register class is not registered because we cannot determine when
126 // we need to zero or sign extend for custom-lowered loads and stores.
127 // NOTE: Ignore the previous note. For now. :-)
128 addRegisterClass(MVT::i8, SPU::R8CRegisterClass);
129 addRegisterClass(MVT::i16, SPU::R16CRegisterClass);
130 addRegisterClass(MVT::i32, SPU::R32CRegisterClass);
131 addRegisterClass(MVT::i64, SPU::R64CRegisterClass);
132 addRegisterClass(MVT::f32, SPU::R32FPRegisterClass);
133 addRegisterClass(MVT::f64, SPU::R64FPRegisterClass);
134 addRegisterClass(MVT::i128, SPU::GPRCRegisterClass);
136 // SPU has no sign or zero extended loads for i1, i8, i16:
137 setLoadXAction(ISD::EXTLOAD, MVT::i1, Promote);
138 setLoadXAction(ISD::SEXTLOAD, MVT::i1, Promote);
139 setLoadXAction(ISD::ZEXTLOAD, MVT::i1, Promote);
140 setTruncStoreAction(MVT::i8, MVT::i1, Custom);
141 setTruncStoreAction(MVT::i16, MVT::i1, Custom);
142 setTruncStoreAction(MVT::i32, MVT::i1, Custom);
143 setTruncStoreAction(MVT::i64, MVT::i1, Custom);
144 setTruncStoreAction(MVT::i128, MVT::i1, Custom);
146 setLoadXAction(ISD::EXTLOAD, MVT::i8, Custom);
147 setLoadXAction(ISD::SEXTLOAD, MVT::i8, Custom);
148 setLoadXAction(ISD::ZEXTLOAD, MVT::i8, Custom);
149 setTruncStoreAction(MVT::i8 , MVT::i8, Custom);
150 setTruncStoreAction(MVT::i16 , MVT::i8, Custom);
151 setTruncStoreAction(MVT::i32 , MVT::i8, Custom);
152 setTruncStoreAction(MVT::i64 , MVT::i8, Custom);
153 setTruncStoreAction(MVT::i128, MVT::i8, Custom);
155 setLoadXAction(ISD::EXTLOAD, MVT::i16, Custom);
156 setLoadXAction(ISD::SEXTLOAD, MVT::i16, Custom);
157 setLoadXAction(ISD::ZEXTLOAD, MVT::i16, Custom);
159 // SPU constant load actions are custom lowered:
160 setOperationAction(ISD::Constant, MVT::i64, Custom);
161 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
162 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
164 // SPU's loads and stores have to be custom lowered:
165 for (unsigned sctype = (unsigned) MVT::i1; sctype < (unsigned) MVT::f128;
167 setOperationAction(ISD::LOAD, sctype, Custom);
168 setOperationAction(ISD::STORE, sctype, Custom);
171 // Custom lower BRCOND for i1, i8 to "promote" the result to
172 // i32 and i16, respectively.
173 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
175 // Expand the jumptable branches
176 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
177 setOperationAction(ISD::BR_CC, MVT::Other, Expand);
178 setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
180 // SPU has no intrinsics for these particular operations:
181 setOperationAction(ISD::MEMMOVE, MVT::Other, Expand);
182 setOperationAction(ISD::MEMSET, MVT::Other, Expand);
183 setOperationAction(ISD::MEMCPY, MVT::Other, Expand);
184 setOperationAction(ISD::MEMBARRIER, MVT::Other, Expand);
186 // PowerPC has no SREM/UREM instructions
187 setOperationAction(ISD::SREM, MVT::i32, Expand);
188 setOperationAction(ISD::UREM, MVT::i32, Expand);
189 setOperationAction(ISD::SREM, MVT::i64, Expand);
190 setOperationAction(ISD::UREM, MVT::i64, Expand);
192 // We don't support sin/cos/sqrt/fmod
193 setOperationAction(ISD::FSIN , MVT::f64, Expand);
194 setOperationAction(ISD::FCOS , MVT::f64, Expand);
195 setOperationAction(ISD::FREM , MVT::f64, Expand);
196 setOperationAction(ISD::FSIN , MVT::f32, Expand);
197 setOperationAction(ISD::FCOS , MVT::f32, Expand);
198 setOperationAction(ISD::FREM , MVT::f32, Expand);
200 // If we're enabling GP optimizations, use hardware square root
201 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
202 setOperationAction(ISD::FSQRT, MVT::f32, Expand);
204 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
205 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
207 // SPU can do rotate right and left, so legalize it... but customize for i8
208 // because instructions don't exist.
209 setOperationAction(ISD::ROTR, MVT::i32, Legal);
210 setOperationAction(ISD::ROTR, MVT::i16, Legal);
211 setOperationAction(ISD::ROTR, MVT::i8, Custom);
212 setOperationAction(ISD::ROTL, MVT::i32, Legal);
213 setOperationAction(ISD::ROTL, MVT::i16, Legal);
214 setOperationAction(ISD::ROTL, MVT::i8, Custom);
215 // SPU has no native version of shift left/right for i8
216 setOperationAction(ISD::SHL, MVT::i8, Custom);
217 setOperationAction(ISD::SRL, MVT::i8, Custom);
218 setOperationAction(ISD::SRA, MVT::i8, Custom);
219 // And SPU needs custom lowering for shift left/right for i64
220 setOperationAction(ISD::SHL, MVT::i64, Custom);
221 setOperationAction(ISD::SRL, MVT::i64, Custom);
222 setOperationAction(ISD::SRA, MVT::i64, Custom);
224 // Custom lower i32 multiplications
225 setOperationAction(ISD::MUL, MVT::i32, Custom);
227 // Need to custom handle (some) common i8 math ops
228 setOperationAction(ISD::SUB, MVT::i8, Custom);
229 setOperationAction(ISD::MUL, MVT::i8, Custom);
231 // SPU does not have BSWAP. It does have i32 support CTLZ.
232 // CTPOP has to be custom lowered.
233 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
234 setOperationAction(ISD::BSWAP, MVT::i64, Expand);
236 setOperationAction(ISD::CTPOP, MVT::i8, Custom);
237 setOperationAction(ISD::CTPOP, MVT::i16, Custom);
238 setOperationAction(ISD::CTPOP, MVT::i32, Custom);
239 setOperationAction(ISD::CTPOP, MVT::i64, Custom);
241 setOperationAction(ISD::CTTZ , MVT::i32, Expand);
242 setOperationAction(ISD::CTTZ , MVT::i64, Expand);
244 setOperationAction(ISD::CTLZ , MVT::i32, Legal);
246 // SPU has a version of select
247 setOperationAction(ISD::SELECT, MVT::i1, Promote);
248 setOperationAction(ISD::SELECT, MVT::i8, Legal);
249 setOperationAction(ISD::SELECT, MVT::i16, Legal);
250 setOperationAction(ISD::SELECT, MVT::i32, Legal);
251 setOperationAction(ISD::SELECT, MVT::i64, Expand);
252 setOperationAction(ISD::SELECT, MVT::f32, Expand);
253 setOperationAction(ISD::SELECT, MVT::f64, Expand);
255 setOperationAction(ISD::SETCC, MVT::i1, Promote);
256 setOperationAction(ISD::SETCC, MVT::i8, Legal);
257 setOperationAction(ISD::SETCC, MVT::i16, Legal);
258 setOperationAction(ISD::SETCC, MVT::i32, Legal);
259 setOperationAction(ISD::SETCC, MVT::i64, Expand);
260 setOperationAction(ISD::SETCC, MVT::f32, Expand);
261 setOperationAction(ISD::SETCC, MVT::f64, Expand);
263 // Zero extension and sign extension for i64 have to be
265 setOperationAction(ISD::ZERO_EXTEND, MVT::i64, Custom);
266 setOperationAction(ISD::SIGN_EXTEND, MVT::i64, Custom);
267 setOperationAction(ISD::ANY_EXTEND, MVT::i64, Custom);
269 // SPU has a legal FP -> signed INT instruction
270 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
271 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
272 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
273 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
275 // FDIV on SPU requires custom lowering
276 setOperationAction(ISD::FDIV, MVT::f32, Custom);
277 //setOperationAction(ISD::FDIV, MVT::f64, Custom);
279 // SPU has [U|S]INT_TO_FP
280 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
281 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
282 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
283 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
284 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
285 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
286 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
287 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
289 setOperationAction(ISD::BIT_CONVERT, MVT::i32, Legal);
290 setOperationAction(ISD::BIT_CONVERT, MVT::f32, Legal);
291 setOperationAction(ISD::BIT_CONVERT, MVT::i64, Legal);
292 setOperationAction(ISD::BIT_CONVERT, MVT::f64, Legal);
294 // We cannot sextinreg(i1). Expand to shifts.
295 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
297 // Support label based line numbers.
298 setOperationAction(ISD::LOCATION, MVT::Other, Expand);
299 setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
301 // We want to legalize GlobalAddress and ConstantPool nodes into the
302 // appropriate instructions to materialize the address.
303 for (unsigned sctype = (unsigned) MVT::i1; sctype < (unsigned) MVT::f128;
305 setOperationAction(ISD::GlobalAddress, sctype, Custom);
306 setOperationAction(ISD::ConstantPool, sctype, Custom);
307 setOperationAction(ISD::JumpTable, sctype, Custom);
310 // RET must be custom lowered, to meet ABI requirements
311 setOperationAction(ISD::RET, MVT::Other, Custom);
313 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
314 setOperationAction(ISD::VASTART , MVT::Other, Custom);
316 // Use the default implementation.
317 setOperationAction(ISD::VAARG , MVT::Other, Expand);
318 setOperationAction(ISD::VACOPY , MVT::Other, Expand);
319 setOperationAction(ISD::VAEND , MVT::Other, Expand);
320 setOperationAction(ISD::STACKSAVE , MVT::Other, Expand);
321 setOperationAction(ISD::STACKRESTORE , MVT::Other, Expand);
322 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Expand);
323 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Expand);
325 // Cell SPU has instructions for converting between i64 and fp.
326 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
327 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
329 // To take advantage of the above i64 FP_TO_SINT, promote i32 FP_TO_UINT
330 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Promote);
332 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
333 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
335 // First set operation action for all vector types to expand. Then we
336 // will selectively turn on ones that can be effectively codegen'd.
337 addRegisterClass(MVT::v16i8, SPU::VECREGRegisterClass);
338 addRegisterClass(MVT::v8i16, SPU::VECREGRegisterClass);
339 addRegisterClass(MVT::v4i32, SPU::VECREGRegisterClass);
340 addRegisterClass(MVT::v2i64, SPU::VECREGRegisterClass);
341 addRegisterClass(MVT::v4f32, SPU::VECREGRegisterClass);
342 addRegisterClass(MVT::v2f64, SPU::VECREGRegisterClass);
344 for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
345 VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) {
346 // add/sub are legal for all supported vector VT's.
347 setOperationAction(ISD::ADD , (MVT::ValueType)VT, Legal);
348 setOperationAction(ISD::SUB , (MVT::ValueType)VT, Legal);
349 // mul has to be custom lowered.
350 setOperationAction(ISD::MUL , (MVT::ValueType)VT, Custom);
352 setOperationAction(ISD::AND , (MVT::ValueType)VT, Legal);
353 setOperationAction(ISD::OR , (MVT::ValueType)VT, Legal);
354 setOperationAction(ISD::XOR , (MVT::ValueType)VT, Legal);
355 setOperationAction(ISD::LOAD , (MVT::ValueType)VT, Legal);
356 setOperationAction(ISD::SELECT, (MVT::ValueType)VT, Legal);
357 setOperationAction(ISD::STORE, (MVT::ValueType)VT, Legal);
359 // These operations need to be expanded:
360 setOperationAction(ISD::SDIV, (MVT::ValueType)VT, Expand);
361 setOperationAction(ISD::SREM, (MVT::ValueType)VT, Expand);
362 setOperationAction(ISD::UDIV, (MVT::ValueType)VT, Expand);
363 setOperationAction(ISD::UREM, (MVT::ValueType)VT, Expand);
364 setOperationAction(ISD::FDIV, (MVT::ValueType)VT, Custom);
366 // Custom lower build_vector, constant pool spills, insert and
367 // extract vector elements:
368 setOperationAction(ISD::BUILD_VECTOR, (MVT::ValueType)VT, Custom);
369 setOperationAction(ISD::ConstantPool, (MVT::ValueType)VT, Custom);
370 setOperationAction(ISD::SCALAR_TO_VECTOR, (MVT::ValueType)VT, Custom);
371 setOperationAction(ISD::EXTRACT_VECTOR_ELT, (MVT::ValueType)VT, Custom);
372 setOperationAction(ISD::INSERT_VECTOR_ELT, (MVT::ValueType)VT, Custom);
373 setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::ValueType)VT, Custom);
376 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
377 setOperationAction(ISD::AND, MVT::v16i8, Custom);
378 setOperationAction(ISD::OR, MVT::v16i8, Custom);
379 setOperationAction(ISD::XOR, MVT::v16i8, Custom);
380 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
382 setShiftAmountType(MVT::i32);
383 setSetCCResultContents(ZeroOrOneSetCCResult);
385 setStackPointerRegisterToSaveRestore(SPU::R1);
387 // We have target-specific dag combine patterns for the following nodes:
388 setTargetDAGCombine(ISD::ADD);
389 setTargetDAGCombine(ISD::ZERO_EXTEND);
390 setTargetDAGCombine(ISD::SIGN_EXTEND);
391 setTargetDAGCombine(ISD::ANY_EXTEND);
393 computeRegisterProperties();
397 SPUTargetLowering::getTargetNodeName(unsigned Opcode) const
399 if (node_names.empty()) {
400 node_names[(unsigned) SPUISD::RET_FLAG] = "SPUISD::RET_FLAG";
401 node_names[(unsigned) SPUISD::Hi] = "SPUISD::Hi";
402 node_names[(unsigned) SPUISD::Lo] = "SPUISD::Lo";
403 node_names[(unsigned) SPUISD::PCRelAddr] = "SPUISD::PCRelAddr";
404 node_names[(unsigned) SPUISD::AFormAddr] = "SPUISD::AFormAddr";
405 node_names[(unsigned) SPUISD::IndirectAddr] = "SPUISD::IndirectAddr";
406 node_names[(unsigned) SPUISD::LDRESULT] = "SPUISD::LDRESULT";
407 node_names[(unsigned) SPUISD::CALL] = "SPUISD::CALL";
408 node_names[(unsigned) SPUISD::SHUFB] = "SPUISD::SHUFB";
409 node_names[(unsigned) SPUISD::INSERT_MASK] = "SPUISD::INSERT_MASK";
410 node_names[(unsigned) SPUISD::CNTB] = "SPUISD::CNTB";
411 node_names[(unsigned) SPUISD::PROMOTE_SCALAR] = "SPUISD::PROMOTE_SCALAR";
412 node_names[(unsigned) SPUISD::EXTRACT_ELT0] = "SPUISD::EXTRACT_ELT0";
413 node_names[(unsigned) SPUISD::EXTRACT_ELT0_CHAINED] = "SPUISD::EXTRACT_ELT0_CHAINED";
414 node_names[(unsigned) SPUISD::EXTRACT_I1_ZEXT] = "SPUISD::EXTRACT_I1_ZEXT";
415 node_names[(unsigned) SPUISD::EXTRACT_I1_SEXT] = "SPUISD::EXTRACT_I1_SEXT";
416 node_names[(unsigned) SPUISD::EXTRACT_I8_ZEXT] = "SPUISD::EXTRACT_I8_ZEXT";
417 node_names[(unsigned) SPUISD::EXTRACT_I8_SEXT] = "SPUISD::EXTRACT_I8_SEXT";
418 node_names[(unsigned) SPUISD::MPY] = "SPUISD::MPY";
419 node_names[(unsigned) SPUISD::MPYU] = "SPUISD::MPYU";
420 node_names[(unsigned) SPUISD::MPYH] = "SPUISD::MPYH";
421 node_names[(unsigned) SPUISD::MPYHH] = "SPUISD::MPYHH";
422 node_names[(unsigned) SPUISD::SHLQUAD_L_BITS] = "SPUISD::SHLQUAD_L_BITS";
423 node_names[(unsigned) SPUISD::SHLQUAD_L_BYTES] = "SPUISD::SHLQUAD_L_BYTES";
424 node_names[(unsigned) SPUISD::VEC_SHL] = "SPUISD::VEC_SHL";
425 node_names[(unsigned) SPUISD::VEC_SRL] = "SPUISD::VEC_SRL";
426 node_names[(unsigned) SPUISD::VEC_SRA] = "SPUISD::VEC_SRA";
427 node_names[(unsigned) SPUISD::VEC_ROTL] = "SPUISD::VEC_ROTL";
428 node_names[(unsigned) SPUISD::VEC_ROTR] = "SPUISD::VEC_ROTR";
429 node_names[(unsigned) SPUISD::ROTQUAD_RZ_BYTES] =
430 "SPUISD::ROTQUAD_RZ_BYTES";
431 node_names[(unsigned) SPUISD::ROTQUAD_RZ_BITS] =
432 "SPUISD::ROTQUAD_RZ_BITS";
433 node_names[(unsigned) SPUISD::ROTBYTES_RIGHT_S] =
434 "SPUISD::ROTBYTES_RIGHT_S";
435 node_names[(unsigned) SPUISD::ROTBYTES_LEFT] = "SPUISD::ROTBYTES_LEFT";
436 node_names[(unsigned) SPUISD::ROTBYTES_LEFT_CHAINED] =
437 "SPUISD::ROTBYTES_LEFT_CHAINED";
438 node_names[(unsigned) SPUISD::FSMBI] = "SPUISD::FSMBI";
439 node_names[(unsigned) SPUISD::SELB] = "SPUISD::SELB";
440 node_names[(unsigned) SPUISD::FPInterp] = "SPUISD::FPInterp";
441 node_names[(unsigned) SPUISD::FPRecipEst] = "SPUISD::FPRecipEst";
442 node_names[(unsigned) SPUISD::SEXT32TO64] = "SPUISD::SEXT32TO64";
445 std::map<unsigned, const char *>::iterator i = node_names.find(Opcode);
447 return ((i != node_names.end()) ? i->second : 0);
451 SPUTargetLowering::getSetCCResultType(const SDOperand &Op) const {
452 return Op.getValueType();
455 //===----------------------------------------------------------------------===//
456 // Calling convention code:
457 //===----------------------------------------------------------------------===//
459 #include "SPUGenCallingConv.inc"
461 //===----------------------------------------------------------------------===//
462 // LowerOperation implementation
463 //===----------------------------------------------------------------------===//
465 /// Aligned load common code for CellSPU
467 \param[in] Op The SelectionDAG load or store operand
468 \param[in] DAG The selection DAG
469 \param[in] ST CellSPU subtarget information structure
470 \param[in,out] alignment Caller initializes this to the load or store node's
471 value from getAlignment(), may be updated while generating the aligned load
472 \param[in,out] alignOffs Aligned offset; set by AlignedLoad to the aligned
473 offset (divisible by 16, modulo 16 == 0)
474 \param[in,out] prefSlotOffs Preferred slot offset; set by AlignedLoad to the
475 offset of the preferred slot (modulo 16 != 0)
476 \param[in,out] VT Caller initializes this value type to the the load or store
477 node's loaded or stored value type; may be updated if an i1-extended load or
479 \param[out] was16aligned true if the base pointer had 16-byte alignment,
480 otherwise false. Can help to determine if the chunk needs to be rotated.
482 Both load and store lowering load a block of data aligned on a 16-byte
483 boundary. This is the common aligned load code shared between both.
486 AlignedLoad(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST,
488 unsigned &alignment, int &alignOffs, int &prefSlotOffs,
489 MVT::ValueType &VT, bool &was16aligned)
491 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
492 const valtype_map_s *vtm = getValueTypeMapEntry(VT);
493 SDOperand basePtr = LSN->getBasePtr();
494 SDOperand chain = LSN->getChain();
496 if (basePtr.getOpcode() == ISD::ADD) {
497 SDOperand Op1 = basePtr.Val->getOperand(1);
499 if (Op1.getOpcode() == ISD::Constant || Op1.getOpcode() == ISD::TargetConstant) {
500 const ConstantSDNode *CN = cast<ConstantSDNode>(basePtr.getOperand(1));
502 alignOffs = (int) CN->getValue();
503 prefSlotOffs = (int) (alignOffs & 0xf);
505 // Adjust the rotation amount to ensure that the final result ends up in
506 // the preferred slot:
507 prefSlotOffs -= vtm->prefslot_byte;
508 basePtr = basePtr.getOperand(0);
510 // Loading from memory, can we adjust alignment?
511 if (basePtr.getOpcode() == SPUISD::AFormAddr) {
512 SDOperand APtr = basePtr.getOperand(0);
513 if (APtr.getOpcode() == ISD::TargetGlobalAddress) {
514 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(APtr);
515 alignment = GSDN->getGlobal()->getAlignment();
520 prefSlotOffs = -vtm->prefslot_byte;
524 prefSlotOffs = -vtm->prefslot_byte;
527 if (alignment == 16) {
528 // Realign the base pointer as a D-Form address:
529 if (!isMemoryOperand(basePtr) || (alignOffs & ~0xf) != 0) {
530 basePtr = DAG.getNode(ISD::ADD, PtrVT,
532 DAG.getConstant((alignOffs & ~0xf), PtrVT));
535 // Emit the vector load:
537 return DAG.getLoad(MVT::v16i8, chain, basePtr,
538 LSN->getSrcValue(), LSN->getSrcValueOffset(),
539 LSN->isVolatile(), 16);
542 // Unaligned load or we're using the "large memory" model, which means that
543 // we have to be very pessimistic:
544 if (isMemoryOperand(basePtr) || isIndirectOperand(basePtr)) {
545 basePtr = DAG.getNode(SPUISD::IndirectAddr, PtrVT, basePtr, DAG.getConstant(0, PtrVT));
549 basePtr = DAG.getNode(ISD::ADD, PtrVT, basePtr,
550 DAG.getConstant((alignOffs & ~0xf), PtrVT));
551 was16aligned = false;
552 return DAG.getLoad(MVT::v16i8, chain, basePtr,
553 LSN->getSrcValue(), LSN->getSrcValueOffset(),
554 LSN->isVolatile(), 16);
557 /// Custom lower loads for CellSPU
559 All CellSPU loads and stores are aligned to 16-byte boundaries, so for elements
560 within a 16-byte block, we have to rotate to extract the requested element.
563 LowerLOAD(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
564 LoadSDNode *LN = cast<LoadSDNode>(Op);
565 SDOperand the_chain = LN->getChain();
566 MVT::ValueType VT = LN->getMemoryVT();
567 MVT::ValueType OpVT = Op.Val->getValueType(0);
568 ISD::LoadExtType ExtType = LN->getExtensionType();
569 unsigned alignment = LN->getAlignment();
572 switch (LN->getAddressingMode()) {
573 case ISD::UNINDEXED: {
577 AlignedLoad(Op, DAG, ST, LN,alignment, offset, rotamt, VT, was16aligned);
582 the_chain = result.getValue(1);
583 // Rotate the chunk if necessary
586 if (rotamt != 0 || !was16aligned) {
587 SDVTList vecvts = DAG.getVTList(MVT::v16i8, MVT::Other);
592 Ops[2] = DAG.getConstant(rotamt, MVT::i16);
594 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
595 LoadSDNode *LN1 = cast<LoadSDNode>(result);
596 Ops[2] = DAG.getNode(ISD::ADD, PtrVT, LN1->getBasePtr(),
597 DAG.getConstant(rotamt, PtrVT));
600 result = DAG.getNode(SPUISD::ROTBYTES_LEFT_CHAINED, vecvts, Ops, 3);
601 the_chain = result.getValue(1);
604 if (VT == OpVT || ExtType == ISD::EXTLOAD) {
606 MVT::ValueType vecVT = MVT::v16i8;
608 // Convert the loaded v16i8 vector to the appropriate vector type
609 // specified by the operand:
612 vecVT = MVT::getVectorType(VT, (128 / MVT::getSizeInBits(VT)));
614 vecVT = MVT::getVectorType(OpVT, (128 / MVT::getSizeInBits(OpVT)));
617 Ops[1] = DAG.getNode(ISD::BIT_CONVERT, vecVT, result);
618 scalarvts = DAG.getVTList((OpVT == VT ? VT : OpVT), MVT::Other);
619 result = DAG.getNode(SPUISD::EXTRACT_ELT0_CHAINED, scalarvts, Ops, 2);
620 the_chain = result.getValue(1);
622 // Handle the sign and zero-extending loads for i1 and i8:
625 if (ExtType == ISD::SEXTLOAD) {
626 NewOpC = (OpVT == MVT::i1
627 ? SPUISD::EXTRACT_I1_SEXT
628 : SPUISD::EXTRACT_I8_SEXT);
630 assert(ExtType == ISD::ZEXTLOAD);
631 NewOpC = (OpVT == MVT::i1
632 ? SPUISD::EXTRACT_I1_ZEXT
633 : SPUISD::EXTRACT_I8_ZEXT);
636 result = DAG.getNode(NewOpC, OpVT, result);
639 SDVTList retvts = DAG.getVTList(OpVT, MVT::Other);
640 SDOperand retops[2] = {
645 result = DAG.getNode(SPUISD::LDRESULT, retvts,
646 retops, sizeof(retops) / sizeof(retops[0]));
653 case ISD::LAST_INDEXED_MODE:
654 cerr << "LowerLOAD: Got a LoadSDNode with an addr mode other than "
656 cerr << (unsigned) LN->getAddressingMode() << "\n";
664 /// Custom lower stores for CellSPU
666 All CellSPU stores are aligned to 16-byte boundaries, so for elements
667 within a 16-byte block, we have to generate a shuffle to insert the
668 requested element into its place, then store the resulting block.
671 LowerSTORE(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
672 StoreSDNode *SN = cast<StoreSDNode>(Op);
673 SDOperand Value = SN->getValue();
674 MVT::ValueType VT = Value.getValueType();
675 MVT::ValueType StVT = (!SN->isTruncatingStore() ? VT : SN->getMemoryVT());
676 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
677 unsigned alignment = SN->getAlignment();
679 switch (SN->getAddressingMode()) {
680 case ISD::UNINDEXED: {
681 int chunk_offset, slot_offset;
684 // The vector type we really want to load from the 16-byte chunk, except
685 // in the case of MVT::i1, which has to be v16i8.
686 unsigned vecVT, stVecVT = MVT::v16i8;
689 stVecVT = MVT::getVectorType(StVT, (128 / MVT::getSizeInBits(StVT)));
690 vecVT = MVT::getVectorType(VT, (128 / MVT::getSizeInBits(VT)));
692 SDOperand alignLoadVec =
693 AlignedLoad(Op, DAG, ST, SN, alignment,
694 chunk_offset, slot_offset, VT, was16aligned);
696 if (alignLoadVec.Val == 0)
699 LoadSDNode *LN = cast<LoadSDNode>(alignLoadVec);
700 SDOperand basePtr = LN->getBasePtr();
701 SDOperand the_chain = alignLoadVec.getValue(1);
702 SDOperand theValue = SN->getValue();
706 && (theValue.getOpcode() == ISD::AssertZext
707 || theValue.getOpcode() == ISD::AssertSext)) {
708 // Drill down and get the value for zero- and sign-extended
710 theValue = theValue.getOperand(0);
715 SDOperand insertEltOffs = DAG.getConstant(chunk_offset, PtrVT);
716 SDOperand insertEltPtr;
717 SDOperand insertEltOp;
719 // If the base pointer is already a D-form address, then just create
720 // a new D-form address with a slot offset and the orignal base pointer.
721 // Otherwise generate a D-form address with the slot offset relative
722 // to the stack pointer, which is always aligned.
723 DEBUG(cerr << "CellSPU LowerSTORE: basePtr = ");
724 DEBUG(basePtr.Val->dump(&DAG));
727 if (basePtr.getOpcode() == SPUISD::IndirectAddr ||
728 (basePtr.getOpcode() == ISD::ADD
729 && basePtr.getOperand(0).getOpcode() == SPUISD::IndirectAddr)) {
730 insertEltPtr = basePtr;
732 insertEltPtr = DAG.getNode(ISD::ADD, PtrVT, basePtr, insertEltOffs);
735 insertEltOp = DAG.getNode(SPUISD::INSERT_MASK, stVecVT, insertEltPtr);
736 result = DAG.getNode(SPUISD::SHUFB, vecVT,
737 DAG.getNode(ISD::SCALAR_TO_VECTOR, vecVT, theValue),
739 DAG.getNode(ISD::BIT_CONVERT, vecVT, insertEltOp));
741 result = DAG.getStore(the_chain, result, basePtr,
742 LN->getSrcValue(), LN->getSrcValueOffset(),
743 LN->isVolatile(), LN->getAlignment());
752 case ISD::LAST_INDEXED_MODE:
753 cerr << "LowerLOAD: Got a LoadSDNode with an addr mode other than "
755 cerr << (unsigned) SN->getAddressingMode() << "\n";
763 /// Generate the address of a constant pool entry.
765 LowerConstantPool(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
766 MVT::ValueType PtrVT = Op.getValueType();
767 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
768 Constant *C = CP->getConstVal();
769 SDOperand CPI = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment());
770 SDOperand Zero = DAG.getConstant(0, PtrVT);
771 const TargetMachine &TM = DAG.getTarget();
773 if (TM.getRelocationModel() == Reloc::Static) {
774 if (!ST->usingLargeMem()) {
775 // Just return the SDOperand with the constant pool address in it.
776 return DAG.getNode(SPUISD::AFormAddr, PtrVT, CPI, Zero);
778 SDOperand Hi = DAG.getNode(SPUISD::Hi, PtrVT, CPI, Zero);
779 SDOperand Lo = DAG.getNode(SPUISD::Lo, PtrVT, CPI, Zero);
780 return DAG.getNode(SPUISD::IndirectAddr, PtrVT, Hi, Lo);
785 "LowerConstantPool: Relocation model other than static not supported.");
790 LowerJumpTable(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
791 MVT::ValueType PtrVT = Op.getValueType();
792 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
793 SDOperand JTI = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
794 SDOperand Zero = DAG.getConstant(0, PtrVT);
795 const TargetMachine &TM = DAG.getTarget();
797 if (TM.getRelocationModel() == Reloc::Static) {
798 if (!ST->usingLargeMem()) {
799 return DAG.getNode(SPUISD::AFormAddr, PtrVT, JTI, Zero);
801 SDOperand Hi = DAG.getNode(SPUISD::Hi, PtrVT, JTI, Zero);
802 SDOperand Lo = DAG.getNode(SPUISD::Lo, PtrVT, JTI, Zero);
803 return DAG.getNode(SPUISD::IndirectAddr, PtrVT, Hi, Lo);
808 "LowerJumpTable: Relocation model other than static not supported.");
813 LowerGlobalAddress(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
814 MVT::ValueType PtrVT = Op.getValueType();
815 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op);
816 GlobalValue *GV = GSDN->getGlobal();
817 SDOperand GA = DAG.getTargetGlobalAddress(GV, PtrVT, GSDN->getOffset());
818 const TargetMachine &TM = DAG.getTarget();
819 SDOperand Zero = DAG.getConstant(0, PtrVT);
821 if (TM.getRelocationModel() == Reloc::Static) {
822 if (!ST->usingLargeMem()) {
823 return DAG.getNode(SPUISD::AFormAddr, PtrVT, GA, Zero);
825 SDOperand Hi = DAG.getNode(SPUISD::Hi, PtrVT, GA, Zero);
826 SDOperand Lo = DAG.getNode(SPUISD::Lo, PtrVT, GA, Zero);
827 return DAG.getNode(SPUISD::IndirectAddr, PtrVT, Hi, Lo);
830 cerr << "LowerGlobalAddress: Relocation model other than static not "
839 //! Custom lower i64 integer constants
841 This code inserts all of the necessary juggling that needs to occur to load
842 a 64-bit constant into a register.
845 LowerConstant(SDOperand Op, SelectionDAG &DAG) {
846 unsigned VT = Op.getValueType();
847 ConstantSDNode *CN = cast<ConstantSDNode>(Op.Val);
849 if (VT == MVT::i64) {
850 SDOperand T = DAG.getConstant(CN->getValue(), MVT::i64);
851 return DAG.getNode(SPUISD::EXTRACT_ELT0, VT,
852 DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i64, T, T));
854 cerr << "LowerConstant: unhandled constant type "
855 << MVT::getValueTypeString(VT)
864 //! Custom lower double precision floating point constants
866 LowerConstantFP(SDOperand Op, SelectionDAG &DAG) {
867 unsigned VT = Op.getValueType();
868 ConstantFPSDNode *FP = cast<ConstantFPSDNode>(Op.Val);
871 "LowerConstantFP: Node is not ConstantFPSDNode");
873 if (VT == MVT::f64) {
874 uint64_t dbits = DoubleToBits(FP->getValueAPF().convertToDouble());
875 return DAG.getNode(ISD::BIT_CONVERT, VT,
876 LowerConstant(DAG.getConstant(dbits, MVT::i64), DAG));
882 //! Lower MVT::i1, MVT::i8 brcond to a promoted type (MVT::i32, MVT::i16)
884 LowerBRCOND(SDOperand Op, SelectionDAG &DAG)
886 SDOperand Cond = Op.getOperand(1);
887 MVT::ValueType CondVT = Cond.getValueType();
888 MVT::ValueType CondNVT;
890 if (CondVT == MVT::i1 || CondVT == MVT::i8) {
891 CondNVT = (CondVT == MVT::i1 ? MVT::i32 : MVT::i16);
892 return DAG.getNode(ISD::BRCOND, Op.getValueType(),
894 DAG.getNode(ISD::ZERO_EXTEND, CondNVT, Op.getOperand(1)),
897 return SDOperand(); // Unchanged
901 LowerFORMAL_ARGUMENTS(SDOperand Op, SelectionDAG &DAG, int &VarArgsFrameIndex)
903 MachineFunction &MF = DAG.getMachineFunction();
904 MachineFrameInfo *MFI = MF.getFrameInfo();
905 MachineRegisterInfo &RegInfo = MF.getRegInfo();
906 SmallVector<SDOperand, 8> ArgValues;
907 SDOperand Root = Op.getOperand(0);
908 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
910 const unsigned *ArgRegs = SPURegisterInfo::getArgRegs();
911 const unsigned NumArgRegs = SPURegisterInfo::getNumArgRegs();
913 unsigned ArgOffset = SPUFrameInfo::minStackSize();
914 unsigned ArgRegIdx = 0;
915 unsigned StackSlotSize = SPUFrameInfo::stackSlotSize();
917 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
919 // Add DAG nodes to load the arguments or copy them out of registers.
920 for (unsigned ArgNo = 0, e = Op.Val->getNumValues()-1; ArgNo != e; ++ArgNo) {
922 bool needsLoad = false;
923 MVT::ValueType ObjectVT = Op.getValue(ArgNo).getValueType();
924 unsigned ObjSize = MVT::getSizeInBits(ObjectVT)/8;
928 cerr << "LowerFORMAL_ARGUMENTS Unhandled argument type: "
929 << MVT::getValueTypeString(ObjectVT)
934 if (!isVarArg && ArgRegIdx < NumArgRegs) {
935 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R8CRegClass);
936 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
937 ArgVal = DAG.getCopyFromReg(Root, VReg, MVT::i8);
944 if (!isVarArg && ArgRegIdx < NumArgRegs) {
945 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R16CRegClass);
946 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
947 ArgVal = DAG.getCopyFromReg(Root, VReg, MVT::i16);
954 if (!isVarArg && ArgRegIdx < NumArgRegs) {
955 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
956 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
957 ArgVal = DAG.getCopyFromReg(Root, VReg, MVT::i32);
964 if (!isVarArg && ArgRegIdx < NumArgRegs) {
965 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R64CRegClass);
966 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
967 ArgVal = DAG.getCopyFromReg(Root, VReg, MVT::i64);
974 if (!isVarArg && ArgRegIdx < NumArgRegs) {
975 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R32FPRegClass);
976 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
977 ArgVal = DAG.getCopyFromReg(Root, VReg, MVT::f32);
984 if (!isVarArg && ArgRegIdx < NumArgRegs) {
985 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R64FPRegClass);
986 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
987 ArgVal = DAG.getCopyFromReg(Root, VReg, MVT::f64);
999 if (!isVarArg && ArgRegIdx < NumArgRegs) {
1000 unsigned VReg = RegInfo.createVirtualRegister(&SPU::VECREGRegClass);
1001 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
1002 ArgVal = DAG.getCopyFromReg(Root, VReg, ObjectVT);
1010 // We need to load the argument to a virtual register if we determined above
1011 // that we ran out of physical registers of the appropriate type
1013 int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
1014 SDOperand FIN = DAG.getFrameIndex(FI, PtrVT);
1015 ArgVal = DAG.getLoad(ObjectVT, Root, FIN, NULL, 0);
1016 ArgOffset += StackSlotSize;
1019 ArgValues.push_back(ArgVal);
1022 // If the function takes variable number of arguments, make a frame index for
1023 // the start of the first vararg value... for expansion of llvm.va_start.
1025 VarArgsFrameIndex = MFI->CreateFixedObject(MVT::getSizeInBits(PtrVT)/8,
1027 SDOperand FIN = DAG.getFrameIndex(VarArgsFrameIndex, PtrVT);
1028 // If this function is vararg, store any remaining integer argument regs to
1029 // their spots on the stack so that they may be loaded by deferencing the
1030 // result of va_next.
1031 SmallVector<SDOperand, 8> MemOps;
1032 for (; ArgRegIdx != NumArgRegs; ++ArgRegIdx) {
1033 unsigned VReg = RegInfo.createVirtualRegister(&SPU::GPRCRegClass);
1034 RegInfo.addLiveIn(ArgRegs[ArgRegIdx], VReg);
1035 SDOperand Val = DAG.getCopyFromReg(Root, VReg, PtrVT);
1036 SDOperand Store = DAG.getStore(Val.getValue(1), Val, FIN, NULL, 0);
1037 MemOps.push_back(Store);
1038 // Increment the address by four for the next argument to store
1039 SDOperand PtrOff = DAG.getConstant(MVT::getSizeInBits(PtrVT)/8, PtrVT);
1040 FIN = DAG.getNode(ISD::ADD, PtrOff.getValueType(), FIN, PtrOff);
1042 if (!MemOps.empty())
1043 Root = DAG.getNode(ISD::TokenFactor, MVT::Other,&MemOps[0],MemOps.size());
1046 ArgValues.push_back(Root);
1048 // Return the new list of results.
1049 std::vector<MVT::ValueType> RetVT(Op.Val->value_begin(),
1050 Op.Val->value_end());
1051 return DAG.getNode(ISD::MERGE_VALUES, RetVT, &ArgValues[0], ArgValues.size());
1054 /// isLSAAddress - Return the immediate to use if the specified
1055 /// value is representable as a LSA address.
1056 static SDNode *isLSAAddress(SDOperand Op, SelectionDAG &DAG) {
1057 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
1060 int Addr = C->getValue();
1061 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
1062 (Addr << 14 >> 14) != Addr)
1063 return 0; // Top 14 bits have to be sext of immediate.
1065 return DAG.getConstant((int)C->getValue() >> 2, MVT::i32).Val;
1070 LowerCALL(SDOperand Op, SelectionDAG &DAG, const SPUSubtarget *ST) {
1071 SDOperand Chain = Op.getOperand(0);
1073 bool isVarArg = cast<ConstantSDNode>(Op.getOperand(2))->getValue() != 0;
1074 bool isTailCall = cast<ConstantSDNode>(Op.getOperand(3))->getValue() != 0;
1076 SDOperand Callee = Op.getOperand(4);
1077 unsigned NumOps = (Op.getNumOperands() - 5) / 2;
1078 unsigned StackSlotSize = SPUFrameInfo::stackSlotSize();
1079 const unsigned *ArgRegs = SPURegisterInfo::getArgRegs();
1080 const unsigned NumArgRegs = SPURegisterInfo::getNumArgRegs();
1082 // Handy pointer type
1083 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
1085 // Accumulate how many bytes are to be pushed on the stack, including the
1086 // linkage area, and parameter passing area. According to the SPU ABI,
1087 // we minimally need space for [LR] and [SP]
1088 unsigned NumStackBytes = SPUFrameInfo::minStackSize();
1090 // Set up a copy of the stack pointer for use loading and storing any
1091 // arguments that may not fit in the registers available for argument
1093 SDOperand StackPtr = DAG.getRegister(SPU::R1, MVT::i32);
1095 // Figure out which arguments are going to go in registers, and which in
1097 unsigned ArgOffset = SPUFrameInfo::minStackSize(); // Just below [LR]
1098 unsigned ArgRegIdx = 0;
1100 // Keep track of registers passing arguments
1101 std::vector<std::pair<unsigned, SDOperand> > RegsToPass;
1102 // And the arguments passed on the stack
1103 SmallVector<SDOperand, 8> MemOpChains;
1105 for (unsigned i = 0; i != NumOps; ++i) {
1106 SDOperand Arg = Op.getOperand(5+2*i);
1108 // PtrOff will be used to store the current argument to the stack if a
1109 // register cannot be found for it.
1110 SDOperand PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType());
1111 PtrOff = DAG.getNode(ISD::ADD, PtrVT, StackPtr, PtrOff);
1113 switch (Arg.getValueType()) {
1114 default: assert(0 && "Unexpected ValueType for argument!");
1118 if (ArgRegIdx != NumArgRegs) {
1119 RegsToPass.push_back(std::make_pair(ArgRegs[ArgRegIdx++], Arg));
1121 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
1122 ArgOffset += StackSlotSize;
1127 if (ArgRegIdx != NumArgRegs) {
1128 RegsToPass.push_back(std::make_pair(ArgRegs[ArgRegIdx++], Arg));
1130 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
1131 ArgOffset += StackSlotSize;
1138 if (ArgRegIdx != NumArgRegs) {
1139 RegsToPass.push_back(std::make_pair(ArgRegs[ArgRegIdx++], Arg));
1141 MemOpChains.push_back(DAG.getStore(Chain, Arg, PtrOff, NULL, 0));
1142 ArgOffset += StackSlotSize;
1148 // Update number of stack bytes actually used, insert a call sequence start
1149 NumStackBytes = (ArgOffset - SPUFrameInfo::minStackSize());
1150 Chain = DAG.getCALLSEQ_START(Chain, DAG.getConstant(NumStackBytes, PtrVT));
1152 if (!MemOpChains.empty()) {
1153 // Adjust the stack pointer for the stack arguments.
1154 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
1155 &MemOpChains[0], MemOpChains.size());
1158 // Build a sequence of copy-to-reg nodes chained together with token chain
1159 // and flag operands which copy the outgoing args into the appropriate regs.
1161 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1162 Chain = DAG.getCopyToReg(Chain, RegsToPass[i].first, RegsToPass[i].second,
1164 InFlag = Chain.getValue(1);
1167 std::vector<MVT::ValueType> NodeTys;
1168 NodeTys.push_back(MVT::Other); // Returns a chain
1169 NodeTys.push_back(MVT::Flag); // Returns a flag for retval copy to use.
1171 SmallVector<SDOperand, 8> Ops;
1172 unsigned CallOpc = SPUISD::CALL;
1174 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
1175 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
1176 // node so that legalize doesn't hack it.
1177 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1178 GlobalValue *GV = G->getGlobal();
1179 unsigned CalleeVT = Callee.getValueType();
1180 SDOperand Zero = DAG.getConstant(0, PtrVT);
1181 SDOperand GA = DAG.getTargetGlobalAddress(GV, CalleeVT);
1183 if (!ST->usingLargeMem()) {
1184 // Turn calls to targets that are defined (i.e., have bodies) into BRSL
1185 // style calls, otherwise, external symbols are BRASL calls. This assumes
1186 // that declared/defined symbols are in the same compilation unit and can
1187 // be reached through PC-relative jumps.
1190 // This may be an unsafe assumption for JIT and really large compilation
1192 if (GV->isDeclaration()) {
1193 Callee = DAG.getNode(SPUISD::AFormAddr, CalleeVT, GA, Zero);
1195 Callee = DAG.getNode(SPUISD::PCRelAddr, CalleeVT, GA, Zero);
1198 // "Large memory" mode: Turn all calls into indirect calls with a X-form
1200 Callee = DAG.getNode(SPUISD::IndirectAddr, PtrVT, GA, Zero);
1202 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
1203 Callee = DAG.getExternalSymbol(S->getSymbol(), Callee.getValueType());
1204 else if (SDNode *Dest = isLSAAddress(Callee, DAG)) {
1205 // If this is an absolute destination address that appears to be a legal
1206 // local store address, use the munged value.
1207 Callee = SDOperand(Dest, 0);
1210 Ops.push_back(Chain);
1211 Ops.push_back(Callee);
1213 // Add argument registers to the end of the list so that they are known live
1215 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
1216 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
1217 RegsToPass[i].second.getValueType()));
1220 Ops.push_back(InFlag);
1221 Chain = DAG.getNode(CallOpc, NodeTys, &Ops[0], Ops.size());
1222 InFlag = Chain.getValue(1);
1224 Chain = DAG.getCALLSEQ_END(Chain,
1225 DAG.getConstant(NumStackBytes, PtrVT),
1226 DAG.getConstant(0, PtrVT),
1228 if (Op.Val->getValueType(0) != MVT::Other)
1229 InFlag = Chain.getValue(1);
1231 SDOperand ResultVals[3];
1232 unsigned NumResults = 0;
1235 // If the call has results, copy the values out of the ret val registers.
1236 switch (Op.Val->getValueType(0)) {
1237 default: assert(0 && "Unexpected ret value!");
1238 case MVT::Other: break;
1240 if (Op.Val->getValueType(1) == MVT::i32) {
1241 Chain = DAG.getCopyFromReg(Chain, SPU::R4, MVT::i32, InFlag).getValue(1);
1242 ResultVals[0] = Chain.getValue(0);
1243 Chain = DAG.getCopyFromReg(Chain, SPU::R3, MVT::i32,
1244 Chain.getValue(2)).getValue(1);
1245 ResultVals[1] = Chain.getValue(0);
1247 NodeTys.push_back(MVT::i32);
1249 Chain = DAG.getCopyFromReg(Chain, SPU::R3, MVT::i32, InFlag).getValue(1);
1250 ResultVals[0] = Chain.getValue(0);
1253 NodeTys.push_back(MVT::i32);
1256 Chain = DAG.getCopyFromReg(Chain, SPU::R3, MVT::i64, InFlag).getValue(1);
1257 ResultVals[0] = Chain.getValue(0);
1259 NodeTys.push_back(MVT::i64);
1263 Chain = DAG.getCopyFromReg(Chain, SPU::R3, Op.Val->getValueType(0),
1264 InFlag).getValue(1);
1265 ResultVals[0] = Chain.getValue(0);
1267 NodeTys.push_back(Op.Val->getValueType(0));
1274 Chain = DAG.getCopyFromReg(Chain, SPU::R3, Op.Val->getValueType(0),
1275 InFlag).getValue(1);
1276 ResultVals[0] = Chain.getValue(0);
1278 NodeTys.push_back(Op.Val->getValueType(0));
1282 NodeTys.push_back(MVT::Other);
1284 // If the function returns void, just return the chain.
1285 if (NumResults == 0)
1288 // Otherwise, merge everything together with a MERGE_VALUES node.
1289 ResultVals[NumResults++] = Chain;
1290 SDOperand Res = DAG.getNode(ISD::MERGE_VALUES, NodeTys,
1291 ResultVals, NumResults);
1292 return Res.getValue(Op.ResNo);
1296 LowerRET(SDOperand Op, SelectionDAG &DAG, TargetMachine &TM) {
1297 SmallVector<CCValAssign, 16> RVLocs;
1298 unsigned CC = DAG.getMachineFunction().getFunction()->getCallingConv();
1299 bool isVarArg = DAG.getMachineFunction().getFunction()->isVarArg();
1300 CCState CCInfo(CC, isVarArg, TM, RVLocs);
1301 CCInfo.AnalyzeReturn(Op.Val, RetCC_SPU);
1303 // If this is the first return lowered for this function, add the regs to the
1304 // liveout set for the function.
1305 if (DAG.getMachineFunction().getRegInfo().liveout_empty()) {
1306 for (unsigned i = 0; i != RVLocs.size(); ++i)
1307 DAG.getMachineFunction().getRegInfo().addLiveOut(RVLocs[i].getLocReg());
1310 SDOperand Chain = Op.getOperand(0);
1313 // Copy the result values into the output registers.
1314 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1315 CCValAssign &VA = RVLocs[i];
1316 assert(VA.isRegLoc() && "Can only return in registers!");
1317 Chain = DAG.getCopyToReg(Chain, VA.getLocReg(), Op.getOperand(i*2+1), Flag);
1318 Flag = Chain.getValue(1);
1322 return DAG.getNode(SPUISD::RET_FLAG, MVT::Other, Chain, Flag);
1324 return DAG.getNode(SPUISD::RET_FLAG, MVT::Other, Chain);
1328 //===----------------------------------------------------------------------===//
1329 // Vector related lowering:
1330 //===----------------------------------------------------------------------===//
1332 static ConstantSDNode *
1333 getVecImm(SDNode *N) {
1334 SDOperand OpVal(0, 0);
1336 // Check to see if this buildvec has a single non-undef value in its elements.
1337 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
1338 if (N->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1340 OpVal = N->getOperand(i);
1341 else if (OpVal != N->getOperand(i))
1345 if (OpVal.Val != 0) {
1346 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) {
1351 return 0; // All UNDEF: use implicit def.; not Constant node
1354 /// get_vec_i18imm - Test if this vector is a vector filled with the same value
1355 /// and the value fits into an unsigned 18-bit constant, and if so, return the
1357 SDOperand SPU::get_vec_u18imm(SDNode *N, SelectionDAG &DAG,
1358 MVT::ValueType ValueType) {
1359 if (ConstantSDNode *CN = getVecImm(N)) {
1360 uint64_t Value = CN->getValue();
1361 if (ValueType == MVT::i64) {
1362 uint64_t UValue = CN->getValue();
1363 uint32_t upper = uint32_t(UValue >> 32);
1364 uint32_t lower = uint32_t(UValue);
1367 Value = Value >> 32;
1369 if (Value <= 0x3ffff)
1370 return DAG.getConstant(Value, ValueType);
1376 /// get_vec_i16imm - Test if this vector is a vector filled with the same value
1377 /// and the value fits into a signed 16-bit constant, and if so, return the
1379 SDOperand SPU::get_vec_i16imm(SDNode *N, SelectionDAG &DAG,
1380 MVT::ValueType ValueType) {
1381 if (ConstantSDNode *CN = getVecImm(N)) {
1382 int64_t Value = CN->getSignExtended();
1383 if (ValueType == MVT::i64) {
1384 uint64_t UValue = CN->getValue();
1385 uint32_t upper = uint32_t(UValue >> 32);
1386 uint32_t lower = uint32_t(UValue);
1389 Value = Value >> 32;
1391 if (Value >= -(1 << 15) && Value <= ((1 << 15) - 1)) {
1392 return DAG.getConstant(Value, ValueType);
1399 /// get_vec_i10imm - Test if this vector is a vector filled with the same value
1400 /// and the value fits into a signed 10-bit constant, and if so, return the
1402 SDOperand SPU::get_vec_i10imm(SDNode *N, SelectionDAG &DAG,
1403 MVT::ValueType ValueType) {
1404 if (ConstantSDNode *CN = getVecImm(N)) {
1405 int64_t Value = CN->getSignExtended();
1406 if (ValueType == MVT::i64) {
1407 uint64_t UValue = CN->getValue();
1408 uint32_t upper = uint32_t(UValue >> 32);
1409 uint32_t lower = uint32_t(UValue);
1412 Value = Value >> 32;
1414 if (isS10Constant(Value))
1415 return DAG.getConstant(Value, ValueType);
1421 /// get_vec_i8imm - Test if this vector is a vector filled with the same value
1422 /// and the value fits into a signed 8-bit constant, and if so, return the
1425 /// @note: The incoming vector is v16i8 because that's the only way we can load
1426 /// constant vectors. Thus, we test to see if the upper and lower bytes are the
1428 SDOperand SPU::get_vec_i8imm(SDNode *N, SelectionDAG &DAG,
1429 MVT::ValueType ValueType) {
1430 if (ConstantSDNode *CN = getVecImm(N)) {
1431 int Value = (int) CN->getValue();
1432 if (ValueType == MVT::i16
1433 && Value <= 0xffff /* truncated from uint64_t */
1434 && ((short) Value >> 8) == ((short) Value & 0xff))
1435 return DAG.getConstant(Value & 0xff, ValueType);
1436 else if (ValueType == MVT::i8
1437 && (Value & 0xff) == Value)
1438 return DAG.getConstant(Value, ValueType);
1444 /// get_ILHUvec_imm - Test if this vector is a vector filled with the same value
1445 /// and the value fits into a signed 16-bit constant, and if so, return the
1447 SDOperand SPU::get_ILHUvec_imm(SDNode *N, SelectionDAG &DAG,
1448 MVT::ValueType ValueType) {
1449 if (ConstantSDNode *CN = getVecImm(N)) {
1450 uint64_t Value = CN->getValue();
1451 if ((ValueType == MVT::i32
1452 && ((unsigned) Value & 0xffff0000) == (unsigned) Value)
1453 || (ValueType == MVT::i64 && (Value & 0xffff0000) == Value))
1454 return DAG.getConstant(Value >> 16, ValueType);
1460 /// get_v4i32_imm - Catch-all for general 32-bit constant vectors
1461 SDOperand SPU::get_v4i32_imm(SDNode *N, SelectionDAG &DAG) {
1462 if (ConstantSDNode *CN = getVecImm(N)) {
1463 return DAG.getConstant((unsigned) CN->getValue(), MVT::i32);
1469 /// get_v4i32_imm - Catch-all for general 64-bit constant vectors
1470 SDOperand SPU::get_v2i64_imm(SDNode *N, SelectionDAG &DAG) {
1471 if (ConstantSDNode *CN = getVecImm(N)) {
1472 return DAG.getConstant((unsigned) CN->getValue(), MVT::i64);
1478 // If this is a vector of constants or undefs, get the bits. A bit in
1479 // UndefBits is set if the corresponding element of the vector is an
1480 // ISD::UNDEF value. For undefs, the corresponding VectorBits values are
1481 // zero. Return true if this is not an array of constants, false if it is.
1483 static bool GetConstantBuildVectorBits(SDNode *BV, uint64_t VectorBits[2],
1484 uint64_t UndefBits[2]) {
1485 // Start with zero'd results.
1486 VectorBits[0] = VectorBits[1] = UndefBits[0] = UndefBits[1] = 0;
1488 unsigned EltBitSize = MVT::getSizeInBits(BV->getOperand(0).getValueType());
1489 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
1490 SDOperand OpVal = BV->getOperand(i);
1492 unsigned PartNo = i >= e/2; // In the upper 128 bits?
1493 unsigned SlotNo = e/2 - (i & (e/2-1))-1; // Which subpiece of the uint64_t.
1495 uint64_t EltBits = 0;
1496 if (OpVal.getOpcode() == ISD::UNDEF) {
1497 uint64_t EltUndefBits = ~0ULL >> (64-EltBitSize);
1498 UndefBits[PartNo] |= EltUndefBits << (SlotNo*EltBitSize);
1500 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) {
1501 EltBits = CN->getValue() & (~0ULL >> (64-EltBitSize));
1502 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) {
1503 const APFloat &apf = CN->getValueAPF();
1504 EltBits = (CN->getValueType(0) == MVT::f32
1505 ? FloatToBits(apf.convertToFloat())
1506 : DoubleToBits(apf.convertToDouble()));
1508 // Nonconstant element.
1512 VectorBits[PartNo] |= EltBits << (SlotNo*EltBitSize);
1515 //printf("%llx %llx %llx %llx\n",
1516 // VectorBits[0], VectorBits[1], UndefBits[0], UndefBits[1]);
1520 /// If this is a splat (repetition) of a value across the whole vector, return
1521 /// the smallest size that splats it. For example, "0x01010101010101..." is a
1522 /// splat of 0x01, 0x0101, and 0x01010101. We return SplatBits = 0x01 and
1523 /// SplatSize = 1 byte.
1524 static bool isConstantSplat(const uint64_t Bits128[2],
1525 const uint64_t Undef128[2],
1527 uint64_t &SplatBits, uint64_t &SplatUndef,
1529 // Don't let undefs prevent splats from matching. See if the top 64-bits are
1530 // the same as the lower 64-bits, ignoring undefs.
1531 uint64_t Bits64 = Bits128[0] | Bits128[1];
1532 uint64_t Undef64 = Undef128[0] & Undef128[1];
1533 uint32_t Bits32 = uint32_t(Bits64) | uint32_t(Bits64 >> 32);
1534 uint32_t Undef32 = uint32_t(Undef64) & uint32_t(Undef64 >> 32);
1535 uint16_t Bits16 = uint16_t(Bits32) | uint16_t(Bits32 >> 16);
1536 uint16_t Undef16 = uint16_t(Undef32) & uint16_t(Undef32 >> 16);
1538 if ((Bits128[0] & ~Undef128[1]) == (Bits128[1] & ~Undef128[0])) {
1539 if (MinSplatBits < 64) {
1541 // Check that the top 32-bits are the same as the lower 32-bits, ignoring
1543 if ((Bits64 & (~Undef64 >> 32)) == ((Bits64 >> 32) & ~Undef64)) {
1544 if (MinSplatBits < 32) {
1546 // If the top 16-bits are different than the lower 16-bits, ignoring
1547 // undefs, we have an i32 splat.
1548 if ((Bits32 & (~Undef32 >> 16)) == ((Bits32 >> 16) & ~Undef32)) {
1549 if (MinSplatBits < 16) {
1550 // If the top 8-bits are different than the lower 8-bits, ignoring
1551 // undefs, we have an i16 splat.
1552 if ((Bits16 & (uint16_t(~Undef16) >> 8)) == ((Bits16 >> 8) & ~Undef16)) {
1553 // Otherwise, we have an 8-bit splat.
1554 SplatBits = uint8_t(Bits16) | uint8_t(Bits16 >> 8);
1555 SplatUndef = uint8_t(Undef16) & uint8_t(Undef16 >> 8);
1561 SplatUndef = Undef16;
1568 SplatUndef = Undef32;
1574 SplatBits = Bits128[0];
1575 SplatUndef = Undef128[0];
1581 return false; // Can't be a splat if two pieces don't match.
1584 // If this is a case we can't handle, return null and let the default
1585 // expansion code take care of it. If we CAN select this case, and if it
1586 // selects to a single instruction, return Op. Otherwise, if we can codegen
1587 // this case more efficiently than a constant pool load, lower it to the
1588 // sequence of ops that should be used.
1589 static SDOperand LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
1590 MVT::ValueType VT = Op.getValueType();
1591 // If this is a vector of constants or undefs, get the bits. A bit in
1592 // UndefBits is set if the corresponding element of the vector is an
1593 // ISD::UNDEF value. For undefs, the corresponding VectorBits values are
1595 uint64_t VectorBits[2];
1596 uint64_t UndefBits[2];
1597 uint64_t SplatBits, SplatUndef;
1599 if (GetConstantBuildVectorBits(Op.Val, VectorBits, UndefBits)
1600 || !isConstantSplat(VectorBits, UndefBits,
1601 MVT::getSizeInBits(MVT::getVectorElementType(VT)),
1602 SplatBits, SplatUndef, SplatSize))
1603 return SDOperand(); // Not a constant vector, not a splat.
1608 uint32_t Value32 = SplatBits;
1609 assert(SplatSize == 4
1610 && "LowerBUILD_VECTOR: Unexpected floating point vector element.");
1611 // NOTE: pretend the constant is an integer. LLVM won't load FP constants
1612 SDOperand T = DAG.getConstant(Value32, MVT::i32);
1613 return DAG.getNode(ISD::BIT_CONVERT, MVT::v4f32,
1614 DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, T, T, T, T));
1618 uint64_t f64val = SplatBits;
1619 assert(SplatSize == 8
1620 && "LowerBUILD_VECTOR: 64-bit float vector element: unexpected size.");
1621 // NOTE: pretend the constant is an integer. LLVM won't load FP constants
1622 SDOperand T = DAG.getConstant(f64val, MVT::i64);
1623 return DAG.getNode(ISD::BIT_CONVERT, MVT::v2f64,
1624 DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i64, T, T));
1628 // 8-bit constants have to be expanded to 16-bits
1629 unsigned short Value16 = SplatBits | (SplatBits << 8);
1631 for (int i = 0; i < 8; ++i)
1632 Ops[i] = DAG.getConstant(Value16, MVT::i16);
1633 return DAG.getNode(ISD::BIT_CONVERT, VT,
1634 DAG.getNode(ISD::BUILD_VECTOR, MVT::v8i16, Ops, 8));
1637 unsigned short Value16;
1639 Value16 = (unsigned short) (SplatBits & 0xffff);
1641 Value16 = (unsigned short) (SplatBits | (SplatBits << 8));
1642 SDOperand T = DAG.getConstant(Value16, MVT::getVectorElementType(VT));
1644 for (int i = 0; i < 8; ++i) Ops[i] = T;
1645 return DAG.getNode(ISD::BUILD_VECTOR, VT, Ops, 8);
1648 unsigned int Value = SplatBits;
1649 SDOperand T = DAG.getConstant(Value, MVT::getVectorElementType(VT));
1650 return DAG.getNode(ISD::BUILD_VECTOR, VT, T, T, T, T);
1653 uint64_t val = SplatBits;
1654 uint32_t upper = uint32_t(val >> 32);
1655 uint32_t lower = uint32_t(val);
1657 if (upper == lower) {
1658 // Magic constant that can be matched by IL, ILA, et. al.
1659 SDOperand Val = DAG.getTargetConstant(val, MVT::i64);
1660 return DAG.getNode(ISD::BUILD_VECTOR, VT, Val, Val);
1664 SmallVector<SDOperand, 16> ShufBytes;
1666 bool upper_special, lower_special;
1668 // NOTE: This code creates common-case shuffle masks that can be easily
1669 // detected as common expressions. It is not attempting to create highly
1670 // specialized masks to replace any and all 0's, 0xff's and 0x80's.
1672 // Detect if the upper or lower half is a special shuffle mask pattern:
1673 upper_special = (upper == 0 || upper == 0xffffffff || upper == 0x80000000);
1674 lower_special = (lower == 0 || lower == 0xffffffff || lower == 0x80000000);
1676 // Create lower vector if not a special pattern
1677 if (!lower_special) {
1678 SDOperand LO32C = DAG.getConstant(lower, MVT::i32);
1679 LO32 = DAG.getNode(ISD::BIT_CONVERT, VT,
1680 DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32,
1681 LO32C, LO32C, LO32C, LO32C));
1684 // Create upper vector if not a special pattern
1685 if (!upper_special) {
1686 SDOperand HI32C = DAG.getConstant(upper, MVT::i32);
1687 HI32 = DAG.getNode(ISD::BIT_CONVERT, VT,
1688 DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32,
1689 HI32C, HI32C, HI32C, HI32C));
1692 // If either upper or lower are special, then the two input operands are
1693 // the same (basically, one of them is a "don't care")
1698 if (lower_special && upper_special) {
1699 // Unhappy situation... both upper and lower are special, so punt with
1700 // a target constant:
1701 SDOperand Zero = DAG.getConstant(0, MVT::i32);
1702 HI32 = LO32 = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Zero, Zero,
1706 for (int i = 0; i < 4; ++i) {
1707 for (int j = 0; j < 4; ++j) {
1709 bool process_upper, process_lower;
1712 process_upper = (upper_special && (i & 1) == 0);
1713 process_lower = (lower_special && (i & 1) == 1);
1715 if (process_upper || process_lower) {
1716 if ((process_upper && upper == 0)
1717 || (process_lower && lower == 0))
1719 else if ((process_upper && upper == 0xffffffff)
1720 || (process_lower && lower == 0xffffffff))
1722 else if ((process_upper && upper == 0x80000000)
1723 || (process_lower && lower == 0x80000000))
1724 val = (j == 0 ? 0xe0 : 0x80);
1726 val = i * 4 + j + ((i & 1) * 16);
1728 ShufBytes.push_back(DAG.getConstant(val, MVT::i8));
1732 return DAG.getNode(SPUISD::SHUFB, VT, HI32, LO32,
1733 DAG.getNode(ISD::BUILD_VECTOR, MVT::v16i8,
1734 &ShufBytes[0], ShufBytes.size()));
1742 /// LowerVECTOR_SHUFFLE - Lower a vector shuffle (V1, V2, V3) to something on
1743 /// which the Cell can operate. The code inspects V3 to ascertain whether the
1744 /// permutation vector, V3, is monotonically increasing with one "exception"
1745 /// element, e.g., (0, 1, _, 3). If this is the case, then generate a
1746 /// INSERT_MASK synthetic instruction. Otherwise, spill V3 to the constant pool.
1747 /// In either case, the net result is going to eventually invoke SHUFB to
1748 /// permute/shuffle the bytes from V1 and V2.
1750 /// INSERT_MASK is eventually selected as one of the C*D instructions, generate
1751 /// control word for byte/halfword/word insertion. This takes care of a single
1752 /// element move from V2 into V1.
1754 /// SPUISD::SHUFB is eventually selected as Cell's <i>shufb</i> instructions.
1755 static SDOperand LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) {
1756 SDOperand V1 = Op.getOperand(0);
1757 SDOperand V2 = Op.getOperand(1);
1758 SDOperand PermMask = Op.getOperand(2);
1760 if (V2.getOpcode() == ISD::UNDEF) V2 = V1;
1762 // If we have a single element being moved from V1 to V2, this can be handled
1763 // using the C*[DX] compute mask instructions, but the vector elements have
1764 // to be monotonically increasing with one exception element.
1765 MVT::ValueType EltVT = MVT::getVectorElementType(V1.getValueType());
1766 unsigned EltsFromV2 = 0;
1768 unsigned V2EltIdx0 = 0;
1769 unsigned CurrElt = 0;
1770 bool monotonic = true;
1771 if (EltVT == MVT::i8)
1773 else if (EltVT == MVT::i16)
1775 else if (EltVT == MVT::i32)
1778 assert(0 && "Unhandled vector type in LowerVECTOR_SHUFFLE");
1780 for (unsigned i = 0, e = PermMask.getNumOperands();
1781 EltsFromV2 <= 1 && monotonic && i != e;
1784 if (PermMask.getOperand(i).getOpcode() == ISD::UNDEF)
1787 SrcElt = cast<ConstantSDNode>(PermMask.getOperand(i))->getValue();
1789 if (SrcElt >= V2EltIdx0) {
1791 V2Elt = (V2EltIdx0 - SrcElt) << 2;
1792 } else if (CurrElt != SrcElt) {
1799 if (EltsFromV2 == 1 && monotonic) {
1800 // Compute mask and shuffle
1801 MachineFunction &MF = DAG.getMachineFunction();
1802 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1803 unsigned VReg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
1804 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
1805 // Initialize temporary register to 0
1806 SDOperand InitTempReg =
1807 DAG.getCopyToReg(DAG.getEntryNode(), VReg, DAG.getConstant(0, PtrVT));
1808 // Copy register's contents as index in INSERT_MASK:
1809 SDOperand ShufMaskOp =
1810 DAG.getNode(SPUISD::INSERT_MASK, V1.getValueType(),
1811 DAG.getTargetConstant(V2Elt, MVT::i32),
1812 DAG.getCopyFromReg(InitTempReg, VReg, PtrVT));
1813 // Use shuffle mask in SHUFB synthetic instruction:
1814 return DAG.getNode(SPUISD::SHUFB, V1.getValueType(), V2, V1, ShufMaskOp);
1816 // Convert the SHUFFLE_VECTOR mask's input element units to the actual bytes.
1817 unsigned BytesPerElement = MVT::getSizeInBits(EltVT)/8;
1819 SmallVector<SDOperand, 16> ResultMask;
1820 for (unsigned i = 0, e = PermMask.getNumOperands(); i != e; ++i) {
1822 if (PermMask.getOperand(i).getOpcode() == ISD::UNDEF)
1825 SrcElt = cast<ConstantSDNode>(PermMask.getOperand(i))->getValue();
1827 for (unsigned j = 0; j < BytesPerElement; ++j) {
1828 ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement+j,
1833 SDOperand VPermMask = DAG.getNode(ISD::BUILD_VECTOR, MVT::v16i8,
1834 &ResultMask[0], ResultMask.size());
1835 return DAG.getNode(SPUISD::SHUFB, V1.getValueType(), V1, V2, VPermMask);
1839 static SDOperand LowerSCALAR_TO_VECTOR(SDOperand Op, SelectionDAG &DAG) {
1840 SDOperand Op0 = Op.getOperand(0); // Op0 = the scalar
1842 if (Op0.Val->getOpcode() == ISD::Constant) {
1843 // For a constant, build the appropriate constant vector, which will
1844 // eventually simplify to a vector register load.
1846 ConstantSDNode *CN = cast<ConstantSDNode>(Op0.Val);
1847 SmallVector<SDOperand, 16> ConstVecValues;
1851 // Create a constant vector:
1852 switch (Op.getValueType()) {
1853 default: assert(0 && "Unexpected constant value type in "
1854 "LowerSCALAR_TO_VECTOR");
1855 case MVT::v16i8: n_copies = 16; VT = MVT::i8; break;
1856 case MVT::v8i16: n_copies = 8; VT = MVT::i16; break;
1857 case MVT::v4i32: n_copies = 4; VT = MVT::i32; break;
1858 case MVT::v4f32: n_copies = 4; VT = MVT::f32; break;
1859 case MVT::v2i64: n_copies = 2; VT = MVT::i64; break;
1860 case MVT::v2f64: n_copies = 2; VT = MVT::f64; break;
1863 SDOperand CValue = DAG.getConstant(CN->getValue(), VT);
1864 for (size_t j = 0; j < n_copies; ++j)
1865 ConstVecValues.push_back(CValue);
1867 return DAG.getNode(ISD::BUILD_VECTOR, Op.getValueType(),
1868 &ConstVecValues[0], ConstVecValues.size());
1870 // Otherwise, copy the value from one register to another:
1871 switch (Op0.getValueType()) {
1872 default: assert(0 && "Unexpected value type in LowerSCALAR_TO_VECTOR");
1879 return DAG.getNode(SPUISD::PROMOTE_SCALAR, Op.getValueType(), Op0, Op0);
1886 static SDOperand LowerVectorMUL(SDOperand Op, SelectionDAG &DAG) {
1887 switch (Op.getValueType()) {
1889 SDOperand rA = Op.getOperand(0);
1890 SDOperand rB = Op.getOperand(1);
1891 SDOperand HiProd1 = DAG.getNode(SPUISD::MPYH, MVT::v4i32, rA, rB);
1892 SDOperand HiProd2 = DAG.getNode(SPUISD::MPYH, MVT::v4i32, rB, rA);
1893 SDOperand LoProd = DAG.getNode(SPUISD::MPYU, MVT::v4i32, rA, rB);
1894 SDOperand Residual1 = DAG.getNode(ISD::ADD, MVT::v4i32, LoProd, HiProd1);
1896 return DAG.getNode(ISD::ADD, MVT::v4i32, Residual1, HiProd2);
1900 // Multiply two v8i16 vectors (pipeline friendly version):
1901 // a) multiply lower halves, mask off upper 16-bit of 32-bit product
1902 // b) multiply upper halves, rotate left by 16 bits (inserts 16 lower zeroes)
1903 // c) Use SELB to select upper and lower halves from the intermediate results
1905 // NOTE: We really want to move the FSMBI to earlier to actually get the
1906 // dual-issue. This code does manage to do this, even if it's a little on
1909 MachineFunction &MF = DAG.getMachineFunction();
1910 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1911 SDOperand Chain = Op.getOperand(0);
1912 SDOperand rA = Op.getOperand(0);
1913 SDOperand rB = Op.getOperand(1);
1914 unsigned FSMBIreg = RegInfo.createVirtualRegister(&SPU::VECREGRegClass);
1915 unsigned HiProdReg = RegInfo.createVirtualRegister(&SPU::VECREGRegClass);
1918 DAG.getCopyToReg(Chain, FSMBIreg,
1919 DAG.getNode(SPUISD::FSMBI, MVT::v8i16,
1920 DAG.getConstant(0xcccc, MVT::i32)));
1923 DAG.getCopyToReg(FSMBOp, HiProdReg,
1924 DAG.getNode(SPUISD::MPYHH, MVT::v8i16, rA, rB));
1926 SDOperand HHProd_v4i32 =
1927 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32,
1928 DAG.getCopyFromReg(HHProd, HiProdReg, MVT::v4i32));
1930 return DAG.getNode(SPUISD::SELB, MVT::v8i16,
1931 DAG.getNode(SPUISD::MPY, MVT::v8i16, rA, rB),
1932 DAG.getNode(ISD::BIT_CONVERT, Op.getValueType(),
1933 DAG.getNode(SPUISD::VEC_SHL, MVT::v4i32,
1935 DAG.getConstant(16, MVT::i16))),
1936 DAG.getCopyFromReg(FSMBOp, FSMBIreg, MVT::v4i32));
1939 // This M00sE is N@stI! (apologies to Monty Python)
1941 // SPU doesn't know how to do any 8-bit multiplication, so the solution
1942 // is to break it all apart, sign extend, and reassemble the various
1943 // intermediate products.
1945 SDOperand rA = Op.getOperand(0);
1946 SDOperand rB = Op.getOperand(1);
1947 SDOperand c8 = DAG.getConstant(8, MVT::i32);
1948 SDOperand c16 = DAG.getConstant(16, MVT::i32);
1951 DAG.getNode(SPUISD::MPY, MVT::v8i16,
1952 DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, rA),
1953 DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, rB));
1955 SDOperand rALH = DAG.getNode(SPUISD::VEC_SRA, MVT::v8i16, rA, c8);
1957 SDOperand rBLH = DAG.getNode(SPUISD::VEC_SRA, MVT::v8i16, rB, c8);
1960 DAG.getNode(SPUISD::VEC_SHL, MVT::v8i16,
1961 DAG.getNode(SPUISD::MPY, MVT::v8i16, rALH, rBLH), c8);
1963 SDOperand FSMBmask = DAG.getNode(SPUISD::FSMBI, MVT::v8i16,
1964 DAG.getConstant(0x2222, MVT::i32));
1966 SDOperand LoProdParts =
1967 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32,
1968 DAG.getNode(SPUISD::SELB, MVT::v8i16,
1969 LLProd, LHProd, FSMBmask));
1971 SDOperand LoProdMask = DAG.getConstant(0xffff, MVT::i32);
1974 DAG.getNode(ISD::AND, MVT::v4i32,
1976 DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32,
1977 LoProdMask, LoProdMask,
1978 LoProdMask, LoProdMask));
1981 DAG.getNode(SPUISD::VEC_SRA, MVT::v4i32,
1982 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, rA), c16);
1985 DAG.getNode(SPUISD::VEC_SRA, MVT::v4i32,
1986 DAG.getNode(ISD::BIT_CONVERT, MVT::v4i32, rB), c16);
1989 DAG.getNode(SPUISD::MPY, MVT::v8i16,
1990 DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, rAH),
1991 DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16, rBH));
1993 SDOperand HHProd_1 =
1994 DAG.getNode(SPUISD::MPY, MVT::v8i16,
1995 DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16,
1996 DAG.getNode(SPUISD::VEC_SRA, MVT::v4i32, rAH, c8)),
1997 DAG.getNode(ISD::BIT_CONVERT, MVT::v8i16,
1998 DAG.getNode(SPUISD::VEC_SRA, MVT::v4i32, rBH, c8)));
2001 DAG.getNode(SPUISD::SELB, MVT::v8i16,
2003 DAG.getNode(SPUISD::VEC_SHL, MVT::v8i16, HHProd_1, c8),
2007 DAG.getNode(SPUISD::VEC_SHL, MVT::v4i32, HHProd, c16);
2009 return DAG.getNode(ISD::BIT_CONVERT, MVT::v16i8,
2010 DAG.getNode(ISD::OR, MVT::v4i32,
2015 cerr << "CellSPU: Unknown vector multiplication, got "
2016 << MVT::getValueTypeString(Op.getValueType())
2025 static SDOperand LowerFDIVf32(SDOperand Op, SelectionDAG &DAG) {
2026 MachineFunction &MF = DAG.getMachineFunction();
2027 MachineRegisterInfo &RegInfo = MF.getRegInfo();
2029 SDOperand A = Op.getOperand(0);
2030 SDOperand B = Op.getOperand(1);
2031 unsigned VT = Op.getValueType();
2033 unsigned VRegBR, VRegC;
2035 if (VT == MVT::f32) {
2036 VRegBR = RegInfo.createVirtualRegister(&SPU::R32FPRegClass);
2037 VRegC = RegInfo.createVirtualRegister(&SPU::R32FPRegClass);
2039 VRegBR = RegInfo.createVirtualRegister(&SPU::VECREGRegClass);
2040 VRegC = RegInfo.createVirtualRegister(&SPU::VECREGRegClass);
2042 // TODO: make sure we're feeding FPInterp the right arguments
2043 // Right now: fi B, frest(B)
2046 // (Floating Interpolate (FP Reciprocal Estimate B))
2048 DAG.getCopyToReg(DAG.getEntryNode(), VRegBR,
2049 DAG.getNode(SPUISD::FPInterp, VT, B,
2050 DAG.getNode(SPUISD::FPRecipEst, VT, B)));
2052 // Computes A * BRcpl and stores in a temporary register
2054 DAG.getCopyToReg(BRcpl, VRegC,
2055 DAG.getNode(ISD::FMUL, VT, A,
2056 DAG.getCopyFromReg(BRcpl, VRegBR, VT)));
2057 // What's the Chain variable do? It's magic!
2058 // TODO: set Chain = Op(0).getEntryNode()
2060 return DAG.getNode(ISD::FADD, VT,
2061 DAG.getCopyFromReg(AxBRcpl, VRegC, VT),
2062 DAG.getNode(ISD::FMUL, VT,
2063 DAG.getCopyFromReg(AxBRcpl, VRegBR, VT),
2064 DAG.getNode(ISD::FSUB, VT, A,
2065 DAG.getNode(ISD::FMUL, VT, B,
2066 DAG.getCopyFromReg(AxBRcpl, VRegC, VT)))));
2069 static SDOperand LowerEXTRACT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) {
2070 unsigned VT = Op.getValueType();
2071 SDOperand N = Op.getOperand(0);
2072 SDOperand Elt = Op.getOperand(1);
2073 SDOperand ShufMask[16];
2074 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt);
2076 assert(C != 0 && "LowerEXTRACT_VECTOR_ELT expecting constant SDNode");
2078 int EltNo = (int) C->getValue();
2081 if (VT == MVT::i8 && EltNo >= 16)
2082 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i8 extraction slot > 15");
2083 else if (VT == MVT::i16 && EltNo >= 8)
2084 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i16 extraction slot > 7");
2085 else if (VT == MVT::i32 && EltNo >= 4)
2086 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i32 extraction slot > 4");
2087 else if (VT == MVT::i64 && EltNo >= 2)
2088 assert(0 && "SPU LowerEXTRACT_VECTOR_ELT: i64 extraction slot > 2");
2090 if (EltNo == 0 && (VT == MVT::i32 || VT == MVT::i64)) {
2091 // i32 and i64: Element 0 is the preferred slot
2092 return DAG.getNode(SPUISD::EXTRACT_ELT0, VT, N);
2095 // Need to generate shuffle mask and extract:
2096 int prefslot_begin = -1, prefslot_end = -1;
2097 int elt_byte = EltNo * MVT::getSizeInBits(VT) / 8;
2101 prefslot_begin = prefslot_end = 3;
2105 prefslot_begin = 2; prefslot_end = 3;
2109 prefslot_begin = 0; prefslot_end = 3;
2113 prefslot_begin = 0; prefslot_end = 7;
2118 assert(prefslot_begin != -1 && prefslot_end != -1 &&
2119 "LowerEXTRACT_VECTOR_ELT: preferred slots uninitialized");
2121 for (int i = 0; i < 16; ++i) {
2122 // zero fill uppper part of preferred slot, don't care about the
2124 unsigned int mask_val;
2126 if (i <= prefslot_end) {
2128 ((i < prefslot_begin)
2130 : elt_byte + (i - prefslot_begin));
2132 ShufMask[i] = DAG.getConstant(mask_val, MVT::i8);
2134 ShufMask[i] = ShufMask[i % (prefslot_end + 1)];
2137 SDOperand ShufMaskVec =
2138 DAG.getNode(ISD::BUILD_VECTOR, MVT::v16i8,
2140 sizeof(ShufMask) / sizeof(ShufMask[0]));
2142 return DAG.getNode(SPUISD::EXTRACT_ELT0, VT,
2143 DAG.getNode(SPUISD::SHUFB, N.getValueType(),
2144 N, N, ShufMaskVec));
2148 static SDOperand LowerINSERT_VECTOR_ELT(SDOperand Op, SelectionDAG &DAG) {
2149 SDOperand VecOp = Op.getOperand(0);
2150 SDOperand ValOp = Op.getOperand(1);
2151 SDOperand IdxOp = Op.getOperand(2);
2152 MVT::ValueType VT = Op.getValueType();
2154 ConstantSDNode *CN = cast<ConstantSDNode>(IdxOp);
2155 assert(CN != 0 && "LowerINSERT_VECTOR_ELT: Index is not constant!");
2157 MVT::ValueType PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
2158 // Use $2 because it's always 16-byte aligned and it's available:
2159 SDOperand PtrBase = DAG.getRegister(SPU::R2, PtrVT);
2162 DAG.getNode(SPUISD::SHUFB, VT,
2163 DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, ValOp),
2165 DAG.getNode(SPUISD::INSERT_MASK, VT,
2166 DAG.getNode(ISD::ADD, PtrVT,
2168 DAG.getConstant(CN->getValue(),
2174 static SDOperand LowerI8Math(SDOperand Op, SelectionDAG &DAG, unsigned Opc)
2176 SDOperand N0 = Op.getOperand(0); // Everything has at least one operand
2178 assert(Op.getValueType() == MVT::i8);
2181 assert(0 && "Unhandled i8 math operator");
2185 // 8-bit subtraction: Promote the arguments up to 16-bits and truncate
2187 SDOperand N1 = Op.getOperand(1);
2188 N0 = (N0.getOpcode() != ISD::Constant
2189 ? DAG.getNode(ISD::SIGN_EXTEND, MVT::i16, N0)
2190 : DAG.getConstant(cast<ConstantSDNode>(N0)->getValue(), MVT::i16));
2191 N1 = (N1.getOpcode() != ISD::Constant
2192 ? DAG.getNode(ISD::SIGN_EXTEND, MVT::i16, N1)
2193 : DAG.getConstant(cast<ConstantSDNode>(N1)->getValue(), MVT::i16));
2194 return DAG.getNode(ISD::TRUNCATE, MVT::i8,
2195 DAG.getNode(Opc, MVT::i16, N0, N1));
2199 SDOperand N1 = Op.getOperand(1);
2201 N0 = (N0.getOpcode() != ISD::Constant
2202 ? DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, N0)
2203 : DAG.getConstant(cast<ConstantSDNode>(N0)->getValue(), MVT::i16));
2204 N1Opc = (N1.getValueType() < MVT::i16 ? ISD::ZERO_EXTEND : ISD::TRUNCATE);
2205 N1 = (N1.getOpcode() != ISD::Constant
2206 ? DAG.getNode(N1Opc, MVT::i16, N1)
2207 : DAG.getConstant(cast<ConstantSDNode>(N1)->getValue(), MVT::i16));
2208 SDOperand ExpandArg =
2209 DAG.getNode(ISD::OR, MVT::i16, N0,
2210 DAG.getNode(ISD::SHL, MVT::i16,
2211 N0, DAG.getConstant(8, MVT::i16)));
2212 return DAG.getNode(ISD::TRUNCATE, MVT::i8,
2213 DAG.getNode(Opc, MVT::i16, ExpandArg, N1));
2217 SDOperand N1 = Op.getOperand(1);
2219 N0 = (N0.getOpcode() != ISD::Constant
2220 ? DAG.getNode(ISD::ZERO_EXTEND, MVT::i16, N0)
2221 : DAG.getConstant(cast<ConstantSDNode>(N0)->getValue(), MVT::i16));
2222 N1Opc = (N1.getValueType() < MVT::i16 ? ISD::ZERO_EXTEND : ISD::TRUNCATE);
2223 N1 = (N1.getOpcode() != ISD::Constant
2224 ? DAG.getNode(N1Opc, MVT::i16, N1)
2225 : DAG.getConstant(cast<ConstantSDNode>(N1)->getValue(), MVT::i16));
2226 return DAG.getNode(ISD::TRUNCATE, MVT::i8,
2227 DAG.getNode(Opc, MVT::i16, N0, N1));
2230 SDOperand N1 = Op.getOperand(1);
2232 N0 = (N0.getOpcode() != ISD::Constant
2233 ? DAG.getNode(ISD::SIGN_EXTEND, MVT::i16, N0)
2234 : DAG.getConstant(cast<ConstantSDNode>(N0)->getValue(), MVT::i16));
2235 N1Opc = (N1.getValueType() < MVT::i16 ? ISD::SIGN_EXTEND : ISD::TRUNCATE);
2236 N1 = (N1.getOpcode() != ISD::Constant
2237 ? DAG.getNode(N1Opc, MVT::i16, N1)
2238 : DAG.getConstant(cast<ConstantSDNode>(N1)->getValue(), MVT::i16));
2239 return DAG.getNode(ISD::TRUNCATE, MVT::i8,
2240 DAG.getNode(Opc, MVT::i16, N0, N1));
2243 SDOperand N1 = Op.getOperand(1);
2245 N0 = (N0.getOpcode() != ISD::Constant
2246 ? DAG.getNode(ISD::SIGN_EXTEND, MVT::i16, N0)
2247 : DAG.getConstant(cast<ConstantSDNode>(N0)->getValue(), MVT::i16));
2248 N1Opc = (N1.getValueType() < MVT::i16 ? ISD::SIGN_EXTEND : ISD::TRUNCATE);
2249 N1 = (N1.getOpcode() != ISD::Constant
2250 ? DAG.getNode(N1Opc, MVT::i16, N1)
2251 : DAG.getConstant(cast<ConstantSDNode>(N1)->getValue(), MVT::i16));
2252 return DAG.getNode(ISD::TRUNCATE, MVT::i8,
2253 DAG.getNode(Opc, MVT::i16, N0, N1));
2261 static SDOperand LowerI64Math(SDOperand Op, SelectionDAG &DAG, unsigned Opc)
2263 MVT::ValueType VT = Op.getValueType();
2265 MVT::getVectorType(VT, (128 / MVT::getSizeInBits(VT)));
2267 SDOperand Op0 = Op.getOperand(0);
2270 case ISD::ZERO_EXTEND:
2271 case ISD::SIGN_EXTEND:
2272 case ISD::ANY_EXTEND: {
2273 MVT::ValueType Op0VT = Op0.getValueType();
2275 MVT::getVectorType(Op0VT, (128 / MVT::getSizeInBits(Op0VT)));
2277 assert(Op0VT == MVT::i32
2278 && "CellSPU: Zero/sign extending something other than i32");
2280 unsigned NewOpc = (Opc == ISD::SIGN_EXTEND
2281 ? SPUISD::ROTBYTES_RIGHT_S
2282 : SPUISD::ROTQUAD_RZ_BYTES);
2283 SDOperand PromoteScalar =
2284 DAG.getNode(SPUISD::PROMOTE_SCALAR, Op0VecVT, Op0);
2286 return DAG.getNode(SPUISD::EXTRACT_ELT0, VT,
2287 DAG.getNode(ISD::BIT_CONVERT, VecVT,
2288 DAG.getNode(NewOpc, Op0VecVT,
2290 DAG.getConstant(4, MVT::i32))));
2294 SDOperand ShiftAmt = Op.getOperand(1);
2295 unsigned ShiftAmtVT = unsigned(ShiftAmt.getValueType());
2296 SDOperand Op0Vec = DAG.getNode(SPUISD::PROMOTE_SCALAR, VecVT, Op0);
2297 SDOperand MaskLower =
2298 DAG.getNode(SPUISD::SELB, VecVT,
2300 DAG.getConstant(0, VecVT),
2301 DAG.getNode(SPUISD::FSMBI, VecVT,
2302 DAG.getConstant(0xff00ULL, MVT::i16)));
2303 SDOperand ShiftAmtBytes =
2304 DAG.getNode(ISD::SRL, ShiftAmtVT,
2306 DAG.getConstant(3, ShiftAmtVT));
2307 SDOperand ShiftAmtBits =
2308 DAG.getNode(ISD::AND, ShiftAmtVT,
2310 DAG.getConstant(7, ShiftAmtVT));
2312 return DAG.getNode(SPUISD::EXTRACT_ELT0, VT,
2313 DAG.getNode(SPUISD::SHLQUAD_L_BITS, VecVT,
2314 DAG.getNode(SPUISD::SHLQUAD_L_BYTES, VecVT,
2315 MaskLower, ShiftAmtBytes),
2320 unsigned VT = unsigned(Op.getValueType());
2321 SDOperand ShiftAmt = Op.getOperand(1);
2322 unsigned ShiftAmtVT = unsigned(ShiftAmt.getValueType());
2323 SDOperand ShiftAmtBytes =
2324 DAG.getNode(ISD::SRL, ShiftAmtVT,
2326 DAG.getConstant(3, ShiftAmtVT));
2327 SDOperand ShiftAmtBits =
2328 DAG.getNode(ISD::AND, ShiftAmtVT,
2330 DAG.getConstant(7, ShiftAmtVT));
2332 return DAG.getNode(SPUISD::ROTQUAD_RZ_BITS, VT,
2333 DAG.getNode(SPUISD::ROTQUAD_RZ_BYTES, VT,
2334 Op0, ShiftAmtBytes),
2342 //! Lower byte immediate operations for v16i8 vectors:
2344 LowerByteImmed(SDOperand Op, SelectionDAG &DAG) {
2347 MVT::ValueType VT = Op.getValueType();
2349 ConstVec = Op.getOperand(0);
2350 Arg = Op.getOperand(1);
2351 if (ConstVec.Val->getOpcode() != ISD::BUILD_VECTOR) {
2352 if (ConstVec.Val->getOpcode() == ISD::BIT_CONVERT) {
2353 ConstVec = ConstVec.getOperand(0);
2355 ConstVec = Op.getOperand(1);
2356 Arg = Op.getOperand(0);
2357 if (ConstVec.Val->getOpcode() == ISD::BIT_CONVERT) {
2358 ConstVec = ConstVec.getOperand(0);
2363 if (ConstVec.Val->getOpcode() == ISD::BUILD_VECTOR) {
2364 uint64_t VectorBits[2];
2365 uint64_t UndefBits[2];
2366 uint64_t SplatBits, SplatUndef;
2369 if (!GetConstantBuildVectorBits(ConstVec.Val, VectorBits, UndefBits)
2370 && isConstantSplat(VectorBits, UndefBits,
2371 MVT::getSizeInBits(MVT::getVectorElementType(VT)),
2372 SplatBits, SplatUndef, SplatSize)) {
2373 SDOperand tcVec[16];
2374 SDOperand tc = DAG.getTargetConstant(SplatBits & 0xff, MVT::i8);
2375 const size_t tcVecSize = sizeof(tcVec) / sizeof(tcVec[0]);
2377 // Turn the BUILD_VECTOR into a set of target constants:
2378 for (size_t i = 0; i < tcVecSize; ++i)
2381 return DAG.getNode(Op.Val->getOpcode(), VT, Arg,
2382 DAG.getNode(ISD::BUILD_VECTOR, VT, tcVec, tcVecSize));
2389 //! Lower i32 multiplication
2390 static SDOperand LowerMUL(SDOperand Op, SelectionDAG &DAG, unsigned VT,
2394 cerr << "CellSPU: Unknown LowerMUL value type, got "
2395 << MVT::getValueTypeString(Op.getValueType())
2401 SDOperand rA = Op.getOperand(0);
2402 SDOperand rB = Op.getOperand(1);
2404 return DAG.getNode(ISD::ADD, MVT::i32,
2405 DAG.getNode(ISD::ADD, MVT::i32,
2406 DAG.getNode(SPUISD::MPYH, MVT::i32, rA, rB),
2407 DAG.getNode(SPUISD::MPYH, MVT::i32, rB, rA)),
2408 DAG.getNode(SPUISD::MPYU, MVT::i32, rA, rB));
2415 //! Custom lowering for CTPOP (count population)
2417 Custom lowering code that counts the number ones in the input
2418 operand. SPU has such an instruction, but it counts the number of
2419 ones per byte, which then have to be accumulated.
2421 static SDOperand LowerCTPOP(SDOperand Op, SelectionDAG &DAG) {
2422 unsigned VT = Op.getValueType();
2423 unsigned vecVT = MVT::getVectorType(VT, (128 / MVT::getSizeInBits(VT)));
2427 SDOperand N = Op.getOperand(0);
2428 SDOperand Elt0 = DAG.getConstant(0, MVT::i32);
2430 SDOperand Promote = DAG.getNode(SPUISD::PROMOTE_SCALAR, vecVT, N, N);
2431 SDOperand CNTB = DAG.getNode(SPUISD::CNTB, vecVT, Promote);
2433 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i8, CNTB, Elt0);
2437 MachineFunction &MF = DAG.getMachineFunction();
2438 MachineRegisterInfo &RegInfo = MF.getRegInfo();
2440 unsigned CNTB_reg = RegInfo.createVirtualRegister(&SPU::R16CRegClass);
2442 SDOperand N = Op.getOperand(0);
2443 SDOperand Elt0 = DAG.getConstant(0, MVT::i16);
2444 SDOperand Mask0 = DAG.getConstant(0x0f, MVT::i16);
2445 SDOperand Shift1 = DAG.getConstant(8, MVT::i16);
2447 SDOperand Promote = DAG.getNode(SPUISD::PROMOTE_SCALAR, vecVT, N, N);
2448 SDOperand CNTB = DAG.getNode(SPUISD::CNTB, vecVT, Promote);
2450 // CNTB_result becomes the chain to which all of the virtual registers
2451 // CNTB_reg, SUM1_reg become associated:
2452 SDOperand CNTB_result =
2453 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i16, CNTB, Elt0);
2455 SDOperand CNTB_rescopy =
2456 DAG.getCopyToReg(CNTB_result, CNTB_reg, CNTB_result);
2458 SDOperand Tmp1 = DAG.getCopyFromReg(CNTB_rescopy, CNTB_reg, MVT::i16);
2460 return DAG.getNode(ISD::AND, MVT::i16,
2461 DAG.getNode(ISD::ADD, MVT::i16,
2462 DAG.getNode(ISD::SRL, MVT::i16,
2469 MachineFunction &MF = DAG.getMachineFunction();
2470 MachineRegisterInfo &RegInfo = MF.getRegInfo();
2472 unsigned CNTB_reg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
2473 unsigned SUM1_reg = RegInfo.createVirtualRegister(&SPU::R32CRegClass);
2475 SDOperand N = Op.getOperand(0);
2476 SDOperand Elt0 = DAG.getConstant(0, MVT::i32);
2477 SDOperand Mask0 = DAG.getConstant(0xff, MVT::i32);
2478 SDOperand Shift1 = DAG.getConstant(16, MVT::i32);
2479 SDOperand Shift2 = DAG.getConstant(8, MVT::i32);
2481 SDOperand Promote = DAG.getNode(SPUISD::PROMOTE_SCALAR, vecVT, N, N);
2482 SDOperand CNTB = DAG.getNode(SPUISD::CNTB, vecVT, Promote);
2484 // CNTB_result becomes the chain to which all of the virtual registers
2485 // CNTB_reg, SUM1_reg become associated:
2486 SDOperand CNTB_result =
2487 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, MVT::i32, CNTB, Elt0);
2489 SDOperand CNTB_rescopy =
2490 DAG.getCopyToReg(CNTB_result, CNTB_reg, CNTB_result);
2493 DAG.getNode(ISD::SRL, MVT::i32,
2494 DAG.getCopyFromReg(CNTB_rescopy, CNTB_reg, MVT::i32), Shift1);
2497 DAG.getNode(ISD::ADD, MVT::i32,
2498 Comp1, DAG.getCopyFromReg(CNTB_rescopy, CNTB_reg, MVT::i32));
2500 SDOperand Sum1_rescopy =
2501 DAG.getCopyToReg(CNTB_result, SUM1_reg, Sum1);
2504 DAG.getNode(ISD::SRL, MVT::i32,
2505 DAG.getCopyFromReg(Sum1_rescopy, SUM1_reg, MVT::i32),
2508 DAG.getNode(ISD::ADD, MVT::i32, Comp2,
2509 DAG.getCopyFromReg(Sum1_rescopy, SUM1_reg, MVT::i32));
2511 return DAG.getNode(ISD::AND, MVT::i32, Sum2, Mask0);
2521 /// LowerOperation - Provide custom lowering hooks for some operations.
2524 SPUTargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG)
2526 unsigned Opc = (unsigned) Op.getOpcode();
2527 unsigned VT = (unsigned) Op.getValueType();
2531 cerr << "SPUTargetLowering::LowerOperation(): need to lower this!\n";
2532 cerr << "Op.getOpcode() = " << Opc << "\n";
2533 cerr << "*Op.Val:\n";
2540 return LowerLOAD(Op, DAG, SPUTM.getSubtargetImpl());
2542 return LowerSTORE(Op, DAG, SPUTM.getSubtargetImpl());
2543 case ISD::ConstantPool:
2544 return LowerConstantPool(Op, DAG, SPUTM.getSubtargetImpl());
2545 case ISD::GlobalAddress:
2546 return LowerGlobalAddress(Op, DAG, SPUTM.getSubtargetImpl());
2547 case ISD::JumpTable:
2548 return LowerJumpTable(Op, DAG, SPUTM.getSubtargetImpl());
2550 return LowerConstant(Op, DAG);
2551 case ISD::ConstantFP:
2552 return LowerConstantFP(Op, DAG);
2554 return LowerBRCOND(Op, DAG);
2555 case ISD::FORMAL_ARGUMENTS:
2556 return LowerFORMAL_ARGUMENTS(Op, DAG, VarArgsFrameIndex);
2558 return LowerCALL(Op, DAG, SPUTM.getSubtargetImpl());
2560 return LowerRET(Op, DAG, getTargetMachine());
2563 // i8, i64 math ops:
2564 case ISD::ZERO_EXTEND:
2565 case ISD::SIGN_EXTEND:
2566 case ISD::ANY_EXTEND:
2574 return LowerI8Math(Op, DAG, Opc);
2575 else if (VT == MVT::i64)
2576 return LowerI64Math(Op, DAG, Opc);
2579 // Vector-related lowering.
2580 case ISD::BUILD_VECTOR:
2581 return LowerBUILD_VECTOR(Op, DAG);
2582 case ISD::SCALAR_TO_VECTOR:
2583 return LowerSCALAR_TO_VECTOR(Op, DAG);
2584 case ISD::VECTOR_SHUFFLE:
2585 return LowerVECTOR_SHUFFLE(Op, DAG);
2586 case ISD::EXTRACT_VECTOR_ELT:
2587 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
2588 case ISD::INSERT_VECTOR_ELT:
2589 return LowerINSERT_VECTOR_ELT(Op, DAG);
2591 // Look for ANDBI, ORBI and XORBI opportunities and lower appropriately:
2595 return LowerByteImmed(Op, DAG);
2597 // Vector and i8 multiply:
2599 if (MVT::isVector(VT))
2600 return LowerVectorMUL(Op, DAG);
2601 else if (VT == MVT::i8)
2602 return LowerI8Math(Op, DAG, Opc);
2604 return LowerMUL(Op, DAG, VT, Opc);
2607 if (VT == MVT::f32 || VT == MVT::v4f32)
2608 return LowerFDIVf32(Op, DAG);
2609 // else if (Op.getValueType() == MVT::f64)
2610 // return LowerFDIVf64(Op, DAG);
2612 assert(0 && "Calling FDIV on unsupported MVT");
2615 return LowerCTPOP(Op, DAG);
2621 //===----------------------------------------------------------------------===//
2622 // Target Optimization Hooks
2623 //===----------------------------------------------------------------------===//
2626 SPUTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
2629 TargetMachine &TM = getTargetMachine();
2631 const SPUSubtarget *ST = SPUTM.getSubtargetImpl();
2632 SelectionDAG &DAG = DCI.DAG;
2633 SDOperand Op0 = N->getOperand(0); // everything has at least one operand
2634 SDOperand Result; // Initially, NULL result
2636 switch (N->getOpcode()) {
2639 SDOperand Op1 = N->getOperand(1);
2641 if ((Op1.getOpcode() == ISD::Constant
2642 || Op1.getOpcode() == ISD::TargetConstant)
2643 && Op0.getOpcode() == SPUISD::IndirectAddr) {
2644 SDOperand Op01 = Op0.getOperand(1);
2645 if (Op01.getOpcode() == ISD::Constant
2646 || Op01.getOpcode() == ISD::TargetConstant) {
2647 // (add <const>, (SPUindirect <arg>, <const>)) ->
2648 // (SPUindirect <arg>, <const + const>)
2649 ConstantSDNode *CN0 = cast<ConstantSDNode>(Op1);
2650 ConstantSDNode *CN1 = cast<ConstantSDNode>(Op01);
2651 SDOperand combinedConst =
2652 DAG.getConstant(CN0->getValue() + CN1->getValue(),
2653 Op0.getValueType());
2655 DEBUG(cerr << "Replace: (add " << CN0->getValue() << ", "
2656 << "(SPUindirect <arg>, " << CN1->getValue() << "))\n");
2657 DEBUG(cerr << "With: (SPUindirect <arg>, "
2658 << CN0->getValue() + CN1->getValue() << ")\n");
2659 return DAG.getNode(SPUISD::IndirectAddr, Op0.getValueType(),
2660 Op0.getOperand(0), combinedConst);
2662 } else if ((Op0.getOpcode() == ISD::Constant
2663 || Op0.getOpcode() == ISD::TargetConstant)
2664 && Op1.getOpcode() == SPUISD::IndirectAddr) {
2665 SDOperand Op11 = Op1.getOperand(1);
2666 if (Op11.getOpcode() == ISD::Constant
2667 || Op11.getOpcode() == ISD::TargetConstant) {
2668 // (add (SPUindirect <arg>, <const>), <const>) ->
2669 // (SPUindirect <arg>, <const + const>)
2670 ConstantSDNode *CN0 = cast<ConstantSDNode>(Op0);
2671 ConstantSDNode *CN1 = cast<ConstantSDNode>(Op11);
2672 SDOperand combinedConst =
2673 DAG.getConstant(CN0->getValue() + CN1->getValue(),
2674 Op0.getValueType());
2676 DEBUG(cerr << "Replace: (add " << CN0->getValue() << ", "
2677 << "(SPUindirect <arg>, " << CN1->getValue() << "))\n");
2678 DEBUG(cerr << "With: (SPUindirect <arg>, "
2679 << CN0->getValue() + CN1->getValue() << ")\n");
2681 return DAG.getNode(SPUISD::IndirectAddr, Op1.getValueType(),
2682 Op1.getOperand(0), combinedConst);
2687 case ISD::SIGN_EXTEND:
2688 case ISD::ZERO_EXTEND:
2689 case ISD::ANY_EXTEND: {
2690 if (Op0.getOpcode() == SPUISD::EXTRACT_ELT0 &&
2691 N->getValueType(0) == Op0.getValueType()) {
2692 // (any_extend (SPUextract_elt0 <arg>)) ->
2693 // (SPUextract_elt0 <arg>)
2694 // Types must match, however...
2695 DEBUG(cerr << "Replace: ");
2696 DEBUG(N->dump(&DAG));
2697 DEBUG(cerr << "\nWith: ");
2698 DEBUG(Op0.Val->dump(&DAG));
2699 DEBUG(cerr << "\n");
2705 case SPUISD::IndirectAddr: {
2706 if (!ST->usingLargeMem() && Op0.getOpcode() == SPUISD::AFormAddr) {
2707 ConstantSDNode *CN = cast<ConstantSDNode>(N->getOperand(1));
2708 if (CN->getValue() == 0) {
2709 // (SPUindirect (SPUaform <addr>, 0), 0) ->
2710 // (SPUaform <addr>, 0)
2712 DEBUG(cerr << "Replace: ");
2713 DEBUG(N->dump(&DAG));
2714 DEBUG(cerr << "\nWith: ");
2715 DEBUG(Op0.Val->dump(&DAG));
2716 DEBUG(cerr << "\n");
2723 case SPUISD::SHLQUAD_L_BITS:
2724 case SPUISD::SHLQUAD_L_BYTES:
2725 case SPUISD::VEC_SHL:
2726 case SPUISD::VEC_SRL:
2727 case SPUISD::VEC_SRA:
2728 case SPUISD::ROTQUAD_RZ_BYTES:
2729 case SPUISD::ROTQUAD_RZ_BITS: {
2730 SDOperand Op1 = N->getOperand(1);
2732 if (isa<ConstantSDNode>(Op1)) {
2733 // Kill degenerate vector shifts:
2734 ConstantSDNode *CN = cast<ConstantSDNode>(Op1);
2736 if (CN->getValue() == 0) {
2742 case SPUISD::PROMOTE_SCALAR: {
2743 switch (Op0.getOpcode()) {
2746 case ISD::ANY_EXTEND:
2747 case ISD::ZERO_EXTEND:
2748 case ISD::SIGN_EXTEND: {
2749 // (SPUpromote_scalar (any|sign|zero_extend (SPUextract_elt0 <arg>))) ->
2751 // but only if the SPUpromote_scalar and <arg> types match.
2752 SDOperand Op00 = Op0.getOperand(0);
2753 if (Op00.getOpcode() == SPUISD::EXTRACT_ELT0) {
2754 SDOperand Op000 = Op00.getOperand(0);
2755 if (Op000.getValueType() == N->getValueType(0)) {
2761 case SPUISD::EXTRACT_ELT0: {
2762 // (SPUpromote_scalar (SPUextract_elt0 <arg>)) ->
2764 Result = Op0.getOperand(0);
2771 // Otherwise, return unchanged.
2774 DEBUG(cerr << "\nReplace.SPU: ");
2775 DEBUG(N->dump(&DAG));
2776 DEBUG(cerr << "\nWith: ");
2777 DEBUG(Result.Val->dump(&DAG));
2778 DEBUG(cerr << "\n");
2785 //===----------------------------------------------------------------------===//
2786 // Inline Assembly Support
2787 //===----------------------------------------------------------------------===//
2789 /// getConstraintType - Given a constraint letter, return the type of
2790 /// constraint it is for this target.
2791 SPUTargetLowering::ConstraintType
2792 SPUTargetLowering::getConstraintType(const std::string &ConstraintLetter) const {
2793 if (ConstraintLetter.size() == 1) {
2794 switch (ConstraintLetter[0]) {
2801 return C_RegisterClass;
2804 return TargetLowering::getConstraintType(ConstraintLetter);
2807 std::pair<unsigned, const TargetRegisterClass*>
2808 SPUTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
2809 MVT::ValueType VT) const
2811 if (Constraint.size() == 1) {
2812 // GCC RS6000 Constraint Letters
2813 switch (Constraint[0]) {
2817 return std::make_pair(0U, SPU::R64CRegisterClass);
2818 return std::make_pair(0U, SPU::R32CRegisterClass);
2821 return std::make_pair(0U, SPU::R32FPRegisterClass);
2822 else if (VT == MVT::f64)
2823 return std::make_pair(0U, SPU::R64FPRegisterClass);
2826 return std::make_pair(0U, SPU::GPRCRegisterClass);
2830 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
2833 //! Compute used/known bits for a SPU operand
2835 SPUTargetLowering::computeMaskedBitsForTargetNode(const SDOperand Op,
2839 const SelectionDAG &DAG,
2840 unsigned Depth ) const {
2841 const uint64_t uint64_sizebits = sizeof(uint64_t) * 8;
2843 switch (Op.getOpcode()) {
2845 // KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0);
2855 case SPUISD::PROMOTE_SCALAR: {
2856 SDOperand Op0 = Op.getOperand(0);
2857 uint64_t InMask = MVT::getIntVTBitMask(Op0.getValueType());
2858 KnownZero |= APInt(uint64_sizebits, ~InMask, false);
2859 KnownOne |= APInt(uint64_sizebits, InMask, false);
2863 case SPUISD::LDRESULT:
2864 case SPUISD::EXTRACT_ELT0:
2865 case SPUISD::EXTRACT_ELT0_CHAINED: {
2866 uint64_t InMask = MVT::getIntVTBitMask(Op.getValueType());
2867 KnownZero |= APInt(uint64_sizebits, ~InMask, false);
2868 KnownOne |= APInt(uint64_sizebits, InMask, false);
2873 case EXTRACT_I1_ZEXT:
2874 case EXTRACT_I1_SEXT:
2875 case EXTRACT_I8_ZEXT:
2876 case EXTRACT_I8_SEXT:
2881 case SHLQUAD_L_BITS:
2882 case SHLQUAD_L_BYTES:
2888 case ROTQUAD_RZ_BYTES:
2889 case ROTQUAD_RZ_BITS:
2890 case ROTBYTES_RIGHT_S:
2892 case ROTBYTES_LEFT_CHAINED:
2903 // LowerAsmOperandForConstraint
2905 SPUTargetLowering::LowerAsmOperandForConstraint(SDOperand Op,
2906 char ConstraintLetter,
2907 std::vector<SDOperand> &Ops,
2908 SelectionDAG &DAG) {
2909 // Default, for the time being, to the base class handler
2910 TargetLowering::LowerAsmOperandForConstraint(Op, ConstraintLetter, Ops, DAG);
2913 /// isLegalAddressImmediate - Return true if the integer value can be used
2914 /// as the offset of the target addressing mode.
2915 bool SPUTargetLowering::isLegalAddressImmediate(int64_t V, const Type *Ty) const {
2916 // SPU's addresses are 256K:
2917 return (V > -(1 << 18) && V < (1 << 18) - 1);
2920 bool SPUTargetLowering::isLegalAddressImmediate(llvm::GlobalValue* GV) const {