1 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This implements the SelectionDAG class.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/CodeGen/SelectionDAG.h"
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/APSInt.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/ADT/StringExtras.h"
22 #include "llvm/Analysis/ValueTracking.h"
23 #include "llvm/CodeGen/MachineBasicBlock.h"
24 #include "llvm/CodeGen/MachineConstantPool.h"
25 #include "llvm/CodeGen/MachineFrameInfo.h"
26 #include "llvm/CodeGen/MachineModuleInfo.h"
27 #include "llvm/IR/CallingConv.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DataLayout.h"
30 #include "llvm/IR/DebugInfo.h"
31 #include "llvm/IR/DerivedTypes.h"
32 #include "llvm/IR/Function.h"
33 #include "llvm/IR/GlobalAlias.h"
34 #include "llvm/IR/GlobalVariable.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/Support/CommandLine.h"
37 #include "llvm/Support/Debug.h"
38 #include "llvm/Support/ErrorHandling.h"
39 #include "llvm/Support/ManagedStatic.h"
40 #include "llvm/Support/MathExtras.h"
41 #include "llvm/Support/Mutex.h"
42 #include "llvm/Support/raw_ostream.h"
43 #include "llvm/Target/TargetInstrInfo.h"
44 #include "llvm/Target/TargetIntrinsicInfo.h"
45 #include "llvm/Target/TargetLowering.h"
46 #include "llvm/Target/TargetMachine.h"
47 #include "llvm/Target/TargetOptions.h"
48 #include "llvm/Target/TargetRegisterInfo.h"
49 #include "llvm/Target/TargetSelectionDAGInfo.h"
50 #include "llvm/Target/TargetSubtargetInfo.h"
57 /// makeVTList - Return an instance of the SDVTList struct initialized with the
58 /// specified members.
59 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
60 SDVTList Res = {VTs, NumVTs};
64 // Default null implementations of the callbacks.
65 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
66 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
68 //===----------------------------------------------------------------------===//
69 // ConstantFPSDNode Class
70 //===----------------------------------------------------------------------===//
72 /// isExactlyValue - We don't rely on operator== working on double values, as
73 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
74 /// As such, this method can be used to do an exact bit-for-bit comparison of
75 /// two floating point values.
76 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
77 return getValueAPF().bitwiseIsEqual(V);
80 bool ConstantFPSDNode::isValueValidForType(EVT VT,
82 assert(VT.isFloatingPoint() && "Can only convert between FP types");
84 // convert modifies in place, so make a copy.
85 APFloat Val2 = APFloat(Val);
87 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
88 APFloat::rmNearestTiesToEven,
93 //===----------------------------------------------------------------------===//
95 //===----------------------------------------------------------------------===//
97 /// isBuildVectorAllOnes - Return true if the specified node is a
98 /// BUILD_VECTOR where all of the elements are ~0 or undef.
99 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
100 // Look through a bit convert.
101 while (N->getOpcode() == ISD::BITCAST)
102 N = N->getOperand(0).getNode();
104 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
106 unsigned i = 0, e = N->getNumOperands();
108 // Skip over all of the undef values.
109 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
112 // Do not accept an all-undef vector.
113 if (i == e) return false;
115 // Do not accept build_vectors that aren't all constants or which have non-~0
116 // elements. We have to be a bit careful here, as the type of the constant
117 // may not be the same as the type of the vector elements due to type
118 // legalization (the elements are promoted to a legal type for the target and
119 // a vector of a type may be legal when the base element type is not).
120 // We only want to check enough bits to cover the vector elements, because
121 // we care if the resultant vector is all ones, not whether the individual
123 SDValue NotZero = N->getOperand(i);
124 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
125 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
126 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
128 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
129 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
134 // Okay, we have at least one ~0 value, check to see if the rest match or are
135 // undefs. Even with the above element type twiddling, this should be OK, as
136 // the same type legalization should have applied to all the elements.
137 for (++i; i != e; ++i)
138 if (N->getOperand(i) != NotZero &&
139 N->getOperand(i).getOpcode() != ISD::UNDEF)
145 /// isBuildVectorAllZeros - Return true if the specified node is a
146 /// BUILD_VECTOR where all of the elements are 0 or undef.
147 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
148 // Look through a bit convert.
149 while (N->getOpcode() == ISD::BITCAST)
150 N = N->getOperand(0).getNode();
152 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
154 bool IsAllUndef = true;
155 for (const SDValue &Op : N->op_values()) {
156 if (Op.getOpcode() == ISD::UNDEF)
159 // Do not accept build_vectors that aren't all constants or which have non-0
160 // elements. We have to be a bit careful here, as the type of the constant
161 // may not be the same as the type of the vector elements due to type
162 // legalization (the elements are promoted to a legal type for the target
163 // and a vector of a type may be legal when the base element type is not).
164 // We only want to check enough bits to cover the vector elements, because
165 // we care if the resultant vector is all zeros, not whether the individual
167 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
168 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
169 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
171 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
172 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
178 // Do not accept an all-undef vector.
184 /// \brief Return true if the specified node is a BUILD_VECTOR node of
185 /// all ConstantSDNode or undef.
186 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
187 if (N->getOpcode() != ISD::BUILD_VECTOR)
190 for (const SDValue &Op : N->op_values()) {
191 if (Op.getOpcode() == ISD::UNDEF)
193 if (!isa<ConstantSDNode>(Op))
199 /// \brief Return true if the specified node is a BUILD_VECTOR node of
200 /// all ConstantFPSDNode or undef.
201 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
202 if (N->getOpcode() != ISD::BUILD_VECTOR)
205 for (const SDValue &Op : N->op_values()) {
206 if (Op.getOpcode() == ISD::UNDEF)
208 if (!isa<ConstantFPSDNode>(Op))
214 /// allOperandsUndef - Return true if the node has at least one operand
215 /// and all operands of the specified node are ISD::UNDEF.
216 bool ISD::allOperandsUndef(const SDNode *N) {
217 // Return false if the node has no operands.
218 // This is "logically inconsistent" with the definition of "all" but
219 // is probably the desired behavior.
220 if (N->getNumOperands() == 0)
223 for (const SDValue &Op : N->op_values())
224 if (Op.getOpcode() != ISD::UNDEF)
230 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
233 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
235 return ISD::SIGN_EXTEND;
237 return ISD::ZERO_EXTEND;
242 llvm_unreachable("Invalid LoadExtType");
245 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
246 /// when given the operation for (X op Y).
247 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
248 // To perform this operation, we just need to swap the L and G bits of the
250 unsigned OldL = (Operation >> 2) & 1;
251 unsigned OldG = (Operation >> 1) & 1;
252 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
253 (OldL << 1) | // New G bit
254 (OldG << 2)); // New L bit.
257 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
258 /// 'op' is a valid SetCC operation.
259 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
260 unsigned Operation = Op;
262 Operation ^= 7; // Flip L, G, E bits, but not U.
264 Operation ^= 15; // Flip all of the condition bits.
266 if (Operation > ISD::SETTRUE2)
267 Operation &= ~8; // Don't let N and U bits get set.
269 return ISD::CondCode(Operation);
273 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
274 /// signed operation and 2 if the result is an unsigned comparison. Return zero
275 /// if the operation does not depend on the sign of the input (setne and seteq).
276 static int isSignedOp(ISD::CondCode Opcode) {
278 default: llvm_unreachable("Illegal integer setcc operation!");
280 case ISD::SETNE: return 0;
284 case ISD::SETGE: return 1;
288 case ISD::SETUGE: return 2;
292 /// getSetCCOrOperation - Return the result of a logical OR between different
293 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function
294 /// returns SETCC_INVALID if it is not possible to represent the resultant
296 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
298 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
299 // Cannot fold a signed integer setcc with an unsigned integer setcc.
300 return ISD::SETCC_INVALID;
302 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
304 // If the N and U bits get set then the resultant comparison DOES suddenly
305 // care about orderedness, and is true when ordered.
306 if (Op > ISD::SETTRUE2)
307 Op &= ~16; // Clear the U bit if the N bit is set.
309 // Canonicalize illegal integer setcc's.
310 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
313 return ISD::CondCode(Op);
316 /// getSetCCAndOperation - Return the result of a logical AND between different
317 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This
318 /// function returns zero if it is not possible to represent the resultant
320 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
322 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
323 // Cannot fold a signed setcc with an unsigned setcc.
324 return ISD::SETCC_INVALID;
326 // Combine all of the condition bits.
327 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
329 // Canonicalize illegal integer setcc's.
333 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
334 case ISD::SETOEQ: // SETEQ & SETU[LG]E
335 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
336 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
337 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
344 //===----------------------------------------------------------------------===//
345 // SDNode Profile Support
346 //===----------------------------------------------------------------------===//
348 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
350 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
354 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
355 /// solely with their pointer.
356 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
357 ID.AddPointer(VTList.VTs);
360 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
362 static void AddNodeIDOperands(FoldingSetNodeID &ID,
363 ArrayRef<SDValue> Ops) {
364 for (auto& Op : Ops) {
365 ID.AddPointer(Op.getNode());
366 ID.AddInteger(Op.getResNo());
370 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
372 static void AddNodeIDOperands(FoldingSetNodeID &ID,
373 ArrayRef<SDUse> Ops) {
374 for (auto& Op : Ops) {
375 ID.AddPointer(Op.getNode());
376 ID.AddInteger(Op.getResNo());
380 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
381 SDVTList VTList, ArrayRef<SDValue> OpList) {
382 AddNodeIDOpcode(ID, OpC);
383 AddNodeIDValueTypes(ID, VTList);
384 AddNodeIDOperands(ID, OpList);
387 /// If this is an SDNode with special info, add this info to the NodeID data.
388 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
389 switch (N->getOpcode()) {
390 case ISD::TargetExternalSymbol:
391 case ISD::ExternalSymbol:
393 llvm_unreachable("Should only be used on nodes with operands");
394 default: break; // Normal nodes don't need extra info.
395 case ISD::TargetConstant:
396 case ISD::Constant: {
397 const ConstantSDNode *C = cast<ConstantSDNode>(N);
398 ID.AddPointer(C->getConstantIntValue());
399 ID.AddBoolean(C->isOpaque());
402 case ISD::TargetConstantFP:
403 case ISD::ConstantFP: {
404 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
407 case ISD::TargetGlobalAddress:
408 case ISD::GlobalAddress:
409 case ISD::TargetGlobalTLSAddress:
410 case ISD::GlobalTLSAddress: {
411 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
412 ID.AddPointer(GA->getGlobal());
413 ID.AddInteger(GA->getOffset());
414 ID.AddInteger(GA->getTargetFlags());
415 ID.AddInteger(GA->getAddressSpace());
418 case ISD::BasicBlock:
419 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
422 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
424 case ISD::RegisterMask:
425 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
428 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
430 case ISD::FrameIndex:
431 case ISD::TargetFrameIndex:
432 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
435 case ISD::TargetJumpTable:
436 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
437 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
439 case ISD::ConstantPool:
440 case ISD::TargetConstantPool: {
441 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
442 ID.AddInteger(CP->getAlignment());
443 ID.AddInteger(CP->getOffset());
444 if (CP->isMachineConstantPoolEntry())
445 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
447 ID.AddPointer(CP->getConstVal());
448 ID.AddInteger(CP->getTargetFlags());
451 case ISD::TargetIndex: {
452 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
453 ID.AddInteger(TI->getIndex());
454 ID.AddInteger(TI->getOffset());
455 ID.AddInteger(TI->getTargetFlags());
459 const LoadSDNode *LD = cast<LoadSDNode>(N);
460 ID.AddInteger(LD->getMemoryVT().getRawBits());
461 ID.AddInteger(LD->getRawSubclassData());
462 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
466 const StoreSDNode *ST = cast<StoreSDNode>(N);
467 ID.AddInteger(ST->getMemoryVT().getRawBits());
468 ID.AddInteger(ST->getRawSubclassData());
469 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
472 case ISD::ATOMIC_CMP_SWAP:
473 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
474 case ISD::ATOMIC_SWAP:
475 case ISD::ATOMIC_LOAD_ADD:
476 case ISD::ATOMIC_LOAD_SUB:
477 case ISD::ATOMIC_LOAD_AND:
478 case ISD::ATOMIC_LOAD_OR:
479 case ISD::ATOMIC_LOAD_XOR:
480 case ISD::ATOMIC_LOAD_NAND:
481 case ISD::ATOMIC_LOAD_MIN:
482 case ISD::ATOMIC_LOAD_MAX:
483 case ISD::ATOMIC_LOAD_UMIN:
484 case ISD::ATOMIC_LOAD_UMAX:
485 case ISD::ATOMIC_LOAD:
486 case ISD::ATOMIC_STORE: {
487 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
488 ID.AddInteger(AT->getMemoryVT().getRawBits());
489 ID.AddInteger(AT->getRawSubclassData());
490 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
493 case ISD::PREFETCH: {
494 const MemSDNode *PF = cast<MemSDNode>(N);
495 ID.AddInteger(PF->getPointerInfo().getAddrSpace());
498 case ISD::VECTOR_SHUFFLE: {
499 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
500 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
502 ID.AddInteger(SVN->getMaskElt(i));
505 case ISD::TargetBlockAddress:
506 case ISD::BlockAddress: {
507 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
508 ID.AddPointer(BA->getBlockAddress());
509 ID.AddInteger(BA->getOffset());
510 ID.AddInteger(BA->getTargetFlags());
513 } // end switch (N->getOpcode())
515 // Target specific memory nodes could also have address spaces to check.
516 if (N->isTargetMemoryOpcode())
517 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
520 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
522 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
523 AddNodeIDOpcode(ID, N->getOpcode());
524 // Add the return value info.
525 AddNodeIDValueTypes(ID, N->getVTList());
526 // Add the operand info.
527 AddNodeIDOperands(ID, N->ops());
529 // Handle SDNode leafs with special info.
530 AddNodeIDCustom(ID, N);
533 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
534 /// the CSE map that carries volatility, temporalness, indexing mode, and
535 /// extension/truncation information.
537 static inline unsigned
538 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
539 bool isNonTemporal, bool isInvariant) {
540 assert((ConvType & 3) == ConvType &&
541 "ConvType may not require more than 2 bits!");
542 assert((AM & 7) == AM &&
543 "AM may not require more than 3 bits!");
547 (isNonTemporal << 6) |
551 //===----------------------------------------------------------------------===//
552 // SelectionDAG Class
553 //===----------------------------------------------------------------------===//
555 /// doNotCSE - Return true if CSE should not be performed for this node.
556 static bool doNotCSE(SDNode *N) {
557 if (N->getValueType(0) == MVT::Glue)
558 return true; // Never CSE anything that produces a flag.
560 switch (N->getOpcode()) {
562 case ISD::HANDLENODE:
564 return true; // Never CSE these nodes.
567 // Check that remaining values produced are not flags.
568 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
569 if (N->getValueType(i) == MVT::Glue)
570 return true; // Never CSE anything that produces a flag.
575 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
577 void SelectionDAG::RemoveDeadNodes() {
578 // Create a dummy node (which is not added to allnodes), that adds a reference
579 // to the root node, preventing it from being deleted.
580 HandleSDNode Dummy(getRoot());
582 SmallVector<SDNode*, 128> DeadNodes;
584 // Add all obviously-dead nodes to the DeadNodes worklist.
585 for (SDNode &Node : allnodes())
586 if (Node.use_empty())
587 DeadNodes.push_back(&Node);
589 RemoveDeadNodes(DeadNodes);
591 // If the root changed (e.g. it was a dead load, update the root).
592 setRoot(Dummy.getValue());
595 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
596 /// given list, and any nodes that become unreachable as a result.
597 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
599 // Process the worklist, deleting the nodes and adding their uses to the
601 while (!DeadNodes.empty()) {
602 SDNode *N = DeadNodes.pop_back_val();
604 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
605 DUL->NodeDeleted(N, nullptr);
607 // Take the node out of the appropriate CSE map.
608 RemoveNodeFromCSEMaps(N);
610 // Next, brutally remove the operand list. This is safe to do, as there are
611 // no cycles in the graph.
612 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
614 SDNode *Operand = Use.getNode();
617 // Now that we removed this operand, see if there are no uses of it left.
618 if (Operand->use_empty())
619 DeadNodes.push_back(Operand);
626 void SelectionDAG::RemoveDeadNode(SDNode *N){
627 SmallVector<SDNode*, 16> DeadNodes(1, N);
629 // Create a dummy node that adds a reference to the root node, preventing
630 // it from being deleted. (This matters if the root is an operand of the
632 HandleSDNode Dummy(getRoot());
634 RemoveDeadNodes(DeadNodes);
637 void SelectionDAG::DeleteNode(SDNode *N) {
638 // First take this out of the appropriate CSE map.
639 RemoveNodeFromCSEMaps(N);
641 // Finally, remove uses due to operands of this node, remove from the
642 // AllNodes list, and delete the node.
643 DeleteNodeNotInCSEMaps(N);
646 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
647 assert(N != AllNodes.begin() && "Cannot delete the entry node!");
648 assert(N->use_empty() && "Cannot delete a node that is not dead!");
650 // Drop all of the operands and decrement used node's use counts.
656 void SDDbgInfo::erase(const SDNode *Node) {
657 DbgValMapType::iterator I = DbgValMap.find(Node);
658 if (I == DbgValMap.end())
660 for (auto &Val: I->second)
661 Val->setIsInvalidated();
665 void SelectionDAG::DeallocateNode(SDNode *N) {
666 if (N->OperandsNeedDelete)
667 delete[] N->OperandList;
669 // Set the opcode to DELETED_NODE to help catch bugs when node
670 // memory is reallocated.
671 N->NodeType = ISD::DELETED_NODE;
673 NodeAllocator.Deallocate(AllNodes.remove(N));
675 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
676 // them and forget about that node.
681 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
682 static void VerifySDNode(SDNode *N) {
683 switch (N->getOpcode()) {
686 case ISD::BUILD_PAIR: {
687 EVT VT = N->getValueType(0);
688 assert(N->getNumValues() == 1 && "Too many results!");
689 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
690 "Wrong return type!");
691 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
692 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
693 "Mismatched operand types!");
694 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
695 "Wrong operand type!");
696 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
697 "Wrong return type size");
700 case ISD::BUILD_VECTOR: {
701 assert(N->getNumValues() == 1 && "Too many results!");
702 assert(N->getValueType(0).isVector() && "Wrong return type!");
703 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
704 "Wrong number of operands!");
705 EVT EltVT = N->getValueType(0).getVectorElementType();
706 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
707 assert((I->getValueType() == EltVT ||
708 (EltVT.isInteger() && I->getValueType().isInteger() &&
709 EltVT.bitsLE(I->getValueType()))) &&
710 "Wrong operand type!");
711 assert(I->getValueType() == N->getOperand(0).getValueType() &&
712 "Operands must all have the same type");
720 /// \brief Insert a newly allocated node into the DAG.
722 /// Handles insertion into the all nodes list and CSE map, as well as
723 /// verification and other common operations when a new node is allocated.
724 void SelectionDAG::InsertNode(SDNode *N) {
725 AllNodes.push_back(N);
727 N->PersistentId = NextPersistentId++;
732 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
733 /// correspond to it. This is useful when we're about to delete or repurpose
734 /// the node. We don't want future request for structurally identical nodes
735 /// to return N anymore.
736 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
738 switch (N->getOpcode()) {
739 case ISD::HANDLENODE: return false; // noop.
741 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
742 "Cond code doesn't exist!");
743 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
744 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
746 case ISD::ExternalSymbol:
747 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
749 case ISD::TargetExternalSymbol: {
750 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
751 Erased = TargetExternalSymbols.erase(
752 std::pair<std::string,unsigned char>(ESN->getSymbol(),
753 ESN->getTargetFlags()));
756 case ISD::MCSymbol: {
757 auto *MCSN = cast<MCSymbolSDNode>(N);
758 Erased = MCSymbols.erase(MCSN->getMCSymbol());
761 case ISD::VALUETYPE: {
762 EVT VT = cast<VTSDNode>(N)->getVT();
763 if (VT.isExtended()) {
764 Erased = ExtendedValueTypeNodes.erase(VT);
766 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
767 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
772 // Remove it from the CSE Map.
773 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
774 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
775 Erased = CSEMap.RemoveNode(N);
779 // Verify that the node was actually in one of the CSE maps, unless it has a
780 // flag result (which cannot be CSE'd) or is one of the special cases that are
781 // not subject to CSE.
782 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
783 !N->isMachineOpcode() && !doNotCSE(N)) {
786 llvm_unreachable("Node is not in map!");
792 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
793 /// maps and modified in place. Add it back to the CSE maps, unless an identical
794 /// node already exists, in which case transfer all its users to the existing
795 /// node. This transfer can potentially trigger recursive merging.
798 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
799 // For node types that aren't CSE'd, just act as if no identical node
802 SDNode *Existing = CSEMap.GetOrInsertNode(N);
804 // If there was already an existing matching node, use ReplaceAllUsesWith
805 // to replace the dead one with the existing one. This can cause
806 // recursive merging of other unrelated nodes down the line.
807 ReplaceAllUsesWith(N, Existing);
809 // N is now dead. Inform the listeners and delete it.
810 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
811 DUL->NodeDeleted(N, Existing);
812 DeleteNodeNotInCSEMaps(N);
817 // If the node doesn't already exist, we updated it. Inform listeners.
818 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
822 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
823 /// were replaced with those specified. If this node is never memoized,
824 /// return null, otherwise return a pointer to the slot it would take. If a
825 /// node already exists with these operands, the slot will be non-null.
826 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
831 SDValue Ops[] = { Op };
833 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
834 AddNodeIDCustom(ID, N);
835 SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
837 if (const SDNodeFlags *Flags = N->getFlags())
838 Node->intersectFlagsWith(Flags);
842 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
843 /// were replaced with those specified. If this node is never memoized,
844 /// return null, otherwise return a pointer to the slot it would take. If a
845 /// node already exists with these operands, the slot will be non-null.
846 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
847 SDValue Op1, SDValue Op2,
852 SDValue Ops[] = { Op1, Op2 };
854 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
855 AddNodeIDCustom(ID, N);
856 SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
858 if (const SDNodeFlags *Flags = N->getFlags())
859 Node->intersectFlagsWith(Flags);
864 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
865 /// were replaced with those specified. If this node is never memoized,
866 /// return null, otherwise return a pointer to the slot it would take. If a
867 /// node already exists with these operands, the slot will be non-null.
868 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
874 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
875 AddNodeIDCustom(ID, N);
876 SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
878 if (const SDNodeFlags *Flags = N->getFlags())
879 Node->intersectFlagsWith(Flags);
883 /// getEVTAlignment - Compute the default alignment value for the
886 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
887 Type *Ty = VT == MVT::iPTR ?
888 PointerType::get(Type::getInt8Ty(*getContext()), 0) :
889 VT.getTypeForEVT(*getContext());
891 return getDataLayout().getABITypeAlignment(Ty);
894 // EntryNode could meaningfully have debug info if we can find it...
895 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
896 : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
897 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
898 Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
899 UpdateListeners(nullptr) {
900 InsertNode(&EntryNode);
901 DbgInfo = new SDDbgInfo();
904 void SelectionDAG::init(MachineFunction &mf) {
906 TLI = getSubtarget().getTargetLowering();
907 TSI = getSubtarget().getSelectionDAGInfo();
908 Context = &mf.getFunction()->getContext();
911 SelectionDAG::~SelectionDAG() {
912 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
917 void SelectionDAG::allnodes_clear() {
918 assert(&*AllNodes.begin() == &EntryNode);
919 AllNodes.remove(AllNodes.begin());
920 while (!AllNodes.empty())
921 DeallocateNode(&AllNodes.front());
923 NextPersistentId = 0;
927 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
928 SDVTList VTs, SDValue N1,
930 const SDNodeFlags *Flags) {
931 if (isBinOpWithFlags(Opcode)) {
932 // If no flags were passed in, use a default flags object.
934 if (Flags == nullptr)
937 BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
938 Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
943 BinarySDNode *N = new (NodeAllocator)
944 BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
948 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
950 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
952 switch (N->getOpcode()) {
955 case ISD::ConstantFP:
956 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
957 "debug location. Use another overload.");
963 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
964 DebugLoc DL, void *&InsertPos) {
965 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
967 switch (N->getOpcode()) {
968 default: break; // Process only regular (non-target) constant nodes.
970 case ISD::ConstantFP:
971 // Erase debug location from the node if the node is used at several
972 // different places to do not propagate one location to all uses as it
973 // leads to incorrect debug info.
974 if (N->getDebugLoc() != DL)
975 N->setDebugLoc(DebugLoc());
982 void SelectionDAG::clear() {
984 OperandAllocator.Reset();
987 ExtendedValueTypeNodes.clear();
988 ExternalSymbols.clear();
989 TargetExternalSymbols.clear();
991 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
992 static_cast<CondCodeSDNode*>(nullptr));
993 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
994 static_cast<SDNode*>(nullptr));
996 EntryNode.UseList = nullptr;
997 InsertNode(&EntryNode);
998 Root = getEntryNode();
1002 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
1003 return VT.bitsGT(Op.getValueType()) ?
1004 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1005 getNode(ISD::TRUNCATE, DL, VT, Op);
1008 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
1009 return VT.bitsGT(Op.getValueType()) ?
1010 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1011 getNode(ISD::TRUNCATE, DL, VT, Op);
1014 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
1015 return VT.bitsGT(Op.getValueType()) ?
1016 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1017 getNode(ISD::TRUNCATE, DL, VT, Op);
1020 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
1022 if (VT.bitsLE(Op.getValueType()))
1023 return getNode(ISD::TRUNCATE, SL, VT, Op);
1025 TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1026 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1029 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
1030 assert(!VT.isVector() &&
1031 "getZeroExtendInReg should use the vector element type instead of "
1032 "the vector type!");
1033 if (Op.getValueType() == VT) return Op;
1034 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1035 APInt Imm = APInt::getLowBitsSet(BitWidth,
1036 VT.getSizeInBits());
1037 return getNode(ISD::AND, DL, Op.getValueType(), Op,
1038 getConstant(Imm, DL, Op.getValueType()));
1041 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
1042 assert(VT.isVector() && "This DAG node is restricted to vector types.");
1043 assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1044 "The sizes of the input and result must match in order to perform the "
1045 "extend in-register.");
1046 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1047 "The destination vector type must have fewer lanes than the input.");
1048 return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
1051 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
1052 assert(VT.isVector() && "This DAG node is restricted to vector types.");
1053 assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1054 "The sizes of the input and result must match in order to perform the "
1055 "extend in-register.");
1056 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1057 "The destination vector type must have fewer lanes than the input.");
1058 return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
1061 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
1062 assert(VT.isVector() && "This DAG node is restricted to vector types.");
1063 assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1064 "The sizes of the input and result must match in order to perform the "
1065 "extend in-register.");
1066 assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1067 "The destination vector type must have fewer lanes than the input.");
1068 return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
1071 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1073 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
1074 EVT EltVT = VT.getScalarType();
1076 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1077 return getNode(ISD::XOR, DL, VT, Val, NegOne);
1080 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
1081 EVT EltVT = VT.getScalarType();
1083 switch (TLI->getBooleanContents(VT)) {
1084 case TargetLowering::ZeroOrOneBooleanContent:
1085 case TargetLowering::UndefinedBooleanContent:
1086 TrueValue = getConstant(1, DL, VT);
1088 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1089 TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL,
1093 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1096 SDValue SelectionDAG::getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isT,
1098 EVT EltVT = VT.getScalarType();
1099 assert((EltVT.getSizeInBits() >= 64 ||
1100 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1101 "getConstant with a uint64_t value that doesn't fit in the type!");
1102 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1105 SDValue SelectionDAG::getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isT,
1108 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1111 SDValue SelectionDAG::getConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
1112 bool isT, bool isO) {
1113 assert(VT.isInteger() && "Cannot create FP integer constant!");
1115 EVT EltVT = VT.getScalarType();
1116 const ConstantInt *Elt = &Val;
1118 // In some cases the vector type is legal but the element type is illegal and
1119 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1120 // inserted value (the type does not need to match the vector element type).
1121 // Any extra bits introduced will be truncated away.
1122 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1123 TargetLowering::TypePromoteInteger) {
1124 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1125 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
1126 Elt = ConstantInt::get(*getContext(), NewVal);
1128 // In other cases the element type is illegal and needs to be expanded, for
1129 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1130 // the value into n parts and use a vector type with n-times the elements.
1131 // Then bitcast to the type requested.
1132 // Legalizing constants too early makes the DAGCombiner's job harder so we
1133 // only legalize if the DAG tells us we must produce legal types.
1134 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1135 TLI->getTypeAction(*getContext(), EltVT) ==
1136 TargetLowering::TypeExpandInteger) {
1137 APInt NewVal = Elt->getValue();
1138 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1139 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1140 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1141 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1143 // Check the temporary vector is the correct size. If this fails then
1144 // getTypeToTransformTo() probably returned a type whose size (in bits)
1145 // isn't a power-of-2 factor of the requested type size.
1146 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1148 SmallVector<SDValue, 2> EltParts;
1149 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1150 EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1151 .trunc(ViaEltSizeInBits), DL,
1152 ViaEltVT, isT, isO));
1155 // EltParts is currently in little endian order. If we actually want
1156 // big-endian order then reverse it now.
1157 if (getDataLayout().isBigEndian())
1158 std::reverse(EltParts.begin(), EltParts.end());
1160 // The elements must be reversed when the element order is different
1161 // to the endianness of the elements (because the BITCAST is itself a
1162 // vector shuffle in this situation). However, we do not need any code to
1163 // perform this reversal because getConstant() is producing a vector
1165 // This situation occurs in MIPS MSA.
1167 SmallVector<SDValue, 8> Ops;
1168 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
1169 Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1171 SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
1172 getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
1177 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1178 "APInt size does not match type size!");
1179 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1180 FoldingSetNodeID ID;
1181 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1185 SDNode *N = nullptr;
1186 if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
1188 return SDValue(N, 0);
1191 N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, DL.getDebugLoc(),
1193 CSEMap.InsertNode(N, IP);
1197 SDValue Result(N, 0);
1198 if (VT.isVector()) {
1199 SmallVector<SDValue, 8> Ops;
1200 Ops.assign(VT.getVectorNumElements(), Result);
1201 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
1206 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget) {
1207 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1210 SDValue SelectionDAG::getConstantFP(const APFloat& V, SDLoc DL, EVT VT,
1212 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1215 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, SDLoc DL, EVT VT,
1217 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1219 EVT EltVT = VT.getScalarType();
1221 // Do the map lookup using the actual bit pattern for the floating point
1222 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1223 // we don't have issues with SNANs.
1224 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1225 FoldingSetNodeID ID;
1226 AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1229 SDNode *N = nullptr;
1230 if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
1232 return SDValue(N, 0);
1235 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, DL.getDebugLoc(),
1237 CSEMap.InsertNode(N, IP);
1241 SDValue Result(N, 0);
1242 if (VT.isVector()) {
1243 SmallVector<SDValue, 8> Ops;
1244 Ops.assign(VT.getVectorNumElements(), Result);
1245 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
1250 SDValue SelectionDAG::getConstantFP(double Val, SDLoc DL, EVT VT,
1252 EVT EltVT = VT.getScalarType();
1253 if (EltVT==MVT::f32)
1254 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1255 else if (EltVT==MVT::f64)
1256 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1257 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
1260 APFloat apf = APFloat(Val);
1261 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1263 return getConstantFP(apf, DL, VT, isTarget);
1265 llvm_unreachable("Unsupported type in getConstantFP");
1268 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
1269 EVT VT, int64_t Offset,
1271 unsigned char TargetFlags) {
1272 assert((TargetFlags == 0 || isTargetGA) &&
1273 "Cannot set target flags on target-independent globals");
1275 // Truncate (with sign-extension) the offset value to the pointer size.
1276 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1278 Offset = SignExtend64(Offset, BitWidth);
1281 if (GV->isThreadLocal())
1282 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1284 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1286 FoldingSetNodeID ID;
1287 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1289 ID.AddInteger(Offset);
1290 ID.AddInteger(TargetFlags);
1291 ID.AddInteger(GV->getType()->getAddressSpace());
1293 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
1294 return SDValue(E, 0);
1296 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
1297 DL.getDebugLoc(), GV, VT,
1298 Offset, TargetFlags);
1299 CSEMap.InsertNode(N, IP);
1301 return SDValue(N, 0);
1304 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1305 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1306 FoldingSetNodeID ID;
1307 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1310 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1311 return SDValue(E, 0);
1313 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
1314 CSEMap.InsertNode(N, IP);
1316 return SDValue(N, 0);
1319 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1320 unsigned char TargetFlags) {
1321 assert((TargetFlags == 0 || isTarget) &&
1322 "Cannot set target flags on target-independent jump tables");
1323 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1324 FoldingSetNodeID ID;
1325 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1327 ID.AddInteger(TargetFlags);
1329 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1330 return SDValue(E, 0);
1332 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
1334 CSEMap.InsertNode(N, IP);
1336 return SDValue(N, 0);
1339 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1340 unsigned Alignment, int Offset,
1342 unsigned char TargetFlags) {
1343 assert((TargetFlags == 0 || isTarget) &&
1344 "Cannot set target flags on target-independent globals");
1346 Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1347 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1348 FoldingSetNodeID ID;
1349 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1350 ID.AddInteger(Alignment);
1351 ID.AddInteger(Offset);
1353 ID.AddInteger(TargetFlags);
1355 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1356 return SDValue(E, 0);
1358 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1359 Alignment, TargetFlags);
1360 CSEMap.InsertNode(N, IP);
1362 return SDValue(N, 0);
1366 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1367 unsigned Alignment, int Offset,
1369 unsigned char TargetFlags) {
1370 assert((TargetFlags == 0 || isTarget) &&
1371 "Cannot set target flags on target-independent globals");
1373 Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1374 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1375 FoldingSetNodeID ID;
1376 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1377 ID.AddInteger(Alignment);
1378 ID.AddInteger(Offset);
1379 C->addSelectionDAGCSEId(ID);
1380 ID.AddInteger(TargetFlags);
1382 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1383 return SDValue(E, 0);
1385 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1386 Alignment, TargetFlags);
1387 CSEMap.InsertNode(N, IP);
1389 return SDValue(N, 0);
1392 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1393 unsigned char TargetFlags) {
1394 FoldingSetNodeID ID;
1395 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1396 ID.AddInteger(Index);
1397 ID.AddInteger(Offset);
1398 ID.AddInteger(TargetFlags);
1400 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1401 return SDValue(E, 0);
1404 new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, TargetFlags);
1405 CSEMap.InsertNode(N, IP);
1407 return SDValue(N, 0);
1410 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1411 FoldingSetNodeID ID;
1412 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1415 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1416 return SDValue(E, 0);
1418 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
1419 CSEMap.InsertNode(N, IP);
1421 return SDValue(N, 0);
1424 SDValue SelectionDAG::getValueType(EVT VT) {
1425 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1426 ValueTypeNodes.size())
1427 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1429 SDNode *&N = VT.isExtended() ?
1430 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1432 if (N) return SDValue(N, 0);
1433 N = new (NodeAllocator) VTSDNode(VT);
1435 return SDValue(N, 0);
1438 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1439 SDNode *&N = ExternalSymbols[Sym];
1440 if (N) return SDValue(N, 0);
1441 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
1443 return SDValue(N, 0);
1446 SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1447 SDNode *&N = MCSymbols[Sym];
1449 return SDValue(N, 0);
1450 N = new (NodeAllocator) MCSymbolSDNode(Sym, VT);
1452 return SDValue(N, 0);
1455 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1456 unsigned char TargetFlags) {
1458 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1460 if (N) return SDValue(N, 0);
1461 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
1463 return SDValue(N, 0);
1466 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1467 if ((unsigned)Cond >= CondCodeNodes.size())
1468 CondCodeNodes.resize(Cond+1);
1470 if (!CondCodeNodes[Cond]) {
1471 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
1472 CondCodeNodes[Cond] = N;
1476 return SDValue(CondCodeNodes[Cond], 0);
1479 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
1480 // the shuffle mask M that point at N1 to point at N2, and indices that point
1481 // N2 to point at N1.
1482 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
1484 ShuffleVectorSDNode::commuteMask(M);
1487 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
1488 SDValue N2, const int *Mask) {
1489 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1490 "Invalid VECTOR_SHUFFLE");
1492 // Canonicalize shuffle undef, undef -> undef
1493 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
1494 return getUNDEF(VT);
1496 // Validate that all indices in Mask are within the range of the elements
1497 // input to the shuffle.
1498 unsigned NElts = VT.getVectorNumElements();
1499 SmallVector<int, 8> MaskVec;
1500 for (unsigned i = 0; i != NElts; ++i) {
1501 assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
1502 MaskVec.push_back(Mask[i]);
1505 // Canonicalize shuffle v, v -> v, undef
1508 for (unsigned i = 0; i != NElts; ++i)
1509 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
1512 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1513 if (N1.getOpcode() == ISD::UNDEF)
1514 commuteShuffle(N1, N2, MaskVec);
1516 // If shuffling a splat, try to blend the splat instead. We do this here so
1517 // that even when this arises during lowering we don't have to re-handle it.
1518 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1519 BitVector UndefElements;
1520 SDValue Splat = BV->getSplatValue(&UndefElements);
1524 for (int i = 0; i < (int)NElts; ++i) {
1525 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + (int)NElts))
1528 // If this input comes from undef, mark it as such.
1529 if (UndefElements[MaskVec[i] - Offset]) {
1534 // If we can blend a non-undef lane, use that instead.
1535 if (!UndefElements[i])
1536 MaskVec[i] = i + Offset;
1539 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1540 BlendSplat(N1BV, 0);
1541 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1542 BlendSplat(N2BV, NElts);
1544 // Canonicalize all index into lhs, -> shuffle lhs, undef
1545 // Canonicalize all index into rhs, -> shuffle rhs, undef
1546 bool AllLHS = true, AllRHS = true;
1547 bool N2Undef = N2.getOpcode() == ISD::UNDEF;
1548 for (unsigned i = 0; i != NElts; ++i) {
1549 if (MaskVec[i] >= (int)NElts) {
1554 } else if (MaskVec[i] >= 0) {
1558 if (AllLHS && AllRHS)
1559 return getUNDEF(VT);
1560 if (AllLHS && !N2Undef)
1564 commuteShuffle(N1, N2, MaskVec);
1566 // Reset our undef status after accounting for the mask.
1567 N2Undef = N2.getOpcode() == ISD::UNDEF;
1568 // Re-check whether both sides ended up undef.
1569 if (N1.getOpcode() == ISD::UNDEF && N2Undef)
1570 return getUNDEF(VT);
1572 // If Identity shuffle return that node.
1573 bool Identity = true, AllSame = true;
1574 for (unsigned i = 0; i != NElts; ++i) {
1575 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
1576 if (MaskVec[i] != MaskVec[0]) AllSame = false;
1578 if (Identity && NElts)
1581 // Shuffling a constant splat doesn't change the result.
1585 // Look through any bitcasts. We check that these don't change the number
1586 // (and size) of elements and just changes their types.
1587 while (V.getOpcode() == ISD::BITCAST)
1588 V = V->getOperand(0);
1590 // A splat should always show up as a build vector node.
1591 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1592 BitVector UndefElements;
1593 SDValue Splat = BV->getSplatValue(&UndefElements);
1594 // If this is a splat of an undef, shuffling it is also undef.
1595 if (Splat && Splat.getOpcode() == ISD::UNDEF)
1596 return getUNDEF(VT);
1599 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1601 // We only have a splat which can skip shuffles if there is a splatted
1602 // value and no undef lanes rearranged by the shuffle.
1603 if (Splat && UndefElements.none()) {
1604 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1605 // number of elements match or the value splatted is a zero constant.
1608 if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1609 if (C->isNullValue())
1613 // If the shuffle itself creates a splat, build the vector directly.
1614 if (AllSame && SameNumElts) {
1615 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1616 SmallVector<SDValue, 8> Ops(NElts, Splatted);
1618 EVT BuildVT = BV->getValueType(0);
1619 SDValue NewBV = getNode(ISD::BUILD_VECTOR, dl, BuildVT, Ops);
1621 // We may have jumped through bitcasts, so the type of the
1622 // BUILD_VECTOR may not match the type of the shuffle.
1624 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1630 FoldingSetNodeID ID;
1631 SDValue Ops[2] = { N1, N2 };
1632 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1633 for (unsigned i = 0; i != NElts; ++i)
1634 ID.AddInteger(MaskVec[i]);
1637 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
1638 return SDValue(E, 0);
1640 // Allocate the mask array for the node out of the BumpPtrAllocator, since
1641 // SDNode doesn't have access to it. This memory will be "leaked" when
1642 // the node is deallocated, but recovered when the NodeAllocator is released.
1643 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1644 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
1646 ShuffleVectorSDNode *N =
1647 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
1648 dl.getDebugLoc(), N1, N2,
1650 CSEMap.InsertNode(N, IP);
1652 return SDValue(N, 0);
1655 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1656 MVT VT = SV.getSimpleValueType(0);
1657 SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1658 ShuffleVectorSDNode::commuteMask(MaskVec);
1660 SDValue Op0 = SV.getOperand(0);
1661 SDValue Op1 = SV.getOperand(1);
1662 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
1665 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
1666 SDValue Val, SDValue DTy,
1667 SDValue STy, SDValue Rnd, SDValue Sat,
1668 ISD::CvtCode Code) {
1669 // If the src and dest types are the same and the conversion is between
1670 // integer types of the same sign or two floats, no conversion is necessary.
1672 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
1675 FoldingSetNodeID ID;
1676 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
1677 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
1679 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
1680 return SDValue(E, 0);
1682 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
1685 CSEMap.InsertNode(N, IP);
1687 return SDValue(N, 0);
1690 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1691 FoldingSetNodeID ID;
1692 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1693 ID.AddInteger(RegNo);
1695 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1696 return SDValue(E, 0);
1698 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
1699 CSEMap.InsertNode(N, IP);
1701 return SDValue(N, 0);
1704 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1705 FoldingSetNodeID ID;
1706 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1707 ID.AddPointer(RegMask);
1709 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1710 return SDValue(E, 0);
1712 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
1713 CSEMap.InsertNode(N, IP);
1715 return SDValue(N, 0);
1718 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
1719 FoldingSetNodeID ID;
1720 SDValue Ops[] = { Root };
1721 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
1722 ID.AddPointer(Label);
1724 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1725 return SDValue(E, 0);
1727 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
1728 dl.getDebugLoc(), Root, Label);
1729 CSEMap.InsertNode(N, IP);
1731 return SDValue(N, 0);
1735 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1738 unsigned char TargetFlags) {
1739 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1741 FoldingSetNodeID ID;
1742 AddNodeIDNode(ID, Opc, getVTList(VT), None);
1744 ID.AddInteger(Offset);
1745 ID.AddInteger(TargetFlags);
1747 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1748 return SDValue(E, 0);
1750 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
1752 CSEMap.InsertNode(N, IP);
1754 return SDValue(N, 0);
1757 SDValue SelectionDAG::getSrcValue(const Value *V) {
1758 assert((!V || V->getType()->isPointerTy()) &&
1759 "SrcValue is not a pointer?");
1761 FoldingSetNodeID ID;
1762 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1766 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1767 return SDValue(E, 0);
1769 SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
1770 CSEMap.InsertNode(N, IP);
1772 return SDValue(N, 0);
1775 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
1776 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1777 FoldingSetNodeID ID;
1778 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1782 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1783 return SDValue(E, 0);
1785 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
1786 CSEMap.InsertNode(N, IP);
1788 return SDValue(N, 0);
1791 SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1792 if (VT == V.getValueType())
1795 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1798 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
1799 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
1800 unsigned SrcAS, unsigned DestAS) {
1801 SDValue Ops[] = {Ptr};
1802 FoldingSetNodeID ID;
1803 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1804 ID.AddInteger(SrcAS);
1805 ID.AddInteger(DestAS);
1808 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
1809 return SDValue(E, 0);
1811 SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
1813 VT, Ptr, SrcAS, DestAS);
1814 CSEMap.InsertNode(N, IP);
1816 return SDValue(N, 0);
1819 /// getShiftAmountOperand - Return the specified value casted to
1820 /// the target's desired shift amount type.
1821 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1822 EVT OpTy = Op.getValueType();
1823 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1824 if (OpTy == ShTy || OpTy.isVector()) return Op;
1826 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1829 SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1831 const TargetLowering &TLI = getTargetLoweringInfo();
1832 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1833 EVT VT = Node->getValueType(0);
1834 SDValue Tmp1 = Node->getOperand(0);
1835 SDValue Tmp2 = Node->getOperand(1);
1836 unsigned Align = Node->getConstantOperandVal(3);
1838 SDValue VAListLoad =
1839 getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, Tmp2,
1840 MachinePointerInfo(V), false, false, false, 0);
1841 SDValue VAList = VAListLoad;
1843 if (Align > TLI.getMinStackArgumentAlignment()) {
1844 assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
1846 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1847 getConstant(Align - 1, dl, VAList.getValueType()));
1849 VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
1850 getConstant(-(int64_t)Align, dl, VAList.getValueType()));
1853 // Increment the pointer, VAList, to the next vaarg
1854 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1855 getConstant(getDataLayout().getTypeAllocSize(
1856 VT.getTypeForEVT(*getContext())),
1857 dl, VAList.getValueType()));
1858 // Store the incremented VAList to the legalized pointer
1859 Tmp1 = getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2,
1860 MachinePointerInfo(V), false, false, 0);
1861 // Load the actual argument out of the pointer VAList
1862 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo(),
1863 false, false, false, 0);
1866 SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1868 const TargetLowering &TLI = getTargetLoweringInfo();
1869 // This defaults to loading a pointer from the input and storing it to the
1870 // output, returning the chain.
1871 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
1872 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
1873 SDValue Tmp1 = getLoad(TLI.getPointerTy(getDataLayout()), dl,
1874 Node->getOperand(0), Node->getOperand(2),
1875 MachinePointerInfo(VS), false, false, false, 0);
1876 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
1877 MachinePointerInfo(VD), false, false, 0);
1880 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
1881 /// specified value type.
1882 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1883 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
1884 unsigned ByteSize = VT.getStoreSize();
1885 Type *Ty = VT.getTypeForEVT(*getContext());
1886 unsigned StackAlign =
1887 std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1889 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
1890 return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
1893 /// CreateStackTemporary - Create a stack temporary suitable for holding
1894 /// either of the specified value types.
1895 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1896 unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
1897 Type *Ty1 = VT1.getTypeForEVT(*getContext());
1898 Type *Ty2 = VT2.getTypeForEVT(*getContext());
1899 const DataLayout &DL = getDataLayout();
1901 std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
1903 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
1904 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
1905 return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
1908 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
1909 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
1910 // These setcc operations always fold.
1914 case ISD::SETFALSE2: return getConstant(0, dl, VT);
1916 case ISD::SETTRUE2: {
1917 TargetLowering::BooleanContent Cnt =
1918 TLI->getBooleanContents(N1->getValueType(0));
1920 Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
1934 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
1938 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
1939 const APInt &C2 = N2C->getAPIntValue();
1940 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
1941 const APInt &C1 = N1C->getAPIntValue();
1944 default: llvm_unreachable("Unknown integer setcc!");
1945 case ISD::SETEQ: return getConstant(C1 == C2, dl, VT);
1946 case ISD::SETNE: return getConstant(C1 != C2, dl, VT);
1947 case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT);
1948 case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT);
1949 case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT);
1950 case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT);
1951 case ISD::SETLT: return getConstant(C1.slt(C2), dl, VT);
1952 case ISD::SETGT: return getConstant(C1.sgt(C2), dl, VT);
1953 case ISD::SETLE: return getConstant(C1.sle(C2), dl, VT);
1954 case ISD::SETGE: return getConstant(C1.sge(C2), dl, VT);
1958 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
1959 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
1960 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
1963 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
1964 return getUNDEF(VT);
1966 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
1967 case ISD::SETNE: if (R==APFloat::cmpUnordered)
1968 return getUNDEF(VT);
1970 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
1971 R==APFloat::cmpLessThan, dl, VT);
1972 case ISD::SETLT: if (R==APFloat::cmpUnordered)
1973 return getUNDEF(VT);
1975 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
1976 case ISD::SETGT: if (R==APFloat::cmpUnordered)
1977 return getUNDEF(VT);
1979 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
1980 case ISD::SETLE: if (R==APFloat::cmpUnordered)
1981 return getUNDEF(VT);
1983 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
1984 R==APFloat::cmpEqual, dl, VT);
1985 case ISD::SETGE: if (R==APFloat::cmpUnordered)
1986 return getUNDEF(VT);
1988 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
1989 R==APFloat::cmpEqual, dl, VT);
1990 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, dl, VT);
1991 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, dl, VT);
1992 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
1993 R==APFloat::cmpEqual, dl, VT);
1994 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT);
1995 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
1996 R==APFloat::cmpLessThan, dl, VT);
1997 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
1998 R==APFloat::cmpUnordered, dl, VT);
1999 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT);
2000 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT);
2003 // Ensure that the constant occurs on the RHS.
2004 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2005 MVT CompVT = N1.getValueType().getSimpleVT();
2006 if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
2009 return getSetCC(dl, VT, N2, N1, SwappedCond);
2013 // Could not fold it.
2017 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2018 /// use this predicate to simplify operations downstream.
2019 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2020 // This predicate is not safe for vector operations.
2021 if (Op.getValueType().isVector())
2024 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
2025 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
2028 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2029 /// this predicate to simplify operations downstream. Mask is known to be zero
2030 /// for bits that V cannot have.
2031 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
2032 unsigned Depth) const {
2033 APInt KnownZero, KnownOne;
2034 computeKnownBits(Op, KnownZero, KnownOne, Depth);
2035 return (KnownZero & Mask) == Mask;
2038 /// Determine which bits of Op are known to be either zero or one and return
2039 /// them in the KnownZero/KnownOne bitsets.
2040 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
2041 APInt &KnownOne, unsigned Depth) const {
2042 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
2044 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
2046 return; // Limit search depth.
2048 APInt KnownZero2, KnownOne2;
2050 switch (Op.getOpcode()) {
2052 // We know all of the bits for a constant!
2053 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
2054 KnownZero = ~KnownOne;
2057 // If either the LHS or the RHS are Zero, the result is zero.
2058 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2059 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2061 // Output known-1 bits are only known if set in both the LHS & RHS.
2062 KnownOne &= KnownOne2;
2063 // Output known-0 are known to be clear if zero in either the LHS | RHS.
2064 KnownZero |= KnownZero2;
2067 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2068 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2070 // Output known-0 bits are only known if clear in both the LHS & RHS.
2071 KnownZero &= KnownZero2;
2072 // Output known-1 are known to be set if set in either the LHS | RHS.
2073 KnownOne |= KnownOne2;
2076 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2077 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2079 // Output known-0 bits are known if clear or set in both the LHS & RHS.
2080 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
2081 // Output known-1 are known to be set if set in only one of the LHS, RHS.
2082 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
2083 KnownZero = KnownZeroOut;
2087 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2088 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2090 // If low bits are zero in either operand, output low known-0 bits.
2091 // Also compute a conserative estimate for high known-0 bits.
2092 // More trickiness is possible, but this is sufficient for the
2093 // interesting case of alignment computation.
2094 KnownOne.clearAllBits();
2095 unsigned TrailZ = KnownZero.countTrailingOnes() +
2096 KnownZero2.countTrailingOnes();
2097 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() +
2098 KnownZero2.countLeadingOnes(),
2099 BitWidth) - BitWidth;
2101 TrailZ = std::min(TrailZ, BitWidth);
2102 LeadZ = std::min(LeadZ, BitWidth);
2103 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
2104 APInt::getHighBitsSet(BitWidth, LeadZ);
2108 // For the purposes of computing leading zeros we can conservatively
2109 // treat a udiv as a logical right shift by the power of 2 known to
2110 // be less than the denominator.
2111 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2112 unsigned LeadZ = KnownZero2.countLeadingOnes();
2114 KnownOne2.clearAllBits();
2115 KnownZero2.clearAllBits();
2116 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2117 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
2118 if (RHSUnknownLeadingOnes != BitWidth)
2119 LeadZ = std::min(BitWidth,
2120 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
2122 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
2126 computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
2127 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2129 // Only known if known in both the LHS and RHS.
2130 KnownOne &= KnownOne2;
2131 KnownZero &= KnownZero2;
2133 case ISD::SELECT_CC:
2134 computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
2135 computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
2137 // Only known if known in both the LHS and RHS.
2138 KnownOne &= KnownOne2;
2139 KnownZero &= KnownZero2;
2147 if (Op.getResNo() != 1)
2149 // The boolean result conforms to getBooleanContents.
2150 // If we know the result of a setcc has the top bits zero, use this info.
2151 // We know that we have an integer-based boolean since these operations
2152 // are only available for integer.
2153 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2154 TargetLowering::ZeroOrOneBooleanContent &&
2156 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2159 // If we know the result of a setcc has the top bits zero, use this info.
2160 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2161 TargetLowering::ZeroOrOneBooleanContent &&
2163 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2166 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
2167 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2168 unsigned ShAmt = SA->getZExtValue();
2170 // If the shift count is an invalid immediate, don't do anything.
2171 if (ShAmt >= BitWidth)
2174 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2175 KnownZero <<= ShAmt;
2177 // low bits known zero.
2178 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
2182 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
2183 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2184 unsigned ShAmt = SA->getZExtValue();
2186 // If the shift count is an invalid immediate, don't do anything.
2187 if (ShAmt >= BitWidth)
2190 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2191 KnownZero = KnownZero.lshr(ShAmt);
2192 KnownOne = KnownOne.lshr(ShAmt);
2194 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
2195 KnownZero |= HighBits; // High bits known zero.
2199 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2200 unsigned ShAmt = SA->getZExtValue();
2202 // If the shift count is an invalid immediate, don't do anything.
2203 if (ShAmt >= BitWidth)
2206 // If any of the demanded bits are produced by the sign extension, we also
2207 // demand the input sign bit.
2208 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
2210 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2211 KnownZero = KnownZero.lshr(ShAmt);
2212 KnownOne = KnownOne.lshr(ShAmt);
2214 // Handle the sign bits.
2215 APInt SignBit = APInt::getSignBit(BitWidth);
2216 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask.
2218 if (KnownZero.intersects(SignBit)) {
2219 KnownZero |= HighBits; // New bits are known zero.
2220 } else if (KnownOne.intersects(SignBit)) {
2221 KnownOne |= HighBits; // New bits are known one.
2225 case ISD::SIGN_EXTEND_INREG: {
2226 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2227 unsigned EBits = EVT.getScalarType().getSizeInBits();
2229 // Sign extension. Compute the demanded bits in the result that are not
2230 // present in the input.
2231 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2233 APInt InSignBit = APInt::getSignBit(EBits);
2234 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2236 // If the sign extended bits are demanded, we know that the sign
2238 InSignBit = InSignBit.zext(BitWidth);
2239 if (NewBits.getBoolValue())
2240 InputDemandedBits |= InSignBit;
2242 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2243 KnownOne &= InputDemandedBits;
2244 KnownZero &= InputDemandedBits;
2246 // If the sign bit of the input is known set or clear, then we know the
2247 // top bits of the result.
2248 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear
2249 KnownZero |= NewBits;
2250 KnownOne &= ~NewBits;
2251 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set
2252 KnownOne |= NewBits;
2253 KnownZero &= ~NewBits;
2254 } else { // Input sign bit unknown
2255 KnownZero &= ~NewBits;
2256 KnownOne &= ~NewBits;
2261 case ISD::CTTZ_ZERO_UNDEF:
2263 case ISD::CTLZ_ZERO_UNDEF:
2265 unsigned LowBits = Log2_32(BitWidth)+1;
2266 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
2267 KnownOne.clearAllBits();
2271 LoadSDNode *LD = cast<LoadSDNode>(Op);
2272 // If this is a ZEXTLoad and we are looking at the loaded value.
2273 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
2274 EVT VT = LD->getMemoryVT();
2275 unsigned MemBits = VT.getScalarType().getSizeInBits();
2276 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
2277 } else if (const MDNode *Ranges = LD->getRanges()) {
2278 if (LD->getExtensionType() == ISD::NON_EXTLOAD)
2279 computeKnownBitsFromRangeMetadata(*Ranges, KnownZero, KnownOne);
2283 case ISD::ZERO_EXTEND: {
2284 EVT InVT = Op.getOperand(0).getValueType();
2285 unsigned InBits = InVT.getScalarType().getSizeInBits();
2286 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2287 KnownZero = KnownZero.trunc(InBits);
2288 KnownOne = KnownOne.trunc(InBits);
2289 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2290 KnownZero = KnownZero.zext(BitWidth);
2291 KnownOne = KnownOne.zext(BitWidth);
2292 KnownZero |= NewBits;
2295 case ISD::SIGN_EXTEND: {
2296 EVT InVT = Op.getOperand(0).getValueType();
2297 unsigned InBits = InVT.getScalarType().getSizeInBits();
2298 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2300 KnownZero = KnownZero.trunc(InBits);
2301 KnownOne = KnownOne.trunc(InBits);
2302 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2304 // Note if the sign bit is known to be zero or one.
2305 bool SignBitKnownZero = KnownZero.isNegative();
2306 bool SignBitKnownOne = KnownOne.isNegative();
2308 KnownZero = KnownZero.zext(BitWidth);
2309 KnownOne = KnownOne.zext(BitWidth);
2311 // If the sign bit is known zero or one, the top bits match.
2312 if (SignBitKnownZero)
2313 KnownZero |= NewBits;
2314 else if (SignBitKnownOne)
2315 KnownOne |= NewBits;
2318 case ISD::ANY_EXTEND: {
2319 EVT InVT = Op.getOperand(0).getValueType();
2320 unsigned InBits = InVT.getScalarType().getSizeInBits();
2321 KnownZero = KnownZero.trunc(InBits);
2322 KnownOne = KnownOne.trunc(InBits);
2323 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2324 KnownZero = KnownZero.zext(BitWidth);
2325 KnownOne = KnownOne.zext(BitWidth);
2328 case ISD::TRUNCATE: {
2329 EVT InVT = Op.getOperand(0).getValueType();
2330 unsigned InBits = InVT.getScalarType().getSizeInBits();
2331 KnownZero = KnownZero.zext(InBits);
2332 KnownOne = KnownOne.zext(InBits);
2333 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2334 KnownZero = KnownZero.trunc(BitWidth);
2335 KnownOne = KnownOne.trunc(BitWidth);
2338 case ISD::AssertZext: {
2339 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2340 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
2341 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2342 KnownZero |= (~InMask);
2343 KnownOne &= (~KnownZero);
2347 // All bits are zero except the low bit.
2348 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2352 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
2353 // We know that the top bits of C-X are clear if X contains less bits
2354 // than C (i.e. no wrap-around can happen). For example, 20-X is
2355 // positive if we can prove that X is >= 0 and < 16.
2356 if (CLHS->getAPIntValue().isNonNegative()) {
2357 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2358 // NLZ can't be BitWidth with no sign bit
2359 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
2360 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2362 // If all of the MaskV bits are known to be zero, then we know the
2363 // output top bits are zero, because we now know that the output is
2365 if ((KnownZero2 & MaskV) == MaskV) {
2366 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2367 // Top bits known zero.
2368 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
2376 // Output known-0 bits are known if clear or set in both the low clear bits
2377 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
2378 // low 3 bits clear.
2379 // Output known-0 bits are also known if the top bits of each input are
2380 // known to be clear. For example, if one input has the top 10 bits clear
2381 // and the other has the top 8 bits clear, we know the top 7 bits of the
2382 // output must be clear.
2383 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2384 unsigned KnownZeroHigh = KnownZero2.countLeadingOnes();
2385 unsigned KnownZeroLow = KnownZero2.countTrailingOnes();
2387 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2388 KnownZeroHigh = std::min(KnownZeroHigh,
2389 KnownZero2.countLeadingOnes());
2390 KnownZeroLow = std::min(KnownZeroLow,
2391 KnownZero2.countTrailingOnes());
2393 if (Op.getOpcode() == ISD::ADD) {
2394 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroLow);
2395 if (KnownZeroHigh > 1)
2396 KnownZero |= APInt::getHighBitsSet(BitWidth, KnownZeroHigh - 1);
2400 // With ADDE, a carry bit may be added in, so we can only use this
2401 // information if we know (at least) that the low two bits are clear. We
2402 // then return to the caller that the low bit is unknown but that other bits
2404 if (KnownZeroLow >= 2) // ADDE
2405 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroLow);
2409 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2410 const APInt &RA = Rem->getAPIntValue().abs();
2411 if (RA.isPowerOf2()) {
2412 APInt LowBits = RA - 1;
2413 computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
2415 // The low bits of the first operand are unchanged by the srem.
2416 KnownZero = KnownZero2 & LowBits;
2417 KnownOne = KnownOne2 & LowBits;
2419 // If the first operand is non-negative or has all low bits zero, then
2420 // the upper bits are all zero.
2421 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
2422 KnownZero |= ~LowBits;
2424 // If the first operand is negative and not all low bits are zero, then
2425 // the upper bits are all one.
2426 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
2427 KnownOne |= ~LowBits;
2428 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
2433 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2434 const APInt &RA = Rem->getAPIntValue();
2435 if (RA.isPowerOf2()) {
2436 APInt LowBits = (RA - 1);
2437 computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
2439 // The upper bits are all zero, the lower ones are unchanged.
2440 KnownZero = KnownZero2 | ~LowBits;
2441 KnownOne = KnownOne2 & LowBits;
2446 // Since the result is less than or equal to either operand, any leading
2447 // zero bits in either operand must also exist in the result.
2448 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2449 computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2451 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
2452 KnownZero2.countLeadingOnes());
2453 KnownOne.clearAllBits();
2454 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
2457 case ISD::EXTRACT_ELEMENT: {
2458 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2459 const unsigned Index =
2460 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
2461 const unsigned BitWidth = Op.getValueType().getSizeInBits();
2463 // Remove low part of known bits mask
2464 KnownZero = KnownZero.getHiBits(KnownZero.getBitWidth() - Index * BitWidth);
2465 KnownOne = KnownOne.getHiBits(KnownOne.getBitWidth() - Index * BitWidth);
2467 // Remove high part of known bit mask
2468 KnownZero = KnownZero.trunc(BitWidth);
2469 KnownOne = KnownOne.trunc(BitWidth);
2476 APInt Op0Zero, Op0One;
2477 APInt Op1Zero, Op1One;
2478 computeKnownBits(Op.getOperand(0), Op0Zero, Op0One, Depth);
2479 computeKnownBits(Op.getOperand(1), Op1Zero, Op1One, Depth);
2481 KnownZero = Op0Zero & Op1Zero;
2482 KnownOne = Op0One & Op1One;
2485 case ISD::FrameIndex:
2486 case ISD::TargetFrameIndex:
2487 if (unsigned Align = InferPtrAlignment(Op)) {
2488 // The low bits are known zero if the pointer is aligned.
2489 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
2495 if (Op.getOpcode() < ISD::BUILTIN_OP_END)
2498 case ISD::INTRINSIC_WO_CHAIN:
2499 case ISD::INTRINSIC_W_CHAIN:
2500 case ISD::INTRINSIC_VOID:
2501 // Allow the target to implement this method for its nodes.
2502 TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
2506 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
2509 /// ComputeNumSignBits - Return the number of times the sign bit of the
2510 /// register is replicated into the other bits. We know that at least 1 bit
2511 /// is always equal to the sign bit (itself), but other cases can give us
2512 /// information. For example, immediately after an "SRA X, 2", we know that
2513 /// the top 3 bits are all equal to each other, so we return 3.
2514 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
2515 EVT VT = Op.getValueType();
2516 assert(VT.isInteger() && "Invalid VT!");
2517 unsigned VTBits = VT.getScalarType().getSizeInBits();
2519 unsigned FirstAnswer = 1;
2522 return 1; // Limit search depth.
2524 switch (Op.getOpcode()) {
2526 case ISD::AssertSext:
2527 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2528 return VTBits-Tmp+1;
2529 case ISD::AssertZext:
2530 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2533 case ISD::Constant: {
2534 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
2535 return Val.getNumSignBits();
2538 case ISD::SIGN_EXTEND:
2540 VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
2541 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
2543 case ISD::SIGN_EXTEND_INREG:
2544 // Max of the input and what this extends.
2546 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
2549 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2550 return std::max(Tmp, Tmp2);
2553 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2554 // SRA X, C -> adds C sign bits.
2555 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2556 Tmp += C->getZExtValue();
2557 if (Tmp > VTBits) Tmp = VTBits;
2561 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2562 // shl destroys sign bits.
2563 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2564 if (C->getZExtValue() >= VTBits || // Bad shift.
2565 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out.
2566 return Tmp - C->getZExtValue();
2571 case ISD::XOR: // NOT is handled here.
2572 // Logical binary ops preserve the number of sign bits at the worst.
2573 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2575 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2576 FirstAnswer = std::min(Tmp, Tmp2);
2577 // We computed what we know about the sign bits as our first
2578 // answer. Now proceed to the generic code that uses
2579 // computeKnownBits, and pick whichever answer is better.
2584 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2585 if (Tmp == 1) return 1; // Early out.
2586 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
2587 return std::min(Tmp, Tmp2);
2588 case ISD::SELECT_CC:
2589 Tmp = ComputeNumSignBits(Op.getOperand(2), Depth+1);
2590 if (Tmp == 1) return 1; // Early out.
2591 Tmp2 = ComputeNumSignBits(Op.getOperand(3), Depth+1);
2592 return std::min(Tmp, Tmp2);
2597 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
2599 return 1; // Early out.
2600 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
2601 return std::min(Tmp, Tmp2);
2608 if (Op.getResNo() != 1)
2610 // The boolean result conforms to getBooleanContents. Fall through.
2611 // If setcc returns 0/-1, all bits are sign bits.
2612 // We know that we have an integer-based boolean since these operations
2613 // are only available for integer.
2614 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2615 TargetLowering::ZeroOrNegativeOneBooleanContent)
2619 // If setcc returns 0/-1, all bits are sign bits.
2620 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2621 TargetLowering::ZeroOrNegativeOneBooleanContent)
2626 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2627 unsigned RotAmt = C->getZExtValue() & (VTBits-1);
2629 // Handle rotate right by N like a rotate left by 32-N.
2630 if (Op.getOpcode() == ISD::ROTR)
2631 RotAmt = (VTBits-RotAmt) & (VTBits-1);
2633 // If we aren't rotating out all of the known-in sign bits, return the
2634 // number that are left. This handles rotl(sext(x), 1) for example.
2635 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2636 if (Tmp > RotAmt+1) return Tmp-RotAmt;
2640 // Add can have at most one carry bit. Thus we know that the output
2641 // is, at worst, one more bit than the inputs.
2642 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2643 if (Tmp == 1) return 1; // Early out.
2645 // Special case decrementing a value (ADD X, -1):
2646 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2647 if (CRHS->isAllOnesValue()) {
2648 APInt KnownZero, KnownOne;
2649 computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2651 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2653 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2656 // If we are subtracting one from a positive number, there is no carry
2657 // out of the result.
2658 if (KnownZero.isNegative())
2662 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2663 if (Tmp2 == 1) return 1;
2664 return std::min(Tmp, Tmp2)-1;
2667 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2668 if (Tmp2 == 1) return 1;
2671 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
2672 if (CLHS->isNullValue()) {
2673 APInt KnownZero, KnownOne;
2674 computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2675 // If the input is known to be 0 or 1, the output is 0/-1, which is all
2677 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2680 // If the input is known to be positive (the sign bit is known clear),
2681 // the output of the NEG has the same number of sign bits as the input.
2682 if (KnownZero.isNegative())
2685 // Otherwise, we treat this like a SUB.
2688 // Sub can have at most one carry bit. Thus we know that the output
2689 // is, at worst, one more bit than the inputs.
2690 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2691 if (Tmp == 1) return 1; // Early out.
2692 return std::min(Tmp, Tmp2)-1;
2694 // FIXME: it's tricky to do anything useful for this, but it is an important
2695 // case for targets like X86.
2697 case ISD::EXTRACT_ELEMENT: {
2698 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2699 const int BitWidth = Op.getValueType().getSizeInBits();
2701 Op.getOperand(0).getValueType().getSizeInBits() / BitWidth;
2703 // Get reverse index (starting from 1), Op1 value indexes elements from
2704 // little end. Sign starts at big end.
2705 const int rIndex = Items - 1 -
2706 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
2708 // If the sign portion ends in our element the subtraction gives correct
2709 // result. Otherwise it gives either negative or > bitwidth result
2710 return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
2714 // If we are looking at the loaded value of the SDNode.
2715 if (Op.getResNo() == 0) {
2716 // Handle LOADX separately here. EXTLOAD case will fallthrough.
2717 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
2718 unsigned ExtType = LD->getExtensionType();
2721 case ISD::SEXTLOAD: // '17' bits known
2722 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2723 return VTBits-Tmp+1;
2724 case ISD::ZEXTLOAD: // '16' bits known
2725 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2731 // Allow the target to implement this method for its nodes.
2732 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2733 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2734 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2735 Op.getOpcode() == ISD::INTRINSIC_VOID) {
2736 unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
2737 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
2740 // Finally, if we can prove that the top bits of the result are 0's or 1's,
2741 // use this information.
2742 APInt KnownZero, KnownOne;
2743 computeKnownBits(Op, KnownZero, KnownOne, Depth);
2746 if (KnownZero.isNegative()) { // sign bit is 0
2748 } else if (KnownOne.isNegative()) { // sign bit is 1;
2755 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
2756 // the number of identical bits in the top of the input value.
2758 Mask <<= Mask.getBitWidth()-VTBits;
2759 // Return # leading zeros. We use 'min' here in case Val was zero before
2760 // shifting. We don't want to return '64' as for an i32 "0".
2761 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
2764 /// isBaseWithConstantOffset - Return true if the specified operand is an
2765 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
2766 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
2767 /// semantics as an ADD. This handles the equivalence:
2768 /// X|Cst == X+Cst iff X&Cst = 0.
2769 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
2770 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
2771 !isa<ConstantSDNode>(Op.getOperand(1)))
2774 if (Op.getOpcode() == ISD::OR &&
2775 !MaskedValueIsZero(Op.getOperand(0),
2776 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
2783 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
2784 // If we're told that NaNs won't happen, assume they won't.
2785 if (getTarget().Options.NoNaNsFPMath)
2788 // If the value is a constant, we can obviously see if it is a NaN or not.
2789 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2790 return !C->getValueAPF().isNaN();
2792 // TODO: Recognize more cases here.
2797 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
2798 // If the value is a constant, we can obviously see if it is a zero or not.
2799 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2800 return !C->isZero();
2802 // TODO: Recognize more cases here.
2803 switch (Op.getOpcode()) {
2806 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2807 return !C->isNullValue();
2814 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
2815 // Check the obvious case.
2816 if (A == B) return true;
2818 // For for negative and positive zero.
2819 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
2820 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
2821 if (CA->isZero() && CB->isZero()) return true;
2823 // Otherwise they may not be equal.
2827 bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
2828 assert(A.getValueType() == B.getValueType() &&
2829 "Values must have the same type");
2832 computeKnownBits(A, AZero, AOne);
2833 computeKnownBits(B, BZero, BOne);
2834 return (AZero | BZero).isAllOnesValue();
2837 static SDValue FoldCONCAT_VECTORS(SDLoc DL, EVT VT, ArrayRef<SDValue> Ops,
2838 llvm::SelectionDAG &DAG) {
2839 if (Ops.size() == 1)
2842 // Concat of UNDEFs is UNDEF.
2843 if (std::all_of(Ops.begin(), Ops.end(),
2844 [](SDValue Op) { return Op.isUndef(); }))
2845 return DAG.getUNDEF(VT);
2847 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified
2848 // to one big BUILD_VECTOR.
2849 // FIXME: Add support for UNDEF and SCALAR_TO_VECTOR as well.
2850 if (!std::all_of(Ops.begin(), Ops.end(), [](SDValue Op) {
2851 return Op.getOpcode() == ISD::BUILD_VECTOR;
2855 EVT SVT = VT.getScalarType();
2856 SmallVector<SDValue, 16> Elts;
2857 for (SDValue Op : Ops)
2858 Elts.append(Op->op_begin(), Op->op_end());
2860 // BUILD_VECTOR requires all inputs to be of the same type, find the
2861 // maximum type and extend them all.
2862 for (SDValue Op : Elts)
2863 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
2865 if (SVT.bitsGT(VT.getScalarType()))
2866 for (SDValue &Op : Elts)
2867 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
2868 ? DAG.getZExtOrTrunc(Op, DL, SVT)
2869 : DAG.getSExtOrTrunc(Op, DL, SVT);
2871 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
2874 /// getNode - Gets or creates the specified node.
2876 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
2877 FoldingSetNodeID ID;
2878 AddNodeIDNode(ID, Opcode, getVTList(VT), None);
2880 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
2881 return SDValue(E, 0);
2883 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
2884 DL.getDebugLoc(), getVTList(VT));
2885 CSEMap.InsertNode(N, IP);
2888 return SDValue(N, 0);
2891 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
2892 EVT VT, SDValue Operand) {
2893 // Constant fold unary operations with an integer constant operand. Even
2894 // opaque constant will be folded, because the folding of unary operations
2895 // doesn't create new constants with different values. Nevertheless, the
2896 // opaque flag is preserved during folding to prevent future folding with
2898 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
2899 const APInt &Val = C->getAPIntValue();
2902 case ISD::SIGN_EXTEND:
2903 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
2904 C->isTargetOpcode(), C->isOpaque());
2905 case ISD::ANY_EXTEND:
2906 case ISD::ZERO_EXTEND:
2908 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
2909 C->isTargetOpcode(), C->isOpaque());
2910 case ISD::UINT_TO_FP:
2911 case ISD::SINT_TO_FP: {
2912 APFloat apf(EVTToAPFloatSemantics(VT),
2913 APInt::getNullValue(VT.getSizeInBits()));
2914 (void)apf.convertFromAPInt(Val,
2915 Opcode==ISD::SINT_TO_FP,
2916 APFloat::rmNearestTiesToEven);
2917 return getConstantFP(apf, DL, VT);
2920 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
2921 return getConstantFP(APFloat(APFloat::IEEEhalf, Val), DL, VT);
2922 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
2923 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), DL, VT);
2924 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
2925 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), DL, VT);
2926 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
2927 return getConstantFP(APFloat(APFloat::IEEEquad, Val), DL, VT);
2930 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
2933 return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
2936 case ISD::CTLZ_ZERO_UNDEF:
2937 return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
2940 case ISD::CTTZ_ZERO_UNDEF:
2941 return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
2946 // Constant fold unary operations with a floating point constant operand.
2947 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
2948 APFloat V = C->getValueAPF(); // make copy
2952 return getConstantFP(V, DL, VT);
2955 return getConstantFP(V, DL, VT);
2957 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
2958 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2959 return getConstantFP(V, DL, VT);
2963 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
2964 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2965 return getConstantFP(V, DL, VT);
2969 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
2970 if (fs == APFloat::opOK || fs == APFloat::opInexact)
2971 return getConstantFP(V, DL, VT);
2974 case ISD::FP_EXTEND: {
2976 // This can return overflow, underflow, or inexact; we don't care.
2977 // FIXME need to be more flexible about rounding mode.
2978 (void)V.convert(EVTToAPFloatSemantics(VT),
2979 APFloat::rmNearestTiesToEven, &ignored);
2980 return getConstantFP(V, DL, VT);
2982 case ISD::FP_TO_SINT:
2983 case ISD::FP_TO_UINT: {
2986 static_assert(integerPartWidth >= 64, "APFloat parts too small!");
2987 // FIXME need to be more flexible about rounding mode.
2988 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
2989 Opcode==ISD::FP_TO_SINT,
2990 APFloat::rmTowardZero, &ignored);
2991 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual
2993 APInt api(VT.getSizeInBits(), x);
2994 return getConstant(api, DL, VT);
2997 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
2998 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
2999 else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
3000 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
3001 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
3002 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
3007 // Constant fold unary operations with a vector integer or float operand.
3008 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
3009 if (BV->isConstant()) {
3012 // FIXME: Entirely reasonable to perform folding of other unary
3013 // operations here as the need arises.
3020 case ISD::FP_EXTEND:
3021 case ISD::FP_TO_SINT:
3022 case ISD::FP_TO_UINT:
3024 case ISD::UINT_TO_FP:
3025 case ISD::SINT_TO_FP:
3028 case ISD::CTLZ_ZERO_UNDEF:
3030 case ISD::CTTZ_ZERO_UNDEF:
3032 SDValue Ops = { Operand };
3033 if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
3040 unsigned OpOpcode = Operand.getNode()->getOpcode();
3042 case ISD::TokenFactor:
3043 case ISD::MERGE_VALUES:
3044 case ISD::CONCAT_VECTORS:
3045 return Operand; // Factor, merge or concat of one node? No need.
3046 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
3047 case ISD::FP_EXTEND:
3048 assert(VT.isFloatingPoint() &&
3049 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
3050 if (Operand.getValueType() == VT) return Operand; // noop conversion.
3051 assert((!VT.isVector() ||
3052 VT.getVectorNumElements() ==
3053 Operand.getValueType().getVectorNumElements()) &&
3054 "Vector element count mismatch!");
3055 assert(Operand.getValueType().bitsLT(VT) &&
3056 "Invalid fpext node, dst < src!");
3057 if (Operand.getOpcode() == ISD::UNDEF)
3058 return getUNDEF(VT);
3060 case ISD::SIGN_EXTEND:
3061 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3062 "Invalid SIGN_EXTEND!");
3063 if (Operand.getValueType() == VT) return Operand; // noop extension
3064 assert((!VT.isVector() ||
3065 VT.getVectorNumElements() ==
3066 Operand.getValueType().getVectorNumElements()) &&
3067 "Vector element count mismatch!");
3068 assert(Operand.getValueType().bitsLT(VT) &&
3069 "Invalid sext node, dst < src!");
3070 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
3071 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
3072 else if (OpOpcode == ISD::UNDEF)
3073 // sext(undef) = 0, because the top bits will all be the same.
3074 return getConstant(0, DL, VT);
3076 case ISD::ZERO_EXTEND:
3077 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3078 "Invalid ZERO_EXTEND!");
3079 if (Operand.getValueType() == VT) return Operand; // noop extension
3080 assert((!VT.isVector() ||
3081 VT.getVectorNumElements() ==
3082 Operand.getValueType().getVectorNumElements()) &&
3083 "Vector element count mismatch!");
3084 assert(Operand.getValueType().bitsLT(VT) &&
3085 "Invalid zext node, dst < src!");
3086 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
3087 return getNode(ISD::ZERO_EXTEND, DL, VT,
3088 Operand.getNode()->getOperand(0));
3089 else if (OpOpcode == ISD::UNDEF)
3090 // zext(undef) = 0, because the top bits will be zero.
3091 return getConstant(0, DL, VT);
3093 case ISD::ANY_EXTEND:
3094 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3095 "Invalid ANY_EXTEND!");
3096 if (Operand.getValueType() == VT) return Operand; // noop extension
3097 assert((!VT.isVector() ||
3098 VT.getVectorNumElements() ==
3099 Operand.getValueType().getVectorNumElements()) &&
3100 "Vector element count mismatch!");
3101 assert(Operand.getValueType().bitsLT(VT) &&
3102 "Invalid anyext node, dst < src!");
3104 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
3105 OpOpcode == ISD::ANY_EXTEND)
3106 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
3107 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
3108 else if (OpOpcode == ISD::UNDEF)
3109 return getUNDEF(VT);
3111 // (ext (trunx x)) -> x
3112 if (OpOpcode == ISD::TRUNCATE) {
3113 SDValue OpOp = Operand.getNode()->getOperand(0);
3114 if (OpOp.getValueType() == VT)
3119 assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3120 "Invalid TRUNCATE!");
3121 if (Operand.getValueType() == VT) return Operand; // noop truncate
3122 assert((!VT.isVector() ||
3123 VT.getVectorNumElements() ==
3124 Operand.getValueType().getVectorNumElements()) &&
3125 "Vector element count mismatch!");
3126 assert(Operand.getValueType().bitsGT(VT) &&
3127 "Invalid truncate node, src < dst!");
3128 if (OpOpcode == ISD::TRUNCATE)
3129 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
3130 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
3131 OpOpcode == ISD::ANY_EXTEND) {
3132 // If the source is smaller than the dest, we still need an extend.
3133 if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
3134 .bitsLT(VT.getScalarType()))
3135 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
3136 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
3137 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
3138 return Operand.getNode()->getOperand(0);
3140 if (OpOpcode == ISD::UNDEF)
3141 return getUNDEF(VT);
3144 assert(VT.isInteger() && VT == Operand.getValueType() &&
3146 assert((VT.getScalarSizeInBits() % 16 == 0) &&
3147 "BSWAP types must be a multiple of 16 bits!");
3148 if (OpOpcode == ISD::UNDEF)
3149 return getUNDEF(VT);
3152 // Basic sanity checking.
3153 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
3154 && "Cannot BITCAST between types of different sizes!");
3155 if (VT == Operand.getValueType()) return Operand; // noop conversion.
3156 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
3157 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
3158 if (OpOpcode == ISD::UNDEF)
3159 return getUNDEF(VT);
3161 case ISD::SCALAR_TO_VECTOR:
3162 assert(VT.isVector() && !Operand.getValueType().isVector() &&
3163 (VT.getVectorElementType() == Operand.getValueType() ||
3164 (VT.getVectorElementType().isInteger() &&
3165 Operand.getValueType().isInteger() &&
3166 VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
3167 "Illegal SCALAR_TO_VECTOR node!");
3168 if (OpOpcode == ISD::UNDEF)
3169 return getUNDEF(VT);
3170 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
3171 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
3172 isa<ConstantSDNode>(Operand.getOperand(1)) &&
3173 Operand.getConstantOperandVal(1) == 0 &&
3174 Operand.getOperand(0).getValueType() == VT)
3175 return Operand.getOperand(0);
3178 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
3179 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
3180 // FIXME: FNEG has no fast-math-flags to propagate; use the FSUB's flags?
3181 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
3182 Operand.getNode()->getOperand(0),
3183 &cast<BinaryWithFlagsSDNode>(Operand.getNode())->Flags);
3184 if (OpOpcode == ISD::FNEG) // --X -> X
3185 return Operand.getNode()->getOperand(0);
3188 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
3189 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
3194 SDVTList VTs = getVTList(VT);
3195 if (VT != MVT::Glue) { // Don't CSE flag producing nodes
3196 FoldingSetNodeID ID;
3197 SDValue Ops[1] = { Operand };
3198 AddNodeIDNode(ID, Opcode, VTs, Ops);
3200 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
3201 return SDValue(E, 0);
3203 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
3204 DL.getDebugLoc(), VTs, Operand);
3205 CSEMap.InsertNode(N, IP);
3207 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
3208 DL.getDebugLoc(), VTs, Operand);
3212 return SDValue(N, 0);
3215 static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
3218 case ISD::ADD: return std::make_pair(C1 + C2, true);
3219 case ISD::SUB: return std::make_pair(C1 - C2, true);
3220 case ISD::MUL: return std::make_pair(C1 * C2, true);
3221 case ISD::AND: return std::make_pair(C1 & C2, true);
3222 case ISD::OR: return std::make_pair(C1 | C2, true);
3223 case ISD::XOR: return std::make_pair(C1 ^ C2, true);
3224 case ISD::SHL: return std::make_pair(C1 << C2, true);
3225 case ISD::SRL: return std::make_pair(C1.lshr(C2), true);
3226 case ISD::SRA: return std::make_pair(C1.ashr(C2), true);
3227 case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
3228 case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
3229 case ISD::SMIN: return std::make_pair(C1.sle(C2) ? C1 : C2, true);
3230 case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
3231 case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
3232 case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
3234 if (!C2.getBoolValue())
3236 return std::make_pair(C1.udiv(C2), true);
3238 if (!C2.getBoolValue())
3240 return std::make_pair(C1.urem(C2), true);
3242 if (!C2.getBoolValue())
3244 return std::make_pair(C1.sdiv(C2), true);
3246 if (!C2.getBoolValue())
3248 return std::make_pair(C1.srem(C2), true);
3250 return std::make_pair(APInt(1, 0), false);
3253 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
3254 const ConstantSDNode *Cst1,
3255 const ConstantSDNode *Cst2) {
3256 if (Cst1->isOpaque() || Cst2->isOpaque())
3259 std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
3260 Cst2->getAPIntValue());
3263 return getConstant(Folded.first, DL, VT);
3266 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
3267 SDNode *Cst1, SDNode *Cst2) {
3268 // If the opcode is a target-specific ISD node, there's nothing we can
3269 // do here and the operand rules may not line up with the below, so
3271 if (Opcode >= ISD::BUILTIN_OP_END)
3274 // Handle the case of two scalars.
3275 if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
3276 if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
3277 if (SDValue Folded =
3278 FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2)) {
3281 SmallVector<SDValue, 4> Outputs;
3282 // We may have a vector type but a scalar result. Create a splat.
3283 Outputs.resize(VT.getVectorNumElements(), Outputs.back());
3284 // Build a big vector out of the scalar elements we generated.
3285 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
3292 // For vectors extract each constant element into Inputs so we can constant
3293 // fold them individually.
3294 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
3295 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
3299 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
3301 EVT SVT = VT.getScalarType();
3302 SmallVector<SDValue, 4> Outputs;
3303 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
3304 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
3305 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
3306 if (!V1 || !V2) // Not a constant, bail.
3309 if (V1->isOpaque() || V2->isOpaque())
3312 // Avoid BUILD_VECTOR nodes that perform implicit truncation.
3313 // FIXME: This is valid and could be handled by truncating the APInts.
3314 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
3317 // Fold one vector element.
3318 std::pair<APInt, bool> Folded = FoldValue(Opcode, V1->getAPIntValue(),
3319 V2->getAPIntValue());
3322 Outputs.push_back(getConstant(Folded.first, DL, SVT));
3325 assert(VT.getVectorNumElements() == Outputs.size() &&
3326 "Vector size mismatch!");
3328 // We may have a vector type but a scalar result. Create a splat.
3329 Outputs.resize(VT.getVectorNumElements(), Outputs.back());
3331 // Build a big vector out of the scalar elements we generated.
3332 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
3335 SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, SDLoc DL,
3337 ArrayRef<SDValue> Ops,
3338 const SDNodeFlags *Flags) {
3339 // If the opcode is a target-specific ISD node, there's nothing we can
3340 // do here and the operand rules may not line up with the below, so
3342 if (Opcode >= ISD::BUILTIN_OP_END)
3345 // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
3349 unsigned NumElts = VT.getVectorNumElements();
3351 auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
3352 return !Op.getValueType().isVector() ||
3353 Op.getValueType().getVectorNumElements() == NumElts;
3356 auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
3357 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
3358 return (Op.getOpcode() == ISD::UNDEF) ||
3359 (Op.getOpcode() == ISD::CONDCODE) || (BV && BV->isConstant());
3362 // All operands must be vector types with the same number of elements as
3363 // the result type and must be either UNDEF or a build vector of constant
3364 // or UNDEF scalars.
3365 if (!std::all_of(Ops.begin(), Ops.end(), IsConstantBuildVectorOrUndef) ||
3366 !std::all_of(Ops.begin(), Ops.end(), IsScalarOrSameVectorSize))
3369 // If we are comparing vectors, then the result needs to be a i1 boolean
3370 // that is then sign-extended back to the legal result type.
3371 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
3373 // Find legal integer scalar type for constant promotion and
3374 // ensure that its scalar size is at least as large as source.
3375 EVT LegalSVT = VT.getScalarType();
3376 if (LegalSVT.isInteger()) {
3377 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
3378 if (LegalSVT.bitsLT(SVT))
3382 // Constant fold each scalar lane separately.
3383 SmallVector<SDValue, 4> ScalarResults;
3384 for (unsigned i = 0; i != NumElts; i++) {
3385 SmallVector<SDValue, 4> ScalarOps;
3386 for (SDValue Op : Ops) {
3387 EVT InSVT = Op.getValueType().getScalarType();
3388 BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
3390 // We've checked that this is UNDEF or a constant of some kind.
3392 ScalarOps.push_back(getUNDEF(InSVT));
3394 ScalarOps.push_back(Op);
3398 SDValue ScalarOp = InBV->getOperand(i);
3399 EVT ScalarVT = ScalarOp.getValueType();
3401 // Build vector (integer) scalar operands may need implicit
3402 // truncation - do this before constant folding.
3403 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT))
3404 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
3406 ScalarOps.push_back(ScalarOp);
3409 // Constant fold the scalar operands.
3410 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
3412 // Legalize the (integer) scalar constant if necessary.
3413 if (LegalSVT != SVT)
3414 ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
3416 // Scalar folding only succeeded if the result is a constant or UNDEF.
3417 if (ScalarResult.getOpcode() != ISD::UNDEF &&
3418 ScalarResult.getOpcode() != ISD::Constant &&
3419 ScalarResult.getOpcode() != ISD::ConstantFP)
3421 ScalarResults.push_back(ScalarResult);
3424 assert(ScalarResults.size() == NumElts &&
3425 "Unexpected number of scalar results for BUILD_VECTOR");
3426 return getNode(ISD::BUILD_VECTOR, DL, VT, ScalarResults);
3429 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
3430 SDValue N2, const SDNodeFlags *Flags) {
3431 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3432 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
3433 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
3434 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
3436 // Canonicalize constant to RHS if commutative.
3437 if (isCommutativeBinOp(Opcode)) {
3439 std::swap(N1C, N2C);
3441 } else if (N1CFP && !N2CFP) {
3442 std::swap(N1CFP, N2CFP);
3449 case ISD::TokenFactor:
3450 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
3451 N2.getValueType() == MVT::Other && "Invalid token factor!");
3452 // Fold trivial token factors.
3453 if (N1.getOpcode() == ISD::EntryToken) return N2;
3454 if (N2.getOpcode() == ISD::EntryToken) return N1;
3455 if (N1 == N2) return N1;
3457 case ISD::CONCAT_VECTORS: {
3458 // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
3459 SDValue Ops[] = {N1, N2};
3460 if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
3465 assert(VT.isInteger() && "This operator does not apply to FP types!");
3466 assert(N1.getValueType() == N2.getValueType() &&
3467 N1.getValueType() == VT && "Binary operator types must match!");
3468 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
3469 // worth handling here.
3470 if (N2C && N2C->isNullValue())
3472 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
3479 assert(VT.isInteger() && "This operator does not apply to FP types!");
3480 assert(N1.getValueType() == N2.getValueType() &&
3481 N1.getValueType() == VT && "Binary operator types must match!");
3482 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
3483 // it's worth handling here.
3484 if (N2C && N2C->isNullValue())
3498 assert(VT.isInteger() && "This operator does not apply to FP types!");
3499 assert(N1.getValueType() == N2.getValueType() &&
3500 N1.getValueType() == VT && "Binary operator types must match!");
3507 if (getTarget().Options.UnsafeFPMath) {
3508 if (Opcode == ISD::FADD) {
3510 if (N2CFP && N2CFP->getValueAPF().isZero())
3512 } else if (Opcode == ISD::FSUB) {
3514 if (N2CFP && N2CFP->getValueAPF().isZero())
3516 } else if (Opcode == ISD::FMUL) {
3518 if (N2CFP && N2CFP->isZero())
3521 if (N2CFP && N2CFP->isExactlyValue(1.0))
3525 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
3526 assert(N1.getValueType() == N2.getValueType() &&
3527 N1.getValueType() == VT && "Binary operator types must match!");
3529 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
3530 assert(N1.getValueType() == VT &&
3531 N1.getValueType().isFloatingPoint() &&
3532 N2.getValueType().isFloatingPoint() &&
3533 "Invalid FCOPYSIGN!");
3540 assert(VT == N1.getValueType() &&
3541 "Shift operators return type must be the same as their first arg");
3542 assert(VT.isInteger() && N2.getValueType().isInteger() &&
3543 "Shifts only work on integers");
3544 assert((!VT.isVector() || VT == N2.getValueType()) &&
3545 "Vector shift amounts must be in the same as their first arg");
3546 // Verify that the shift amount VT is bit enough to hold valid shift
3547 // amounts. This catches things like trying to shift an i1024 value by an
3548 // i8, which is easy to fall into in generic code that uses
3549 // TLI.getShiftAmount().
3550 assert(N2.getValueType().getSizeInBits() >=
3551 Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
3552 "Invalid use of small shift amount with oversized value!");
3554 // Always fold shifts of i1 values so the code generator doesn't need to
3555 // handle them. Since we know the size of the shift has to be less than the
3556 // size of the value, the shift/rotate count is guaranteed to be zero.
3559 if (N2C && N2C->isNullValue())
3562 case ISD::FP_ROUND_INREG: {
3563 EVT EVT = cast<VTSDNode>(N2)->getVT();
3564 assert(VT == N1.getValueType() && "Not an inreg round!");
3565 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
3566 "Cannot FP_ROUND_INREG integer types");
3567 assert(EVT.isVector() == VT.isVector() &&
3568 "FP_ROUND_INREG type should be vector iff the operand "
3570 assert((!EVT.isVector() ||
3571 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3572 "Vector element counts must match in FP_ROUND_INREG");
3573 assert(EVT.bitsLE(VT) && "Not rounding down!");
3575 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
3579 assert(VT.isFloatingPoint() &&
3580 N1.getValueType().isFloatingPoint() &&
3581 VT.bitsLE(N1.getValueType()) &&
3582 N2C && "Invalid FP_ROUND!");
3583 if (N1.getValueType() == VT) return N1; // noop conversion.
3585 case ISD::AssertSext:
3586 case ISD::AssertZext: {
3587 EVT EVT = cast<VTSDNode>(N2)->getVT();
3588 assert(VT == N1.getValueType() && "Not an inreg extend!");
3589 assert(VT.isInteger() && EVT.isInteger() &&
3590 "Cannot *_EXTEND_INREG FP types");
3591 assert(!EVT.isVector() &&
3592 "AssertSExt/AssertZExt type should be the vector element type "
3593 "rather than the vector type!");
3594 assert(EVT.bitsLE(VT) && "Not extending!");
3595 if (VT == EVT) return N1; // noop assertion.
3598 case ISD::SIGN_EXTEND_INREG: {
3599 EVT EVT = cast<VTSDNode>(N2)->getVT();
3600 assert(VT == N1.getValueType() && "Not an inreg extend!");
3601 assert(VT.isInteger() && EVT.isInteger() &&
3602 "Cannot *_EXTEND_INREG FP types");
3603 assert(EVT.isVector() == VT.isVector() &&
3604 "SIGN_EXTEND_INREG type should be vector iff the operand "
3606 assert((!EVT.isVector() ||
3607 EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3608 "Vector element counts must match in SIGN_EXTEND_INREG");
3609 assert(EVT.bitsLE(VT) && "Not extending!");
3610 if (EVT == VT) return N1; // Not actually extending
3612 auto SignExtendInReg = [&](APInt Val) {
3613 unsigned FromBits = EVT.getScalarType().getSizeInBits();
3614 Val <<= Val.getBitWidth() - FromBits;
3615 Val = Val.ashr(Val.getBitWidth() - FromBits);
3616 return getConstant(Val, DL, VT.getScalarType());
3620 APInt Val = N1C->getAPIntValue();
3621 return SignExtendInReg(Val);
3623 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
3624 SmallVector<SDValue, 8> Ops;
3625 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
3626 SDValue Op = N1.getOperand(i);
3627 if (Op.getOpcode() == ISD::UNDEF) {
3628 Ops.push_back(getUNDEF(VT.getScalarType()));
3631 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3632 APInt Val = C->getAPIntValue();
3633 Val = Val.zextOrTrunc(VT.getScalarSizeInBits());
3634 Ops.push_back(SignExtendInReg(Val));
3639 if (Ops.size() == VT.getVectorNumElements())
3640 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
3644 case ISD::EXTRACT_VECTOR_ELT:
3645 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
3646 if (N1.getOpcode() == ISD::UNDEF)
3647 return getUNDEF(VT);
3649 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
3650 if (N2C && N2C->getZExtValue() >= N1.getValueType().getVectorNumElements())
3651 return getUNDEF(VT);
3653 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
3654 // expanding copies of large vectors from registers.
3656 N1.getOpcode() == ISD::CONCAT_VECTORS &&
3657 N1.getNumOperands() > 0) {
3659 N1.getOperand(0).getValueType().getVectorNumElements();
3660 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
3661 N1.getOperand(N2C->getZExtValue() / Factor),
3662 getConstant(N2C->getZExtValue() % Factor, DL,
3663 N2.getValueType()));
3666 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
3667 // expanding large vector constants.
3668 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
3669 SDValue Elt = N1.getOperand(N2C->getZExtValue());
3671 if (VT != Elt.getValueType())
3672 // If the vector element type is not legal, the BUILD_VECTOR operands
3673 // are promoted and implicitly truncated, and the result implicitly
3674 // extended. Make that explicit here.
3675 Elt = getAnyExtOrTrunc(Elt, DL, VT);
3680 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
3681 // operations are lowered to scalars.
3682 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
3683 // If the indices are the same, return the inserted element else
3684 // if the indices are known different, extract the element from
3685 // the original vector.
3686 SDValue N1Op2 = N1.getOperand(2);
3687 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
3689 if (N1Op2C && N2C) {
3690 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
3691 if (VT == N1.getOperand(1).getValueType())
3692 return N1.getOperand(1);
3694 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
3697 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
3701 case ISD::EXTRACT_ELEMENT:
3702 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
3703 assert(!N1.getValueType().isVector() && !VT.isVector() &&
3704 (N1.getValueType().isInteger() == VT.isInteger()) &&
3705 N1.getValueType() != VT &&
3706 "Wrong types for EXTRACT_ELEMENT!");
3708 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
3709 // 64-bit integers into 32-bit parts. Instead of building the extract of
3710 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
3711 if (N1.getOpcode() == ISD::BUILD_PAIR)
3712 return N1.getOperand(N2C->getZExtValue());
3714 // EXTRACT_ELEMENT of a constant int is also very common.
3716 unsigned ElementSize = VT.getSizeInBits();
3717 unsigned Shift = ElementSize * N2C->getZExtValue();
3718 APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
3719 return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
3722 case ISD::EXTRACT_SUBVECTOR:
3723 if (VT.isSimple() && N1.getValueType().isSimple()) {
3724 assert(VT.isVector() && N1.getValueType().isVector() &&
3725 "Extract subvector VTs must be a vectors!");
3726 assert(VT.getVectorElementType() ==
3727 N1.getValueType().getVectorElementType() &&
3728 "Extract subvector VTs must have the same element type!");
3729 assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
3730 "Extract subvector must be from larger vector to smaller vector!");
3733 assert((VT.getVectorNumElements() + N2C->getZExtValue()
3734 <= N1.getValueType().getVectorNumElements())
3735 && "Extract subvector overflow!");
3738 // Trivial extraction.
3739 if (VT.getSimpleVT() == N1.getSimpleValueType())
3745 // Perform trivial constant folding.
3747 FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
3750 // Constant fold FP operations.
3751 bool HasFPExceptions = TLI->hasFloatingPointExceptions();
3754 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
3755 APFloat::opStatus s;
3758 s = V1.add(V2, APFloat::rmNearestTiesToEven);
3759 if (!HasFPExceptions || s != APFloat::opInvalidOp)
3760 return getConstantFP(V1, DL, VT);
3763 s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
3764 if (!HasFPExceptions || s!=APFloat::opInvalidOp)
3765 return getConstantFP(V1, DL, VT);
3768 s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
3769 if (!HasFPExceptions || s!=APFloat::opInvalidOp)
3770 return getConstantFP(V1, DL, VT);
3773 s = V1.divide(V2, APFloat::rmNearestTiesToEven);
3774 if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
3775 s!=APFloat::opDivByZero)) {
3776 return getConstantFP(V1, DL, VT);
3781 if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
3782 s!=APFloat::opDivByZero)) {
3783 return getConstantFP(V1, DL, VT);
3786 case ISD::FCOPYSIGN:
3788 return getConstantFP(V1, DL, VT);
3793 if (Opcode == ISD::FP_ROUND) {
3794 APFloat V = N1CFP->getValueAPF(); // make copy
3796 // This can return overflow, underflow, or inexact; we don't care.
3797 // FIXME need to be more flexible about rounding mode.
3798 (void)V.convert(EVTToAPFloatSemantics(VT),
3799 APFloat::rmNearestTiesToEven, &ignored);
3800 return getConstantFP(V, DL, VT);
3804 // Canonicalize an UNDEF to the RHS, even over a constant.
3805 if (N1.getOpcode() == ISD::UNDEF) {
3806 if (isCommutativeBinOp(Opcode)) {
3810 case ISD::FP_ROUND_INREG:
3811 case ISD::SIGN_EXTEND_INREG:
3817 return N1; // fold op(undef, arg2) -> undef
3825 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
3826 // For vectors, we can't easily build an all zero vector, just return
3833 // Fold a bunch of operators when the RHS is undef.
3834 if (N2.getOpcode() == ISD::UNDEF) {
3837 if (N1.getOpcode() == ISD::UNDEF)
3838 // Handle undef ^ undef -> 0 special case. This is a common
3840 return getConstant(0, DL, VT);
3850 return N2; // fold op(arg1, undef) -> undef
3856 if (getTarget().Options.UnsafeFPMath)
3864 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
3865 // For vectors, we can't easily build an all zero vector, just return
3870 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
3871 // For vectors, we can't easily build an all one vector, just return
3879 // Memoize this node if possible.
3881 SDVTList VTs = getVTList(VT);
3882 if (VT != MVT::Glue) {
3883 SDValue Ops[] = {N1, N2};
3884 FoldingSetNodeID ID;
3885 AddNodeIDNode(ID, Opcode, VTs, Ops);
3887 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
3889 E->intersectFlagsWith(Flags);
3890 return SDValue(E, 0);
3893 N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
3895 CSEMap.InsertNode(N, IP);
3897 N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
3901 return SDValue(N, 0);
3904 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3905 SDValue N1, SDValue N2, SDValue N3) {
3906 // Perform various simplifications.
3909 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
3910 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
3911 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
3912 if (N1CFP && N2CFP && N3CFP) {
3913 APFloat V1 = N1CFP->getValueAPF();
3914 const APFloat &V2 = N2CFP->getValueAPF();
3915 const APFloat &V3 = N3CFP->getValueAPF();
3916 APFloat::opStatus s =
3917 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
3918 if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
3919 return getConstantFP(V1, DL, VT);
3923 case ISD::CONCAT_VECTORS: {
3924 // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
3925 SDValue Ops[] = {N1, N2, N3};
3926 if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
3931 // Use FoldSetCC to simplify SETCC's.
3932 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
3934 // Vector constant folding.
3935 SDValue Ops[] = {N1, N2, N3};
3936 if (SDValue V = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
3941 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
3942 if (N1C->getZExtValue())
3943 return N2; // select true, X, Y -> X
3944 return N3; // select false, X, Y -> Y
3947 if (N2 == N3) return N2; // select C, X, X -> X
3949 case ISD::VECTOR_SHUFFLE:
3950 llvm_unreachable("should use getVectorShuffle constructor!");
3951 case ISD::INSERT_SUBVECTOR: {
3953 if (VT.isSimple() && N1.getValueType().isSimple()
3954 && N2.getValueType().isSimple()) {
3955 assert(VT.isVector() && N1.getValueType().isVector() &&
3956 N2.getValueType().isVector() &&
3957 "Insert subvector VTs must be a vectors");
3958 assert(VT == N1.getValueType() &&
3959 "Dest and insert subvector source types must match!");
3960 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
3961 "Insert subvector must be from smaller vector to larger vector!");
3962 if (isa<ConstantSDNode>(Index)) {
3963 assert((N2.getValueType().getVectorNumElements() +
3964 cast<ConstantSDNode>(Index)->getZExtValue()
3965 <= VT.getVectorNumElements())
3966 && "Insert subvector overflow!");
3969 // Trivial insertion.
3970 if (VT.getSimpleVT() == N2.getSimpleValueType())
3976 // Fold bit_convert nodes from a type to themselves.
3977 if (N1.getValueType() == VT)
3982 // Memoize node if it doesn't produce a flag.
3984 SDVTList VTs = getVTList(VT);
3985 if (VT != MVT::Glue) {
3986 SDValue Ops[] = { N1, N2, N3 };
3987 FoldingSetNodeID ID;
3988 AddNodeIDNode(ID, Opcode, VTs, Ops);
3990 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
3991 return SDValue(E, 0);
3993 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3994 DL.getDebugLoc(), VTs, N1, N2, N3);
3995 CSEMap.InsertNode(N, IP);
3997 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3998 DL.getDebugLoc(), VTs, N1, N2, N3);
4002 return SDValue(N, 0);
4005 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
4006 SDValue N1, SDValue N2, SDValue N3,
4008 SDValue Ops[] = { N1, N2, N3, N4 };
4009 return getNode(Opcode, DL, VT, Ops);
4012 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
4013 SDValue N1, SDValue N2, SDValue N3,
4014 SDValue N4, SDValue N5) {
4015 SDValue Ops[] = { N1, N2, N3, N4, N5 };
4016 return getNode(Opcode, DL, VT, Ops);
4019 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
4020 /// the incoming stack arguments to be loaded from the stack.
4021 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
4022 SmallVector<SDValue, 8> ArgChains;
4024 // Include the original chain at the beginning of the list. When this is
4025 // used by target LowerCall hooks, this helps legalize find the
4026 // CALLSEQ_BEGIN node.
4027 ArgChains.push_back(Chain);
4029 // Add a chain value for each stack argument.
4030 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
4031 UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
4032 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
4033 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
4034 if (FI->getIndex() < 0)
4035 ArgChains.push_back(SDValue(L, 1));
4037 // Build a tokenfactor for all the chains.
4038 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
4041 /// getMemsetValue - Vectorized representation of the memset value
4043 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
4045 assert(Value.getOpcode() != ISD::UNDEF);
4047 unsigned NumBits = VT.getScalarType().getSizeInBits();
4048 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
4049 assert(C->getAPIntValue().getBitWidth() == 8);
4050 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
4052 return DAG.getConstant(Val, dl, VT);
4053 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
4057 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
4058 EVT IntVT = VT.getScalarType();
4059 if (!IntVT.isInteger())
4060 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
4062 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
4064 // Use a multiplication with 0x010101... to extend the input to the
4066 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
4067 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
4068 DAG.getConstant(Magic, dl, IntVT));
4071 if (VT != Value.getValueType() && !VT.isInteger())
4072 Value = DAG.getNode(ISD::BITCAST, dl, VT.getScalarType(), Value);
4073 if (VT != Value.getValueType()) {
4074 assert(VT.getVectorElementType() == Value.getValueType() &&
4075 "value type should be one vector element here");
4076 SmallVector<SDValue, 8> BVOps(VT.getVectorNumElements(), Value);
4077 Value = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, BVOps);
4083 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
4084 /// used when a memcpy is turned into a memset when the source is a constant
4086 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
4087 const TargetLowering &TLI, StringRef Str) {
4088 // Handle vector with all elements zero.
4091 return DAG.getConstant(0, dl, VT);
4092 else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
4093 return DAG.getConstantFP(0.0, dl, VT);
4094 else if (VT.isVector()) {
4095 unsigned NumElts = VT.getVectorNumElements();
4096 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
4097 return DAG.getNode(ISD::BITCAST, dl, VT,
4098 DAG.getConstant(0, dl,
4099 EVT::getVectorVT(*DAG.getContext(),
4102 llvm_unreachable("Expected type!");
4105 assert(!VT.isVector() && "Can't handle vector type here!");
4106 unsigned NumVTBits = VT.getSizeInBits();
4107 unsigned NumVTBytes = NumVTBits / 8;
4108 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
4110 APInt Val(NumVTBits, 0);
4111 if (DAG.getDataLayout().isLittleEndian()) {
4112 for (unsigned i = 0; i != NumBytes; ++i)
4113 Val |= (uint64_t)(unsigned char)Str[i] << i*8;
4115 for (unsigned i = 0; i != NumBytes; ++i)
4116 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
4119 // If the "cost" of materializing the integer immediate is less than the cost
4120 // of a load, then it is cost effective to turn the load into the immediate.
4121 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
4122 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
4123 return DAG.getConstant(Val, dl, VT);
4124 return SDValue(nullptr, 0);
4127 /// getMemBasePlusOffset - Returns base and offset node for the
4129 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
4130 SelectionDAG &DAG) {
4131 EVT VT = Base.getValueType();
4132 return DAG.getNode(ISD::ADD, dl,
4133 VT, Base, DAG.getConstant(Offset, dl, VT));
4136 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
4138 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
4139 unsigned SrcDelta = 0;
4140 GlobalAddressSDNode *G = nullptr;
4141 if (Src.getOpcode() == ISD::GlobalAddress)
4142 G = cast<GlobalAddressSDNode>(Src);
4143 else if (Src.getOpcode() == ISD::ADD &&
4144 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
4145 Src.getOperand(1).getOpcode() == ISD::Constant) {
4146 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
4147 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
4152 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
4155 /// Determines the optimal series of memory ops to replace the memset / memcpy.
4156 /// Return true if the number of memory ops is below the threshold (Limit).
4157 /// It returns the types of the sequence of memory ops to perform
4158 /// memset / memcpy by reference.
4159 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
4160 unsigned Limit, uint64_t Size,
4161 unsigned DstAlign, unsigned SrcAlign,
4167 const TargetLowering &TLI) {
4168 assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
4169 "Expecting memcpy / memset source to meet alignment requirement!");
4170 // If 'SrcAlign' is zero, that means the memory operation does not need to
4171 // load the value, i.e. memset or memcpy from constant string. Otherwise,
4172 // it's the inferred alignment of the source. 'DstAlign', on the other hand,
4173 // is the specified alignment of the memory operation. If it is zero, that
4174 // means it's possible to change the alignment of the destination.
4175 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
4176 // not need to be loaded.
4177 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
4178 IsMemset, ZeroMemset, MemcpyStrSrc,
4179 DAG.getMachineFunction());
4181 if (VT == MVT::Other) {
4183 if (DstAlign >= DAG.getDataLayout().getPointerPrefAlignment(AS) ||
4184 TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
4185 VT = TLI.getPointerTy(DAG.getDataLayout());
4187 switch (DstAlign & 7) {
4188 case 0: VT = MVT::i64; break;
4189 case 4: VT = MVT::i32; break;
4190 case 2: VT = MVT::i16; break;
4191 default: VT = MVT::i8; break;
4196 while (!TLI.isTypeLegal(LVT))
4197 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
4198 assert(LVT.isInteger());
4204 unsigned NumMemOps = 0;
4206 unsigned VTSize = VT.getSizeInBits() / 8;
4207 while (VTSize > Size) {
4208 // For now, only use non-vector load / store's for the left-over pieces.
4213 if (VT.isVector() || VT.isFloatingPoint()) {
4214 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
4215 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
4216 TLI.isSafeMemOpType(NewVT.getSimpleVT()))
4218 else if (NewVT == MVT::i64 &&
4219 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
4220 TLI.isSafeMemOpType(MVT::f64)) {
4221 // i64 is usually not legal on 32-bit targets, but f64 may be.
4229 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
4230 if (NewVT == MVT::i8)
4232 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
4234 NewVTSize = NewVT.getSizeInBits() / 8;
4236 // If the new VT cannot cover all of the remaining bits, then consider
4237 // issuing a (or a pair of) unaligned and overlapping load / store.
4238 // FIXME: Only does this for 64-bit or more since we don't have proper
4239 // cost model for unaligned load / store.
4242 if (NumMemOps && AllowOverlap &&
4243 VTSize >= 8 && NewVTSize < Size &&
4244 TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
4252 if (++NumMemOps > Limit)
4255 MemOps.push_back(VT);
4262 static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
4263 // On Darwin, -Os means optimize for size without hurting performance, so
4264 // only really optimize for size when -Oz (MinSize) is used.
4265 if (MF.getTarget().getTargetTriple().isOSDarwin())
4266 return MF.getFunction()->optForMinSize();
4267 return MF.getFunction()->optForSize();
4270 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
4271 SDValue Chain, SDValue Dst,
4272 SDValue Src, uint64_t Size,
4273 unsigned Align, bool isVol,
4275 MachinePointerInfo DstPtrInfo,
4276 MachinePointerInfo SrcPtrInfo) {
4277 // Turn a memcpy of undef to nop.
4278 if (Src.getOpcode() == ISD::UNDEF)
4281 // Expand memcpy to a series of load and store ops if the size operand falls
4282 // below a certain threshold.
4283 // TODO: In the AlwaysInline case, if the size is big then generate a loop
4284 // rather than maybe a humongous number of loads and stores.
4285 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4286 std::vector<EVT> MemOps;
4287 bool DstAlignCanChange = false;
4288 MachineFunction &MF = DAG.getMachineFunction();
4289 MachineFrameInfo *MFI = MF.getFrameInfo();
4290 bool OptSize = shouldLowerMemFuncForSize(MF);
4291 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
4292 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4293 DstAlignCanChange = true;
4294 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
4295 if (Align > SrcAlign)
4298 bool CopyFromStr = isMemSrcFromString(Src, Str);
4299 bool isZeroStr = CopyFromStr && Str.empty();
4300 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
4302 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
4303 (DstAlignCanChange ? 0 : Align),
4304 (isZeroStr ? 0 : SrcAlign),
4305 false, false, CopyFromStr, true, DAG, TLI))
4308 if (DstAlignCanChange) {
4309 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4310 unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
4312 // Don't promote to an alignment that would require dynamic stack
4314 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
4315 if (!TRI->needsStackRealignment(MF))
4316 while (NewAlign > Align &&
4317 DAG.getDataLayout().exceedsNaturalStackAlignment(NewAlign))
4320 if (NewAlign > Align) {
4321 // Give the stack frame object a larger alignment if needed.
4322 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4323 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4328 SmallVector<SDValue, 8> OutChains;
4329 unsigned NumMemOps = MemOps.size();
4330 uint64_t SrcOff = 0, DstOff = 0;
4331 for (unsigned i = 0; i != NumMemOps; ++i) {
4333 unsigned VTSize = VT.getSizeInBits() / 8;
4334 SDValue Value, Store;
4336 if (VTSize > Size) {
4337 // Issuing an unaligned load / store pair that overlaps with the previous
4338 // pair. Adjust the offset accordingly.
4339 assert(i == NumMemOps-1 && i != 0);
4340 SrcOff -= VTSize - Size;
4341 DstOff -= VTSize - Size;
4345 (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
4346 // It's unlikely a store of a vector immediate can be done in a single
4347 // instruction. It would require a load from a constantpool first.
4348 // We only handle zero vectors here.
4349 // FIXME: Handle other cases where store of vector immediate is done in
4350 // a single instruction.
4351 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
4352 if (Value.getNode())
4353 Store = DAG.getStore(Chain, dl, Value,
4354 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4355 DstPtrInfo.getWithOffset(DstOff), isVol,
4359 if (!Store.getNode()) {
4360 // The type might not be legal for the target. This should only happen
4361 // if the type is smaller than a legal type, as on PPC, so the right
4362 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
4363 // to Load/Store if NVT==VT.
4364 // FIXME does the case above also need this?
4365 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4366 assert(NVT.bitsGE(VT));
4367 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
4368 getMemBasePlusOffset(Src, SrcOff, dl, DAG),
4369 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
4370 false, MinAlign(SrcAlign, SrcOff));
4371 Store = DAG.getTruncStore(Chain, dl, Value,
4372 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4373 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
4376 OutChains.push_back(Store);
4382 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4385 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
4386 SDValue Chain, SDValue Dst,
4387 SDValue Src, uint64_t Size,
4388 unsigned Align, bool isVol,
4390 MachinePointerInfo DstPtrInfo,
4391 MachinePointerInfo SrcPtrInfo) {
4392 // Turn a memmove of undef to nop.
4393 if (Src.getOpcode() == ISD::UNDEF)
4396 // Expand memmove to a series of load and store ops if the size operand falls
4397 // below a certain threshold.
4398 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4399 std::vector<EVT> MemOps;
4400 bool DstAlignCanChange = false;
4401 MachineFunction &MF = DAG.getMachineFunction();
4402 MachineFrameInfo *MFI = MF.getFrameInfo();
4403 bool OptSize = shouldLowerMemFuncForSize(MF);
4404 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
4405 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4406 DstAlignCanChange = true;
4407 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
4408 if (Align > SrcAlign)
4410 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
4412 if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
4413 (DstAlignCanChange ? 0 : Align), SrcAlign,
4414 false, false, false, false, DAG, TLI))
4417 if (DstAlignCanChange) {
4418 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4419 unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
4420 if (NewAlign > Align) {
4421 // Give the stack frame object a larger alignment if needed.
4422 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4423 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4428 uint64_t SrcOff = 0, DstOff = 0;
4429 SmallVector<SDValue, 8> LoadValues;
4430 SmallVector<SDValue, 8> LoadChains;
4431 SmallVector<SDValue, 8> OutChains;
4432 unsigned NumMemOps = MemOps.size();
4433 for (unsigned i = 0; i < NumMemOps; i++) {
4435 unsigned VTSize = VT.getSizeInBits() / 8;
4438 Value = DAG.getLoad(VT, dl, Chain,
4439 getMemBasePlusOffset(Src, SrcOff, dl, DAG),
4440 SrcPtrInfo.getWithOffset(SrcOff), isVol,
4441 false, false, SrcAlign);
4442 LoadValues.push_back(Value);
4443 LoadChains.push_back(Value.getValue(1));
4446 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
4448 for (unsigned i = 0; i < NumMemOps; i++) {
4450 unsigned VTSize = VT.getSizeInBits() / 8;
4453 Store = DAG.getStore(Chain, dl, LoadValues[i],
4454 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4455 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
4456 OutChains.push_back(Store);
4460 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4463 /// \brief Lower the call to 'memset' intrinsic function into a series of store
4466 /// \param DAG Selection DAG where lowered code is placed.
4467 /// \param dl Link to corresponding IR location.
4468 /// \param Chain Control flow dependency.
4469 /// \param Dst Pointer to destination memory location.
4470 /// \param Src Value of byte to write into the memory.
4471 /// \param Size Number of bytes to write.
4472 /// \param Align Alignment of the destination in bytes.
4473 /// \param isVol True if destination is volatile.
4474 /// \param DstPtrInfo IR information on the memory pointer.
4475 /// \returns New head in the control flow, if lowering was successful, empty
4476 /// SDValue otherwise.
4478 /// The function tries to replace 'llvm.memset' intrinsic with several store
4479 /// operations and value calculation code. This is usually profitable for small
4481 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
4482 SDValue Chain, SDValue Dst,
4483 SDValue Src, uint64_t Size,
4484 unsigned Align, bool isVol,
4485 MachinePointerInfo DstPtrInfo) {
4486 // Turn a memset of undef to nop.
4487 if (Src.getOpcode() == ISD::UNDEF)
4490 // Expand memset to a series of load/store ops if the size operand
4491 // falls below a certain threshold.
4492 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4493 std::vector<EVT> MemOps;
4494 bool DstAlignCanChange = false;
4495 MachineFunction &MF = DAG.getMachineFunction();
4496 MachineFrameInfo *MFI = MF.getFrameInfo();
4497 bool OptSize = shouldLowerMemFuncForSize(MF);
4498 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
4499 if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4500 DstAlignCanChange = true;
4502 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
4503 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
4504 Size, (DstAlignCanChange ? 0 : Align), 0,
4505 true, IsZeroVal, false, true, DAG, TLI))
4508 if (DstAlignCanChange) {
4509 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4510 unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
4511 if (NewAlign > Align) {
4512 // Give the stack frame object a larger alignment if needed.
4513 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4514 MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4519 SmallVector<SDValue, 8> OutChains;
4520 uint64_t DstOff = 0;
4521 unsigned NumMemOps = MemOps.size();
4523 // Find the largest store and generate the bit pattern for it.
4524 EVT LargestVT = MemOps[0];
4525 for (unsigned i = 1; i < NumMemOps; i++)
4526 if (MemOps[i].bitsGT(LargestVT))
4527 LargestVT = MemOps[i];
4528 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
4530 for (unsigned i = 0; i < NumMemOps; i++) {
4532 unsigned VTSize = VT.getSizeInBits() / 8;
4533 if (VTSize > Size) {
4534 // Issuing an unaligned load / store pair that overlaps with the previous
4535 // pair. Adjust the offset accordingly.
4536 assert(i == NumMemOps-1 && i != 0);
4537 DstOff -= VTSize - Size;
4540 // If this store is smaller than the largest store see whether we can get
4541 // the smaller value for free with a truncate.
4542 SDValue Value = MemSetValue;
4543 if (VT.bitsLT(LargestVT)) {
4544 if (!LargestVT.isVector() && !VT.isVector() &&
4545 TLI.isTruncateFree(LargestVT, VT))
4546 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
4548 Value = getMemsetValue(Src, VT, DAG, dl);
4550 assert(Value.getValueType() == VT && "Value with wrong type.");
4551 SDValue Store = DAG.getStore(Chain, dl, Value,
4552 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4553 DstPtrInfo.getWithOffset(DstOff),
4554 isVol, false, Align);
4555 OutChains.push_back(Store);
4556 DstOff += VT.getSizeInBits() / 8;
4560 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4563 static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
4565 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
4566 // pointer operands can be losslessly bitcasted to pointers of address space 0
4567 if (AS != 0 && !TLI->isNoopAddrSpaceCast(AS, 0)) {
4568 report_fatal_error("cannot lower memory intrinsic in address space " +
4573 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
4574 SDValue Src, SDValue Size,
4575 unsigned Align, bool isVol, bool AlwaysInline,
4576 bool isTailCall, MachinePointerInfo DstPtrInfo,
4577 MachinePointerInfo SrcPtrInfo) {
4578 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4580 // Check to see if we should lower the memcpy to loads and stores first.
4581 // For cases within the target-specified limits, this is the best choice.
4582 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4584 // Memcpy with size zero? Just return the original chain.
4585 if (ConstantSize->isNullValue())
4588 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4589 ConstantSize->getZExtValue(),Align,
4590 isVol, false, DstPtrInfo, SrcPtrInfo);
4591 if (Result.getNode())
4595 // Then check to see if we should lower the memcpy with target-specific
4596 // code. If the target chooses to do this, this is the next best.
4598 SDValue Result = TSI->EmitTargetCodeForMemcpy(
4599 *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
4600 DstPtrInfo, SrcPtrInfo);
4601 if (Result.getNode())
4605 // If we really need inline code and the target declined to provide it,
4606 // use a (potentially long) sequence of loads and stores.
4608 assert(ConstantSize && "AlwaysInline requires a constant size!");
4609 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4610 ConstantSize->getZExtValue(), Align, isVol,
4611 true, DstPtrInfo, SrcPtrInfo);
4614 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
4615 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
4617 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
4618 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
4619 // respect volatile, so they may do things like read or write memory
4620 // beyond the given memory regions. But fixing this isn't easy, and most
4621 // people don't care.
4623 // Emit a library call.
4624 TargetLowering::ArgListTy Args;
4625 TargetLowering::ArgListEntry Entry;
4626 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
4627 Entry.Node = Dst; Args.push_back(Entry);
4628 Entry.Node = Src; Args.push_back(Entry);
4629 Entry.Node = Size; Args.push_back(Entry);
4630 // FIXME: pass in SDLoc
4631 TargetLowering::CallLoweringInfo CLI(*this);
4634 .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
4635 Type::getVoidTy(*getContext()),
4636 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
4637 TLI->getPointerTy(getDataLayout())),
4640 .setTailCall(isTailCall);
4642 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4643 return CallResult.second;
4646 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
4647 SDValue Src, SDValue Size,
4648 unsigned Align, bool isVol, bool isTailCall,
4649 MachinePointerInfo DstPtrInfo,
4650 MachinePointerInfo SrcPtrInfo) {
4651 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4653 // Check to see if we should lower the memmove to loads and stores first.
4654 // For cases within the target-specified limits, this is the best choice.
4655 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4657 // Memmove with size zero? Just return the original chain.
4658 if (ConstantSize->isNullValue())
4662 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
4663 ConstantSize->getZExtValue(), Align, isVol,
4664 false, DstPtrInfo, SrcPtrInfo);
4665 if (Result.getNode())
4669 // Then check to see if we should lower the memmove with target-specific
4670 // code. If the target chooses to do this, this is the next best.
4672 SDValue Result = TSI->EmitTargetCodeForMemmove(
4673 *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
4674 if (Result.getNode())
4678 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
4679 checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
4681 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
4682 // not be safe. See memcpy above for more details.
4684 // Emit a library call.
4685 TargetLowering::ArgListTy Args;
4686 TargetLowering::ArgListEntry Entry;
4687 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
4688 Entry.Node = Dst; Args.push_back(Entry);
4689 Entry.Node = Src; Args.push_back(Entry);
4690 Entry.Node = Size; Args.push_back(Entry);
4691 // FIXME: pass in SDLoc
4692 TargetLowering::CallLoweringInfo CLI(*this);
4695 .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
4696 Type::getVoidTy(*getContext()),
4697 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
4698 TLI->getPointerTy(getDataLayout())),
4701 .setTailCall(isTailCall);
4703 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4704 return CallResult.second;
4707 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
4708 SDValue Src, SDValue Size,
4709 unsigned Align, bool isVol, bool isTailCall,
4710 MachinePointerInfo DstPtrInfo) {
4711 assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4713 // Check to see if we should lower the memset to stores first.
4714 // For cases within the target-specified limits, this is the best choice.
4715 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4717 // Memset with size zero? Just return the original chain.
4718 if (ConstantSize->isNullValue())
4722 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
4723 Align, isVol, DstPtrInfo);
4725 if (Result.getNode())
4729 // Then check to see if we should lower the memset with target-specific
4730 // code. If the target chooses to do this, this is the next best.
4732 SDValue Result = TSI->EmitTargetCodeForMemset(
4733 *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
4734 if (Result.getNode())
4738 checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
4740 // Emit a library call.
4741 Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
4742 TargetLowering::ArgListTy Args;
4743 TargetLowering::ArgListEntry Entry;
4744 Entry.Node = Dst; Entry.Ty = IntPtrTy;
4745 Args.push_back(Entry);
4747 Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
4748 Args.push_back(Entry);
4750 Entry.Ty = IntPtrTy;
4751 Args.push_back(Entry);
4753 // FIXME: pass in SDLoc
4754 TargetLowering::CallLoweringInfo CLI(*this);
4757 .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
4758 Type::getVoidTy(*getContext()),
4759 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
4760 TLI->getPointerTy(getDataLayout())),
4763 .setTailCall(isTailCall);
4765 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4766 return CallResult.second;
4769 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4770 SDVTList VTList, ArrayRef<SDValue> Ops,
4771 MachineMemOperand *MMO,
4772 AtomicOrdering SuccessOrdering,
4773 AtomicOrdering FailureOrdering,
4774 SynchronizationScope SynchScope) {
4775 FoldingSetNodeID ID;
4776 ID.AddInteger(MemVT.getRawBits());
4777 AddNodeIDNode(ID, Opcode, VTList, Ops);
4778 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4780 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
4781 cast<AtomicSDNode>(E)->refineAlignment(MMO);
4782 return SDValue(E, 0);
4785 // Allocate the operands array for the node out of the BumpPtrAllocator, since
4786 // SDNode doesn't have access to it. This memory will be "leaked" when
4787 // the node is deallocated, but recovered when the allocator is released.
4788 // If the number of operands is less than 5 we use AtomicSDNode's internal
4790 unsigned NumOps = Ops.size();
4791 SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
4794 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
4795 dl.getDebugLoc(), VTList, MemVT,
4796 Ops.data(), DynOps, NumOps, MMO,
4797 SuccessOrdering, FailureOrdering,
4799 CSEMap.InsertNode(N, IP);
4801 return SDValue(N, 0);
4804 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4805 SDVTList VTList, ArrayRef<SDValue> Ops,
4806 MachineMemOperand *MMO,
4807 AtomicOrdering Ordering,
4808 SynchronizationScope SynchScope) {
4809 return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
4810 Ordering, SynchScope);
4813 SDValue SelectionDAG::getAtomicCmpSwap(
4814 unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
4815 SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
4816 unsigned Alignment, AtomicOrdering SuccessOrdering,
4817 AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
4818 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4819 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
4820 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4822 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4823 Alignment = getEVTAlignment(MemVT);
4825 MachineFunction &MF = getMachineFunction();
4827 // FIXME: Volatile isn't really correct; we should keep track of atomic
4828 // orderings in the memoperand.
4829 unsigned Flags = MachineMemOperand::MOVolatile;
4830 Flags |= MachineMemOperand::MOLoad;
4831 Flags |= MachineMemOperand::MOStore;
4833 MachineMemOperand *MMO =
4834 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
4836 return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
4837 SuccessOrdering, FailureOrdering, SynchScope);
4840 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
4841 SDVTList VTs, SDValue Chain, SDValue Ptr,
4842 SDValue Cmp, SDValue Swp,
4843 MachineMemOperand *MMO,
4844 AtomicOrdering SuccessOrdering,
4845 AtomicOrdering FailureOrdering,
4846 SynchronizationScope SynchScope) {
4847 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4848 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
4849 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4851 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4852 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
4853 SuccessOrdering, FailureOrdering, SynchScope);
4856 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4858 SDValue Ptr, SDValue Val,
4859 const Value* PtrVal,
4861 AtomicOrdering Ordering,
4862 SynchronizationScope SynchScope) {
4863 if (Alignment == 0) // Ensure that codegen never sees alignment 0
4864 Alignment = getEVTAlignment(MemVT);
4866 MachineFunction &MF = getMachineFunction();
4867 // An atomic store does not load. An atomic load does not store.
4868 // (An atomicrmw obviously both loads and stores.)
4869 // For now, atomics are considered to be volatile always, and they are
4871 // FIXME: Volatile isn't really correct; we should keep track of atomic
4872 // orderings in the memoperand.
4873 unsigned Flags = MachineMemOperand::MOVolatile;
4874 if (Opcode != ISD::ATOMIC_STORE)
4875 Flags |= MachineMemOperand::MOLoad;
4876 if (Opcode != ISD::ATOMIC_LOAD)
4877 Flags |= MachineMemOperand::MOStore;
4879 MachineMemOperand *MMO =
4880 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4881 MemVT.getStoreSize(), Alignment);
4883 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
4884 Ordering, SynchScope);
4887 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4889 SDValue Ptr, SDValue Val,
4890 MachineMemOperand *MMO,
4891 AtomicOrdering Ordering,
4892 SynchronizationScope SynchScope) {
4893 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
4894 Opcode == ISD::ATOMIC_LOAD_SUB ||
4895 Opcode == ISD::ATOMIC_LOAD_AND ||
4896 Opcode == ISD::ATOMIC_LOAD_OR ||
4897 Opcode == ISD::ATOMIC_LOAD_XOR ||
4898 Opcode == ISD::ATOMIC_LOAD_NAND ||
4899 Opcode == ISD::ATOMIC_LOAD_MIN ||
4900 Opcode == ISD::ATOMIC_LOAD_MAX ||
4901 Opcode == ISD::ATOMIC_LOAD_UMIN ||
4902 Opcode == ISD::ATOMIC_LOAD_UMAX ||
4903 Opcode == ISD::ATOMIC_SWAP ||
4904 Opcode == ISD::ATOMIC_STORE) &&
4905 "Invalid Atomic Op");
4907 EVT VT = Val.getValueType();
4909 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
4910 getVTList(VT, MVT::Other);
4911 SDValue Ops[] = {Chain, Ptr, Val};
4912 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4915 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4916 EVT VT, SDValue Chain,
4918 MachineMemOperand *MMO,
4919 AtomicOrdering Ordering,
4920 SynchronizationScope SynchScope) {
4921 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
4923 SDVTList VTs = getVTList(VT, MVT::Other);
4924 SDValue Ops[] = {Chain, Ptr};
4925 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4928 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
4929 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
4930 if (Ops.size() == 1)
4933 SmallVector<EVT, 4> VTs;
4934 VTs.reserve(Ops.size());
4935 for (unsigned i = 0; i < Ops.size(); ++i)
4936 VTs.push_back(Ops[i].getValueType());
4937 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
4941 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
4942 ArrayRef<SDValue> Ops,
4943 EVT MemVT, MachinePointerInfo PtrInfo,
4944 unsigned Align, bool Vol,
4945 bool ReadMem, bool WriteMem, unsigned Size) {
4946 if (Align == 0) // Ensure that codegen never sees alignment 0
4947 Align = getEVTAlignment(MemVT);
4949 MachineFunction &MF = getMachineFunction();
4952 Flags |= MachineMemOperand::MOStore;
4954 Flags |= MachineMemOperand::MOLoad;
4956 Flags |= MachineMemOperand::MOVolatile;
4958 Size = MemVT.getStoreSize();
4959 MachineMemOperand *MMO =
4960 MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
4962 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
4966 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
4967 ArrayRef<SDValue> Ops, EVT MemVT,
4968 MachineMemOperand *MMO) {
4969 assert((Opcode == ISD::INTRINSIC_VOID ||
4970 Opcode == ISD::INTRINSIC_W_CHAIN ||
4971 Opcode == ISD::PREFETCH ||
4972 Opcode == ISD::LIFETIME_START ||
4973 Opcode == ISD::LIFETIME_END ||
4974 (Opcode <= INT_MAX &&
4975 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
4976 "Opcode is not a memory-accessing opcode!");
4978 // Memoize the node unless it returns a flag.
4979 MemIntrinsicSDNode *N;
4980 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
4981 FoldingSetNodeID ID;
4982 AddNodeIDNode(ID, Opcode, VTList, Ops);
4983 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4985 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
4986 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
4987 return SDValue(E, 0);
4990 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4991 dl.getDebugLoc(), VTList, Ops,
4993 CSEMap.InsertNode(N, IP);
4995 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4996 dl.getDebugLoc(), VTList, Ops,
5000 return SDValue(N, 0);
5003 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
5004 /// MachinePointerInfo record from it. This is particularly useful because the
5005 /// code generator has many cases where it doesn't bother passing in a
5006 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
5007 static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
5008 int64_t Offset = 0) {
5009 // If this is FI+Offset, we can model it.
5010 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
5011 return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
5012 FI->getIndex(), Offset);
5014 // If this is (FI+Offset1)+Offset2, we can model it.
5015 if (Ptr.getOpcode() != ISD::ADD ||
5016 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
5017 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
5018 return MachinePointerInfo();
5020 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
5021 return MachinePointerInfo::getFixedStack(
5022 DAG.getMachineFunction(), FI,
5023 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
5026 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
5027 /// MachinePointerInfo record from it. This is particularly useful because the
5028 /// code generator has many cases where it doesn't bother passing in a
5029 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
5030 static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
5032 // If the 'Offset' value isn't a constant, we can't handle this.
5033 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
5034 return InferPointerInfo(DAG, Ptr, OffsetNode->getSExtValue());
5035 if (OffsetOp.getOpcode() == ISD::UNDEF)
5036 return InferPointerInfo(DAG, Ptr);
5037 return MachinePointerInfo();
5042 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
5043 EVT VT, SDLoc dl, SDValue Chain,
5044 SDValue Ptr, SDValue Offset,
5045 MachinePointerInfo PtrInfo, EVT MemVT,
5046 bool isVolatile, bool isNonTemporal, bool isInvariant,
5047 unsigned Alignment, const AAMDNodes &AAInfo,
5048 const MDNode *Ranges) {
5049 assert(Chain.getValueType() == MVT::Other &&
5050 "Invalid chain type");
5051 if (Alignment == 0) // Ensure that codegen never sees alignment 0
5052 Alignment = getEVTAlignment(VT);
5054 unsigned Flags = MachineMemOperand::MOLoad;
5056 Flags |= MachineMemOperand::MOVolatile;
5058 Flags |= MachineMemOperand::MONonTemporal;
5060 Flags |= MachineMemOperand::MOInvariant;
5062 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
5064 if (PtrInfo.V.isNull())
5065 PtrInfo = InferPointerInfo(*this, Ptr, Offset);
5067 MachineFunction &MF = getMachineFunction();
5068 MachineMemOperand *MMO =
5069 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
5071 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
5075 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
5076 EVT VT, SDLoc dl, SDValue Chain,
5077 SDValue Ptr, SDValue Offset, EVT MemVT,
5078 MachineMemOperand *MMO) {
5080 ExtType = ISD::NON_EXTLOAD;
5081 } else if (ExtType == ISD::NON_EXTLOAD) {
5082 assert(VT == MemVT && "Non-extending load from different memory type!");
5085 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
5086 "Should only be an extending load, not truncating!");
5087 assert(VT.isInteger() == MemVT.isInteger() &&
5088 "Cannot convert from FP to Int or Int -> FP!");
5089 assert(VT.isVector() == MemVT.isVector() &&
5090 "Cannot use an ext load to convert to or from a vector!");
5091 assert((!VT.isVector() ||
5092 VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
5093 "Cannot use an ext load to change the number of vector elements!");
5096 bool Indexed = AM != ISD::UNINDEXED;
5097 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
5098 "Unindexed load with an offset!");
5100 SDVTList VTs = Indexed ?
5101 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
5102 SDValue Ops[] = { Chain, Ptr, Offset };
5103 FoldingSetNodeID ID;
5104 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
5105 ID.AddInteger(MemVT.getRawBits());
5106 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
5107 MMO->isNonTemporal(),
5108 MMO->isInvariant()));
5109 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5111 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5112 cast<LoadSDNode>(E)->refineAlignment(MMO);
5113 return SDValue(E, 0);
5115 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
5116 dl.getDebugLoc(), VTs, AM, ExtType,
5118 CSEMap.InsertNode(N, IP);
5120 return SDValue(N, 0);
5123 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
5124 SDValue Chain, SDValue Ptr,
5125 MachinePointerInfo PtrInfo,
5126 bool isVolatile, bool isNonTemporal,
5127 bool isInvariant, unsigned Alignment,
5128 const AAMDNodes &AAInfo,
5129 const MDNode *Ranges) {
5130 SDValue Undef = getUNDEF(Ptr.getValueType());
5131 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
5132 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
5136 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
5137 SDValue Chain, SDValue Ptr,
5138 MachineMemOperand *MMO) {
5139 SDValue Undef = getUNDEF(Ptr.getValueType());
5140 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
5144 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
5145 SDValue Chain, SDValue Ptr,
5146 MachinePointerInfo PtrInfo, EVT MemVT,
5147 bool isVolatile, bool isNonTemporal,
5148 bool isInvariant, unsigned Alignment,
5149 const AAMDNodes &AAInfo) {
5150 SDValue Undef = getUNDEF(Ptr.getValueType());
5151 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
5152 PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
5157 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
5158 SDValue Chain, SDValue Ptr, EVT MemVT,
5159 MachineMemOperand *MMO) {
5160 SDValue Undef = getUNDEF(Ptr.getValueType());
5161 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
5166 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
5167 SDValue Offset, ISD::MemIndexedMode AM) {
5168 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
5169 assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
5170 "Load is already a indexed load!");
5171 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
5172 LD->getChain(), Base, Offset, LD->getPointerInfo(),
5173 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
5174 false, LD->getAlignment());
5177 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
5178 SDValue Ptr, MachinePointerInfo PtrInfo,
5179 bool isVolatile, bool isNonTemporal,
5180 unsigned Alignment, const AAMDNodes &AAInfo) {
5181 assert(Chain.getValueType() == MVT::Other &&
5182 "Invalid chain type");
5183 if (Alignment == 0) // Ensure that codegen never sees alignment 0
5184 Alignment = getEVTAlignment(Val.getValueType());
5186 unsigned Flags = MachineMemOperand::MOStore;
5188 Flags |= MachineMemOperand::MOVolatile;
5190 Flags |= MachineMemOperand::MONonTemporal;
5192 if (PtrInfo.V.isNull())
5193 PtrInfo = InferPointerInfo(*this, Ptr);
5195 MachineFunction &MF = getMachineFunction();
5196 MachineMemOperand *MMO =
5197 MF.getMachineMemOperand(PtrInfo, Flags,
5198 Val.getValueType().getStoreSize(), Alignment,
5201 return getStore(Chain, dl, Val, Ptr, MMO);
5204 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
5205 SDValue Ptr, MachineMemOperand *MMO) {
5206 assert(Chain.getValueType() == MVT::Other &&
5207 "Invalid chain type");
5208 EVT VT = Val.getValueType();
5209 SDVTList VTs = getVTList(MVT::Other);
5210 SDValue Undef = getUNDEF(Ptr.getValueType());
5211 SDValue Ops[] = { Chain, Val, Ptr, Undef };
5212 FoldingSetNodeID ID;
5213 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5214 ID.AddInteger(VT.getRawBits());
5215 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
5216 MMO->isNonTemporal(), MMO->isInvariant()));
5217 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5219 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5220 cast<StoreSDNode>(E)->refineAlignment(MMO);
5221 return SDValue(E, 0);
5223 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
5224 dl.getDebugLoc(), VTs,
5225 ISD::UNINDEXED, false, VT, MMO);
5226 CSEMap.InsertNode(N, IP);
5228 return SDValue(N, 0);
5231 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
5232 SDValue Ptr, MachinePointerInfo PtrInfo,
5233 EVT SVT,bool isVolatile, bool isNonTemporal,
5235 const AAMDNodes &AAInfo) {
5236 assert(Chain.getValueType() == MVT::Other &&
5237 "Invalid chain type");
5238 if (Alignment == 0) // Ensure that codegen never sees alignment 0
5239 Alignment = getEVTAlignment(SVT);
5241 unsigned Flags = MachineMemOperand::MOStore;
5243 Flags |= MachineMemOperand::MOVolatile;
5245 Flags |= MachineMemOperand::MONonTemporal;
5247 if (PtrInfo.V.isNull())
5248 PtrInfo = InferPointerInfo(*this, Ptr);
5250 MachineFunction &MF = getMachineFunction();
5251 MachineMemOperand *MMO =
5252 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
5255 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
5258 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
5259 SDValue Ptr, EVT SVT,
5260 MachineMemOperand *MMO) {
5261 EVT VT = Val.getValueType();
5263 assert(Chain.getValueType() == MVT::Other &&
5264 "Invalid chain type");
5266 return getStore(Chain, dl, Val, Ptr, MMO);
5268 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
5269 "Should only be a truncating store, not extending!");
5270 assert(VT.isInteger() == SVT.isInteger() &&
5271 "Can't do FP-INT conversion!");
5272 assert(VT.isVector() == SVT.isVector() &&
5273 "Cannot use trunc store to convert to or from a vector!");
5274 assert((!VT.isVector() ||
5275 VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
5276 "Cannot use trunc store to change the number of vector elements!");
5278 SDVTList VTs = getVTList(MVT::Other);
5279 SDValue Undef = getUNDEF(Ptr.getValueType());
5280 SDValue Ops[] = { Chain, Val, Ptr, Undef };
5281 FoldingSetNodeID ID;
5282 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5283 ID.AddInteger(SVT.getRawBits());
5284 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
5285 MMO->isNonTemporal(), MMO->isInvariant()));
5286 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5288 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5289 cast<StoreSDNode>(E)->refineAlignment(MMO);
5290 return SDValue(E, 0);
5292 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
5293 dl.getDebugLoc(), VTs,
5294 ISD::UNINDEXED, true, SVT, MMO);
5295 CSEMap.InsertNode(N, IP);
5297 return SDValue(N, 0);
5301 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
5302 SDValue Offset, ISD::MemIndexedMode AM) {
5303 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
5304 assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
5305 "Store is already a indexed store!");
5306 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
5307 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
5308 FoldingSetNodeID ID;
5309 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5310 ID.AddInteger(ST->getMemoryVT().getRawBits());
5311 ID.AddInteger(ST->getRawSubclassData());
5312 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
5314 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
5315 return SDValue(E, 0);
5317 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
5318 dl.getDebugLoc(), VTs, AM,
5319 ST->isTruncatingStore(),
5321 ST->getMemOperand());
5322 CSEMap.InsertNode(N, IP);
5324 return SDValue(N, 0);
5328 SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
5329 SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT,
5330 MachineMemOperand *MMO, ISD::LoadExtType ExtTy) {
5332 SDVTList VTs = getVTList(VT, MVT::Other);
5333 SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
5334 FoldingSetNodeID ID;
5335 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
5336 ID.AddInteger(VT.getRawBits());
5337 ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED,
5339 MMO->isNonTemporal(),
5340 MMO->isInvariant()));
5341 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5343 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5344 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
5345 return SDValue(E, 0);
5347 SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
5348 dl.getDebugLoc(), Ops, 4, VTs,
5350 CSEMap.InsertNode(N, IP);
5352 return SDValue(N, 0);
5355 SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
5356 SDValue Ptr, SDValue Mask, EVT MemVT,
5357 MachineMemOperand *MMO, bool isTrunc) {
5358 assert(Chain.getValueType() == MVT::Other &&
5359 "Invalid chain type");
5360 EVT VT = Val.getValueType();
5361 SDVTList VTs = getVTList(MVT::Other);
5362 SDValue Ops[] = { Chain, Ptr, Mask, Val };
5363 FoldingSetNodeID ID;
5364 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
5365 ID.AddInteger(VT.getRawBits());
5366 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
5367 MMO->isNonTemporal(), MMO->isInvariant()));
5368 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5370 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5371 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
5372 return SDValue(E, 0);
5374 SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
5375 dl.getDebugLoc(), Ops, 4,
5376 VTs, isTrunc, MemVT, MMO);
5377 CSEMap.InsertNode(N, IP);
5379 return SDValue(N, 0);
5383 SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
5384 ArrayRef<SDValue> Ops,
5385 MachineMemOperand *MMO) {
5387 FoldingSetNodeID ID;
5388 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
5389 ID.AddInteger(VT.getRawBits());
5390 ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED,
5392 MMO->isNonTemporal(),
5393 MMO->isInvariant()));
5394 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5396 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5397 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
5398 return SDValue(E, 0);
5400 MaskedGatherSDNode *N =
5401 new (NodeAllocator) MaskedGatherSDNode(dl.getIROrder(), dl.getDebugLoc(),
5403 CSEMap.InsertNode(N, IP);
5405 return SDValue(N, 0);
5408 SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
5409 ArrayRef<SDValue> Ops,
5410 MachineMemOperand *MMO) {
5411 FoldingSetNodeID ID;
5412 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
5413 ID.AddInteger(VT.getRawBits());
5414 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
5415 MMO->isNonTemporal(),
5416 MMO->isInvariant()));
5417 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5419 if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5420 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
5421 return SDValue(E, 0);
5424 new (NodeAllocator) MaskedScatterSDNode(dl.getIROrder(), dl.getDebugLoc(),
5426 CSEMap.InsertNode(N, IP);
5428 return SDValue(N, 0);
5431 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
5432 SDValue Chain, SDValue Ptr,
5435 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
5436 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
5439 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
5440 ArrayRef<SDUse> Ops) {
5441 switch (Ops.size()) {
5442 case 0: return getNode(Opcode, DL, VT);
5443 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
5444 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
5445 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
5449 // Copy from an SDUse array into an SDValue array for use with
5450 // the regular getNode logic.
5451 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
5452 return getNode(Opcode, DL, VT, NewOps);
5455 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
5456 ArrayRef<SDValue> Ops, const SDNodeFlags *Flags) {
5457 unsigned NumOps = Ops.size();
5459 case 0: return getNode(Opcode, DL, VT);
5460 case 1: return getNode(Opcode, DL, VT, Ops[0]);
5461 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
5462 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
5468 case ISD::CONCAT_VECTORS: {
5469 // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
5470 if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
5474 case ISD::SELECT_CC: {
5475 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
5476 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
5477 "LHS and RHS of condition must have same type!");
5478 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
5479 "True and False arms of SelectCC must have same type!");
5480 assert(Ops[2].getValueType() == VT &&
5481 "select_cc node must be of same type as true and false value!");
5485 assert(NumOps == 5 && "BR_CC takes 5 operands!");
5486 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
5487 "LHS/RHS of comparison should match types!");
5494 SDVTList VTs = getVTList(VT);
5496 if (VT != MVT::Glue) {
5497 FoldingSetNodeID ID;
5498 AddNodeIDNode(ID, Opcode, VTs, Ops);
5501 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
5502 return SDValue(E, 0);
5504 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5506 CSEMap.InsertNode(N, IP);
5508 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5513 return SDValue(N, 0);
5516 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
5517 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
5518 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
5521 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5522 ArrayRef<SDValue> Ops) {
5523 if (VTList.NumVTs == 1)
5524 return getNode(Opcode, DL, VTList.VTs[0], Ops);
5528 // FIXME: figure out how to safely handle things like
5529 // int foo(int x) { return 1 << (x & 255); }
5530 // int bar() { return foo(256); }
5531 case ISD::SRA_PARTS:
5532 case ISD::SRL_PARTS:
5533 case ISD::SHL_PARTS:
5534 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5535 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
5536 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
5537 else if (N3.getOpcode() == ISD::AND)
5538 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
5539 // If the and is only masking out bits that cannot effect the shift,
5540 // eliminate the and.
5541 unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
5542 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
5543 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
5549 // Memoize the node unless it returns a flag.
5551 unsigned NumOps = Ops.size();
5552 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
5553 FoldingSetNodeID ID;
5554 AddNodeIDNode(ID, Opcode, VTList, Ops);
5556 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
5557 return SDValue(E, 0);
5560 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
5561 DL.getDebugLoc(), VTList, Ops[0]);
5562 } else if (NumOps == 2) {
5563 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
5564 DL.getDebugLoc(), VTList, Ops[0],
5566 } else if (NumOps == 3) {
5567 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
5568 DL.getDebugLoc(), VTList, Ops[0],
5571 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5574 CSEMap.InsertNode(N, IP);
5577 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
5578 DL.getDebugLoc(), VTList, Ops[0]);
5579 } else if (NumOps == 2) {
5580 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
5581 DL.getDebugLoc(), VTList, Ops[0],
5583 } else if (NumOps == 3) {
5584 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
5585 DL.getDebugLoc(), VTList, Ops[0],
5588 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5593 return SDValue(N, 0);
5596 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
5597 return getNode(Opcode, DL, VTList, None);
5600 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5602 SDValue Ops[] = { N1 };
5603 return getNode(Opcode, DL, VTList, Ops);
5606 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5607 SDValue N1, SDValue N2) {
5608 SDValue Ops[] = { N1, N2 };
5609 return getNode(Opcode, DL, VTList, Ops);
5612 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5613 SDValue N1, SDValue N2, SDValue N3) {
5614 SDValue Ops[] = { N1, N2, N3 };
5615 return getNode(Opcode, DL, VTList, Ops);
5618 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5619 SDValue N1, SDValue N2, SDValue N3,
5621 SDValue Ops[] = { N1, N2, N3, N4 };
5622 return getNode(Opcode, DL, VTList, Ops);
5625 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5626 SDValue N1, SDValue N2, SDValue N3,
5627 SDValue N4, SDValue N5) {
5628 SDValue Ops[] = { N1, N2, N3, N4, N5 };
5629 return getNode(Opcode, DL, VTList, Ops);
5632 SDVTList SelectionDAG::getVTList(EVT VT) {
5633 return makeVTList(SDNode::getValueTypeList(VT), 1);
5636 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
5637 FoldingSetNodeID ID;
5639 ID.AddInteger(VT1.getRawBits());
5640 ID.AddInteger(VT2.getRawBits());
5643 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5645 EVT *Array = Allocator.Allocate<EVT>(2);
5648 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
5649 VTListMap.InsertNode(Result, IP);
5651 return Result->getSDVTList();
5654 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
5655 FoldingSetNodeID ID;
5657 ID.AddInteger(VT1.getRawBits());
5658 ID.AddInteger(VT2.getRawBits());
5659 ID.AddInteger(VT3.getRawBits());
5662 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5664 EVT *Array = Allocator.Allocate<EVT>(3);
5668 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
5669 VTListMap.InsertNode(Result, IP);
5671 return Result->getSDVTList();
5674 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
5675 FoldingSetNodeID ID;
5677 ID.AddInteger(VT1.getRawBits());
5678 ID.AddInteger(VT2.getRawBits());
5679 ID.AddInteger(VT3.getRawBits());
5680 ID.AddInteger(VT4.getRawBits());
5683 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5685 EVT *Array = Allocator.Allocate<EVT>(4);
5690 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
5691 VTListMap.InsertNode(Result, IP);
5693 return Result->getSDVTList();
5696 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
5697 unsigned NumVTs = VTs.size();
5698 FoldingSetNodeID ID;
5699 ID.AddInteger(NumVTs);
5700 for (unsigned index = 0; index < NumVTs; index++) {
5701 ID.AddInteger(VTs[index].getRawBits());
5705 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5707 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
5708 std::copy(VTs.begin(), VTs.end(), Array);
5709 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
5710 VTListMap.InsertNode(Result, IP);
5712 return Result->getSDVTList();
5716 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
5717 /// specified operands. If the resultant node already exists in the DAG,
5718 /// this does not modify the specified node, instead it returns the node that
5719 /// already exists. If the resultant node does not exist in the DAG, the
5720 /// input node is returned. As a degenerate case, if you specify the same
5721 /// input operands as the node already has, the input node is returned.
5722 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
5723 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
5725 // Check to see if there is no change.
5726 if (Op == N->getOperand(0)) return N;
5728 // See if the modified node already exists.
5729 void *InsertPos = nullptr;
5730 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
5733 // Nope it doesn't. Remove the node from its current place in the maps.
5735 if (!RemoveNodeFromCSEMaps(N))
5736 InsertPos = nullptr;
5738 // Now we update the operands.
5739 N->OperandList[0].set(Op);
5741 // If this gets put into a CSE map, add it.
5742 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5746 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
5747 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
5749 // Check to see if there is no change.
5750 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
5751 return N; // No operands changed, just return the input node.
5753 // See if the modified node already exists.
5754 void *InsertPos = nullptr;
5755 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
5758 // Nope it doesn't. Remove the node from its current place in the maps.
5760 if (!RemoveNodeFromCSEMaps(N))
5761 InsertPos = nullptr;
5763 // Now we update the operands.
5764 if (N->OperandList[0] != Op1)
5765 N->OperandList[0].set(Op1);
5766 if (N->OperandList[1] != Op2)
5767 N->OperandList[1].set(Op2);
5769 // If this gets put into a CSE map, add it.
5770 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5774 SDNode *SelectionDAG::
5775 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
5776 SDValue Ops[] = { Op1, Op2, Op3 };
5777 return UpdateNodeOperands(N, Ops);
5780 SDNode *SelectionDAG::
5781 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
5782 SDValue Op3, SDValue Op4) {
5783 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
5784 return UpdateNodeOperands(N, Ops);
5787 SDNode *SelectionDAG::
5788 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
5789 SDValue Op3, SDValue Op4, SDValue Op5) {
5790 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
5791 return UpdateNodeOperands(N, Ops);
5794 SDNode *SelectionDAG::
5795 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
5796 unsigned NumOps = Ops.size();
5797 assert(N->getNumOperands() == NumOps &&
5798 "Update with wrong number of operands");
5800 // If no operands changed just return the input node.
5801 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
5804 // See if the modified node already exists.
5805 void *InsertPos = nullptr;
5806 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
5809 // Nope it doesn't. Remove the node from its current place in the maps.
5811 if (!RemoveNodeFromCSEMaps(N))
5812 InsertPos = nullptr;
5814 // Now we update the operands.
5815 for (unsigned i = 0; i != NumOps; ++i)
5816 if (N->OperandList[i] != Ops[i])
5817 N->OperandList[i].set(Ops[i]);
5819 // If this gets put into a CSE map, add it.
5820 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5824 /// DropOperands - Release the operands and set this node to have
5826 void SDNode::DropOperands() {
5827 // Unlike the code in MorphNodeTo that does this, we don't need to
5828 // watch for dead nodes here.
5829 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
5835 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
5838 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5840 SDVTList VTs = getVTList(VT);
5841 return SelectNodeTo(N, MachineOpc, VTs, None);
5844 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5845 EVT VT, SDValue Op1) {
5846 SDVTList VTs = getVTList(VT);
5847 SDValue Ops[] = { Op1 };
5848 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5851 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5852 EVT VT, SDValue Op1,
5854 SDVTList VTs = getVTList(VT);
5855 SDValue Ops[] = { Op1, Op2 };
5856 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5859 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5860 EVT VT, SDValue Op1,
5861 SDValue Op2, SDValue Op3) {
5862 SDVTList VTs = getVTList(VT);
5863 SDValue Ops[] = { Op1, Op2, Op3 };
5864 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5867 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5868 EVT VT, ArrayRef<SDValue> Ops) {
5869 SDVTList VTs = getVTList(VT);
5870 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5873 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5874 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
5875 SDVTList VTs = getVTList(VT1, VT2);
5876 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5879 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5881 SDVTList VTs = getVTList(VT1, VT2);
5882 return SelectNodeTo(N, MachineOpc, VTs, None);
5885 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5886 EVT VT1, EVT VT2, EVT VT3,
5887 ArrayRef<SDValue> Ops) {
5888 SDVTList VTs = getVTList(VT1, VT2, VT3);
5889 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5892 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5893 EVT VT1, EVT VT2, EVT VT3, EVT VT4,
5894 ArrayRef<SDValue> Ops) {
5895 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5896 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5899 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5902 SDVTList VTs = getVTList(VT1, VT2);
5903 SDValue Ops[] = { Op1 };
5904 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5907 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5909 SDValue Op1, SDValue Op2) {
5910 SDVTList VTs = getVTList(VT1, VT2);
5911 SDValue Ops[] = { Op1, Op2 };
5912 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5915 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5917 SDValue Op1, SDValue Op2,
5919 SDVTList VTs = getVTList(VT1, VT2);
5920 SDValue Ops[] = { Op1, Op2, Op3 };
5921 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5924 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5925 EVT VT1, EVT VT2, EVT VT3,
5926 SDValue Op1, SDValue Op2,
5928 SDVTList VTs = getVTList(VT1, VT2, VT3);
5929 SDValue Ops[] = { Op1, Op2, Op3 };
5930 return SelectNodeTo(N, MachineOpc, VTs, Ops);
5933 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5934 SDVTList VTs,ArrayRef<SDValue> Ops) {
5935 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
5936 // Reset the NodeID to -1.
5941 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
5942 /// the line number information on the merged node since it is not possible to
5943 /// preserve the information that operation is associated with multiple lines.
5944 /// This will make the debugger working better at -O0, were there is a higher
5945 /// probability having other instructions associated with that line.
5947 /// For IROrder, we keep the smaller of the two
5948 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
5949 DebugLoc NLoc = N->getDebugLoc();
5950 if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
5951 N->setDebugLoc(DebugLoc());
5953 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
5954 N->setIROrder(Order);
5958 /// MorphNodeTo - This *mutates* the specified node to have the specified
5959 /// return type, opcode, and operands.
5961 /// Note that MorphNodeTo returns the resultant node. If there is already a
5962 /// node of the specified opcode and operands, it returns that node instead of
5963 /// the current one. Note that the SDLoc need not be the same.
5965 /// Using MorphNodeTo is faster than creating a new node and swapping it in
5966 /// with ReplaceAllUsesWith both because it often avoids allocating a new
5967 /// node, and because it doesn't require CSE recalculation for any of
5968 /// the node's users.
5970 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
5971 /// As a consequence it isn't appropriate to use from within the DAG combiner or
5972 /// the legalizer which maintain worklists that would need to be updated when
5973 /// deleting things.
5974 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
5975 SDVTList VTs, ArrayRef<SDValue> Ops) {
5976 unsigned NumOps = Ops.size();
5977 // If an identical node already exists, use it.
5979 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
5980 FoldingSetNodeID ID;
5981 AddNodeIDNode(ID, Opc, VTs, Ops);
5982 if (SDNode *ON = FindNodeOrInsertPos(ID, N->getDebugLoc(), IP))
5983 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
5986 if (!RemoveNodeFromCSEMaps(N))
5989 // Start the morphing.
5991 N->ValueList = VTs.VTs;
5992 N->NumValues = VTs.NumVTs;
5994 // Clear the operands list, updating used nodes to remove this from their
5995 // use list. Keep track of any operands that become dead as a result.
5996 SmallPtrSet<SDNode*, 16> DeadNodeSet;
5997 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
5999 SDNode *Used = Use.getNode();
6001 if (Used->use_empty())
6002 DeadNodeSet.insert(Used);
6005 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
6006 // Initialize the memory references information.
6007 MN->setMemRefs(nullptr, nullptr);
6008 // If NumOps is larger than the # of operands we can have in a
6009 // MachineSDNode, reallocate the operand list.
6010 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
6011 if (MN->OperandsNeedDelete)
6012 delete[] MN->OperandList;
6013 if (NumOps > array_lengthof(MN->LocalOperands))
6014 // We're creating a final node that will live unmorphed for the
6015 // remainder of the current SelectionDAG iteration, so we can allocate
6016 // the operands directly out of a pool with no recycling metadata.
6017 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
6018 Ops.data(), NumOps);
6020 MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
6021 MN->OperandsNeedDelete = false;
6023 MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
6025 // If NumOps is larger than the # of operands we currently have, reallocate
6026 // the operand list.
6027 if (NumOps > N->NumOperands) {
6028 if (N->OperandsNeedDelete)
6029 delete[] N->OperandList;
6030 N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
6031 N->OperandsNeedDelete = true;
6033 N->InitOperands(N->OperandList, Ops.data(), NumOps);
6036 // Delete any nodes that are still dead after adding the uses for the
6038 if (!DeadNodeSet.empty()) {
6039 SmallVector<SDNode *, 16> DeadNodes;
6040 for (SDNode *N : DeadNodeSet)
6042 DeadNodes.push_back(N);
6043 RemoveDeadNodes(DeadNodes);
6047 CSEMap.InsertNode(N, IP); // Memoize the new node.
6052 /// getMachineNode - These are used for target selectors to create a new node
6053 /// with specified return type(s), MachineInstr opcode, and operands.
6055 /// Note that getMachineNode returns the resultant node. If there is already a
6056 /// node of the specified opcode and operands, it returns that node instead of
6057 /// the current one.
6059 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
6060 SDVTList VTs = getVTList(VT);
6061 return getMachineNode(Opcode, dl, VTs, None);
6065 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
6066 SDVTList VTs = getVTList(VT);
6067 SDValue Ops[] = { Op1 };
6068 return getMachineNode(Opcode, dl, VTs, Ops);
6072 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
6073 SDValue Op1, SDValue Op2) {
6074 SDVTList VTs = getVTList(VT);
6075 SDValue Ops[] = { Op1, Op2 };
6076 return getMachineNode(Opcode, dl, VTs, Ops);
6080 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
6081 SDValue Op1, SDValue Op2, SDValue Op3) {
6082 SDVTList VTs = getVTList(VT);
6083 SDValue Ops[] = { Op1, Op2, Op3 };
6084 return getMachineNode(Opcode, dl, VTs, Ops);
6088 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
6089 ArrayRef<SDValue> Ops) {
6090 SDVTList VTs = getVTList(VT);
6091 return getMachineNode(Opcode, dl, VTs, Ops);
6095 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
6096 SDVTList VTs = getVTList(VT1, VT2);
6097 return getMachineNode(Opcode, dl, VTs, None);
6101 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6102 EVT VT1, EVT VT2, SDValue Op1) {
6103 SDVTList VTs = getVTList(VT1, VT2);
6104 SDValue Ops[] = { Op1 };
6105 return getMachineNode(Opcode, dl, VTs, Ops);
6109 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6110 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
6111 SDVTList VTs = getVTList(VT1, VT2);
6112 SDValue Ops[] = { Op1, Op2 };
6113 return getMachineNode(Opcode, dl, VTs, Ops);
6117 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6118 EVT VT1, EVT VT2, SDValue Op1,
6119 SDValue Op2, SDValue Op3) {
6120 SDVTList VTs = getVTList(VT1, VT2);
6121 SDValue Ops[] = { Op1, Op2, Op3 };
6122 return getMachineNode(Opcode, dl, VTs, Ops);
6126 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6128 ArrayRef<SDValue> Ops) {
6129 SDVTList VTs = getVTList(VT1, VT2);
6130 return getMachineNode(Opcode, dl, VTs, Ops);
6134 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6135 EVT VT1, EVT VT2, EVT VT3,
6136 SDValue Op1, SDValue Op2) {
6137 SDVTList VTs = getVTList(VT1, VT2, VT3);
6138 SDValue Ops[] = { Op1, Op2 };
6139 return getMachineNode(Opcode, dl, VTs, Ops);
6143 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6144 EVT VT1, EVT VT2, EVT VT3,
6145 SDValue Op1, SDValue Op2, SDValue Op3) {
6146 SDVTList VTs = getVTList(VT1, VT2, VT3);
6147 SDValue Ops[] = { Op1, Op2, Op3 };
6148 return getMachineNode(Opcode, dl, VTs, Ops);
6152 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6153 EVT VT1, EVT VT2, EVT VT3,
6154 ArrayRef<SDValue> Ops) {
6155 SDVTList VTs = getVTList(VT1, VT2, VT3);
6156 return getMachineNode(Opcode, dl, VTs, Ops);
6160 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
6161 EVT VT2, EVT VT3, EVT VT4,
6162 ArrayRef<SDValue> Ops) {
6163 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
6164 return getMachineNode(Opcode, dl, VTs, Ops);
6168 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
6169 ArrayRef<EVT> ResultTys,
6170 ArrayRef<SDValue> Ops) {
6171 SDVTList VTs = getVTList(ResultTys);
6172 return getMachineNode(Opcode, dl, VTs, Ops);
6176 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
6177 ArrayRef<SDValue> OpsArray) {
6178 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
6181 const SDValue *Ops = OpsArray.data();
6182 unsigned NumOps = OpsArray.size();
6185 FoldingSetNodeID ID;
6186 AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
6188 if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
6189 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
6193 // Allocate a new MachineSDNode.
6194 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
6195 DL.getDebugLoc(), VTs);
6197 // Initialize the operands list.
6198 if (NumOps > array_lengthof(N->LocalOperands))
6199 // We're creating a final node that will live unmorphed for the
6200 // remainder of the current SelectionDAG iteration, so we can allocate
6201 // the operands directly out of a pool with no recycling metadata.
6202 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
6205 N->InitOperands(N->LocalOperands, Ops, NumOps);
6206 N->OperandsNeedDelete = false;
6209 CSEMap.InsertNode(N, IP);
6215 /// getTargetExtractSubreg - A convenience function for creating
6216 /// TargetOpcode::EXTRACT_SUBREG nodes.
6218 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
6220 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
6221 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
6222 VT, Operand, SRIdxVal);
6223 return SDValue(Subreg, 0);
6226 /// getTargetInsertSubreg - A convenience function for creating
6227 /// TargetOpcode::INSERT_SUBREG nodes.
6229 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
6230 SDValue Operand, SDValue Subreg) {
6231 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
6232 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
6233 VT, Operand, Subreg, SRIdxVal);
6234 return SDValue(Result, 0);
6237 /// getNodeIfExists - Get the specified node if it's already available, or
6238 /// else return NULL.
6239 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
6240 ArrayRef<SDValue> Ops,
6241 const SDNodeFlags *Flags) {
6242 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
6243 FoldingSetNodeID ID;
6244 AddNodeIDNode(ID, Opcode, VTList, Ops);
6246 if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP)) {
6248 E->intersectFlagsWith(Flags);
6255 /// getDbgValue - Creates a SDDbgValue node.
6258 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
6259 unsigned R, bool IsIndirect, uint64_t Off,
6260 DebugLoc DL, unsigned O) {
6261 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6262 "Expected inlined-at fields to agree");
6263 return new (DbgInfo->getAlloc())
6264 SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
6268 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
6269 const Value *C, uint64_t Off,
6270 DebugLoc DL, unsigned O) {
6271 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6272 "Expected inlined-at fields to agree");
6273 return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, Off, DL, O);
6277 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
6278 unsigned FI, uint64_t Off,
6279 DebugLoc DL, unsigned O) {
6280 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6281 "Expected inlined-at fields to agree");
6282 return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, Off, DL, O);
6287 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
6288 /// pointed to by a use iterator is deleted, increment the use iterator
6289 /// so that it doesn't dangle.
6291 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
6292 SDNode::use_iterator &UI;
6293 SDNode::use_iterator &UE;
6295 void NodeDeleted(SDNode *N, SDNode *E) override {
6296 // Increment the iterator as needed.
6297 while (UI != UE && N == *UI)
6302 RAUWUpdateListener(SelectionDAG &d,
6303 SDNode::use_iterator &ui,
6304 SDNode::use_iterator &ue)
6305 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
6310 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
6311 /// This can cause recursive merging of nodes in the DAG.
6313 /// This version assumes From has a single result value.
6315 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
6316 SDNode *From = FromN.getNode();
6317 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
6318 "Cannot replace with this method!");
6319 assert(From != To.getNode() && "Cannot replace uses of with self");
6321 // Iterate over all the existing uses of From. New uses will be added
6322 // to the beginning of the use list, which we avoid visiting.
6323 // This specifically avoids visiting uses of From that arise while the
6324 // replacement is happening, because any such uses would be the result
6325 // of CSE: If an existing node looks like From after one of its operands
6326 // is replaced by To, we don't want to replace of all its users with To
6327 // too. See PR3018 for more info.
6328 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
6329 RAUWUpdateListener Listener(*this, UI, UE);
6333 // This node is about to morph, remove its old self from the CSE maps.
6334 RemoveNodeFromCSEMaps(User);
6336 // A user can appear in a use list multiple times, and when this
6337 // happens the uses are usually next to each other in the list.
6338 // To help reduce the number of CSE recomputations, process all
6339 // the uses of this user that we can find this way.
6341 SDUse &Use = UI.getUse();
6344 } while (UI != UE && *UI == User);
6346 // Now that we have modified User, add it back to the CSE maps. If it
6347 // already exists there, recursively merge the results together.
6348 AddModifiedNodeToCSEMaps(User);
6351 // If we just RAUW'd the root, take note.
6352 if (FromN == getRoot())
6356 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
6357 /// This can cause recursive merging of nodes in the DAG.
6359 /// This version assumes that for each value of From, there is a
6360 /// corresponding value in To in the same position with the same type.
6362 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
6364 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
6365 assert((!From->hasAnyUseOfValue(i) ||
6366 From->getValueType(i) == To->getValueType(i)) &&
6367 "Cannot use this version of ReplaceAllUsesWith!");
6370 // Handle the trivial case.
6374 // Iterate over just the existing users of From. See the comments in
6375 // the ReplaceAllUsesWith above.
6376 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
6377 RAUWUpdateListener Listener(*this, UI, UE);
6381 // This node is about to morph, remove its old self from the CSE maps.
6382 RemoveNodeFromCSEMaps(User);
6384 // A user can appear in a use list multiple times, and when this
6385 // happens the uses are usually next to each other in the list.
6386 // To help reduce the number of CSE recomputations, process all
6387 // the uses of this user that we can find this way.
6389 SDUse &Use = UI.getUse();
6392 } while (UI != UE && *UI == User);
6394 // Now that we have modified User, add it back to the CSE maps. If it
6395 // already exists there, recursively merge the results together.
6396 AddModifiedNodeToCSEMaps(User);
6399 // If we just RAUW'd the root, take note.
6400 if (From == getRoot().getNode())
6401 setRoot(SDValue(To, getRoot().getResNo()));
6404 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
6405 /// This can cause recursive merging of nodes in the DAG.
6407 /// This version can replace From with any result values. To must match the
6408 /// number and types of values returned by From.
6409 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
6410 if (From->getNumValues() == 1) // Handle the simple case efficiently.
6411 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
6413 // Iterate over just the existing users of From. See the comments in
6414 // the ReplaceAllUsesWith above.
6415 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
6416 RAUWUpdateListener Listener(*this, UI, UE);
6420 // This node is about to morph, remove its old self from the CSE maps.
6421 RemoveNodeFromCSEMaps(User);
6423 // A user can appear in a use list multiple times, and when this
6424 // happens the uses are usually next to each other in the list.
6425 // To help reduce the number of CSE recomputations, process all
6426 // the uses of this user that we can find this way.
6428 SDUse &Use = UI.getUse();
6429 const SDValue &ToOp = To[Use.getResNo()];
6432 } while (UI != UE && *UI == User);
6434 // Now that we have modified User, add it back to the CSE maps. If it
6435 // already exists there, recursively merge the results together.
6436 AddModifiedNodeToCSEMaps(User);
6439 // If we just RAUW'd the root, take note.
6440 if (From == getRoot().getNode())
6441 setRoot(SDValue(To[getRoot().getResNo()]));
6444 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
6445 /// uses of other values produced by From.getNode() alone. The Deleted
6446 /// vector is handled the same way as for ReplaceAllUsesWith.
6447 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
6448 // Handle the really simple, really trivial case efficiently.
6449 if (From == To) return;
6451 // Handle the simple, trivial, case efficiently.
6452 if (From.getNode()->getNumValues() == 1) {
6453 ReplaceAllUsesWith(From, To);
6457 // Iterate over just the existing users of From. See the comments in
6458 // the ReplaceAllUsesWith above.
6459 SDNode::use_iterator UI = From.getNode()->use_begin(),
6460 UE = From.getNode()->use_end();
6461 RAUWUpdateListener Listener(*this, UI, UE);
6464 bool UserRemovedFromCSEMaps = false;
6466 // A user can appear in a use list multiple times, and when this
6467 // happens the uses are usually next to each other in the list.
6468 // To help reduce the number of CSE recomputations, process all
6469 // the uses of this user that we can find this way.
6471 SDUse &Use = UI.getUse();
6473 // Skip uses of different values from the same node.
6474 if (Use.getResNo() != From.getResNo()) {
6479 // If this node hasn't been modified yet, it's still in the CSE maps,
6480 // so remove its old self from the CSE maps.
6481 if (!UserRemovedFromCSEMaps) {
6482 RemoveNodeFromCSEMaps(User);
6483 UserRemovedFromCSEMaps = true;
6488 } while (UI != UE && *UI == User);
6490 // We are iterating over all uses of the From node, so if a use
6491 // doesn't use the specific value, no changes are made.
6492 if (!UserRemovedFromCSEMaps)
6495 // Now that we have modified User, add it back to the CSE maps. If it
6496 // already exists there, recursively merge the results together.
6497 AddModifiedNodeToCSEMaps(User);
6500 // If we just RAUW'd the root, take note.
6501 if (From == getRoot())
6506 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
6507 /// to record information about a use.
6514 /// operator< - Sort Memos by User.
6515 bool operator<(const UseMemo &L, const UseMemo &R) {
6516 return (intptr_t)L.User < (intptr_t)R.User;
6520 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
6521 /// uses of other values produced by From.getNode() alone. The same value
6522 /// may appear in both the From and To list. The Deleted vector is
6523 /// handled the same way as for ReplaceAllUsesWith.
6524 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
6527 // Handle the simple, trivial case efficiently.
6529 return ReplaceAllUsesOfValueWith(*From, *To);
6531 // Read up all the uses and make records of them. This helps
6532 // processing new uses that are introduced during the
6533 // replacement process.
6534 SmallVector<UseMemo, 4> Uses;
6535 for (unsigned i = 0; i != Num; ++i) {
6536 unsigned FromResNo = From[i].getResNo();
6537 SDNode *FromNode = From[i].getNode();
6538 for (SDNode::use_iterator UI = FromNode->use_begin(),
6539 E = FromNode->use_end(); UI != E; ++UI) {
6540 SDUse &Use = UI.getUse();
6541 if (Use.getResNo() == FromResNo) {
6542 UseMemo Memo = { *UI, i, &Use };
6543 Uses.push_back(Memo);
6548 // Sort the uses, so that all the uses from a given User are together.
6549 std::sort(Uses.begin(), Uses.end());
6551 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
6552 UseIndex != UseIndexEnd; ) {
6553 // We know that this user uses some value of From. If it is the right
6554 // value, update it.
6555 SDNode *User = Uses[UseIndex].User;
6557 // This node is about to morph, remove its old self from the CSE maps.
6558 RemoveNodeFromCSEMaps(User);
6560 // The Uses array is sorted, so all the uses for a given User
6561 // are next to each other in the list.
6562 // To help reduce the number of CSE recomputations, process all
6563 // the uses of this user that we can find this way.
6565 unsigned i = Uses[UseIndex].Index;
6566 SDUse &Use = *Uses[UseIndex].Use;
6570 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
6572 // Now that we have modified User, add it back to the CSE maps. If it
6573 // already exists there, recursively merge the results together.
6574 AddModifiedNodeToCSEMaps(User);
6578 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
6579 /// based on their topological order. It returns the maximum id and a vector
6580 /// of the SDNodes* in assigned order by reference.
6581 unsigned SelectionDAG::AssignTopologicalOrder() {
6583 unsigned DAGSize = 0;
6585 // SortedPos tracks the progress of the algorithm. Nodes before it are
6586 // sorted, nodes after it are unsorted. When the algorithm completes
6587 // it is at the end of the list.
6588 allnodes_iterator SortedPos = allnodes_begin();
6590 // Visit all the nodes. Move nodes with no operands to the front of
6591 // the list immediately. Annotate nodes that do have operands with their
6592 // operand count. Before we do this, the Node Id fields of the nodes
6593 // may contain arbitrary values. After, the Node Id fields for nodes
6594 // before SortedPos will contain the topological sort index, and the
6595 // Node Id fields for nodes At SortedPos and after will contain the
6596 // count of outstanding operands.
6597 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
6599 checkForCycles(N, this);
6600 unsigned Degree = N->getNumOperands();
6602 // A node with no uses, add it to the result array immediately.
6603 N->setNodeId(DAGSize++);
6604 allnodes_iterator Q(N);
6606 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
6607 assert(SortedPos != AllNodes.end() && "Overran node list");
6610 // Temporarily use the Node Id as scratch space for the degree count.
6611 N->setNodeId(Degree);
6615 // Visit all the nodes. As we iterate, move nodes into sorted order,
6616 // such that by the time the end is reached all nodes will be sorted.
6617 for (SDNode &Node : allnodes()) {
6619 checkForCycles(N, this);
6620 // N is in sorted position, so all its uses have one less operand
6621 // that needs to be sorted.
6622 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
6625 unsigned Degree = P->getNodeId();
6626 assert(Degree != 0 && "Invalid node degree");
6629 // All of P's operands are sorted, so P may sorted now.
6630 P->setNodeId(DAGSize++);
6632 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
6633 assert(SortedPos != AllNodes.end() && "Overran node list");
6636 // Update P's outstanding operand count.
6637 P->setNodeId(Degree);
6640 if (&Node == SortedPos) {
6642 allnodes_iterator I(N);
6644 dbgs() << "Overran sorted position:\n";
6645 S->dumprFull(this); dbgs() << "\n";
6646 dbgs() << "Checking if this is due to cycles\n";
6647 checkForCycles(this, true);
6649 llvm_unreachable(nullptr);
6653 assert(SortedPos == AllNodes.end() &&
6654 "Topological sort incomplete!");
6655 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
6656 "First node in topological sort is not the entry token!");
6657 assert(AllNodes.front().getNodeId() == 0 &&
6658 "First node in topological sort has non-zero id!");
6659 assert(AllNodes.front().getNumOperands() == 0 &&
6660 "First node in topological sort has operands!");
6661 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
6662 "Last node in topologic sort has unexpected id!");
6663 assert(AllNodes.back().use_empty() &&
6664 "Last node in topologic sort has users!");
6665 assert(DAGSize == allnodes_size() && "Node count mismatch!");
6669 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
6670 /// value is produced by SD.
6671 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
6673 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
6674 SD->setHasDebugValue(true);
6676 DbgInfo->add(DB, SD, isParameter);
6679 /// TransferDbgValues - Transfer SDDbgValues.
6680 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
6681 if (From == To || !From.getNode()->getHasDebugValue())
6683 SDNode *FromNode = From.getNode();
6684 SDNode *ToNode = To.getNode();
6685 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
6686 SmallVector<SDDbgValue *, 2> ClonedDVs;
6687 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
6689 SDDbgValue *Dbg = *I;
6690 if (Dbg->getKind() == SDDbgValue::SDNODE) {
6692 getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
6693 To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
6694 Dbg->getDebugLoc(), Dbg->getOrder());
6695 ClonedDVs.push_back(Clone);
6698 for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
6699 E = ClonedDVs.end(); I != E; ++I)
6700 AddDbgValue(*I, ToNode, false);
6703 //===----------------------------------------------------------------------===//
6705 //===----------------------------------------------------------------------===//
6707 bool llvm::isNullConstant(SDValue V) {
6708 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
6709 return Const != nullptr && Const->isNullValue();
6712 bool llvm::isNullFPConstant(SDValue V) {
6713 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
6714 return Const != nullptr && Const->isZero() && !Const->isNegative();
6717 bool llvm::isAllOnesConstant(SDValue V) {
6718 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
6719 return Const != nullptr && Const->isAllOnesValue();
6722 bool llvm::isOneConstant(SDValue V) {
6723 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
6724 return Const != nullptr && Const->isOne();
6727 HandleSDNode::~HandleSDNode() {
6731 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
6732 DebugLoc DL, const GlobalValue *GA,
6733 EVT VT, int64_t o, unsigned char TF)
6734 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
6738 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
6739 SDValue X, unsigned SrcAS,
6741 : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
6742 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
6744 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6745 EVT memvt, MachineMemOperand *mmo)
6746 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
6747 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
6748 MMO->isNonTemporal(), MMO->isInvariant());
6749 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6750 assert(isNonTemporal() == MMO->isNonTemporal() &&
6751 "Non-temporal encoding error!");
6752 // We check here that the size of the memory operand fits within the size of
6753 // the MMO. This is because the MMO might indicate only a possible address
6754 // range instead of specifying the affected memory addresses precisely.
6755 assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
6758 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6759 ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
6760 : SDNode(Opc, Order, dl, VTs, Ops),
6761 MemoryVT(memvt), MMO(mmo) {
6762 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
6763 MMO->isNonTemporal(), MMO->isInvariant());
6764 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6765 assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
6768 /// Profile - Gather unique data for the node.
6770 void SDNode::Profile(FoldingSetNodeID &ID) const {
6771 AddNodeIDNode(ID, this);
6776 std::vector<EVT> VTs;
6779 VTs.reserve(MVT::LAST_VALUETYPE);
6780 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
6781 VTs.push_back(MVT((MVT::SimpleValueType)i));
6786 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
6787 static ManagedStatic<EVTArray> SimpleVTArray;
6788 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
6790 /// getValueTypeList - Return a pointer to the specified value type.
6792 const EVT *SDNode::getValueTypeList(EVT VT) {
6793 if (VT.isExtended()) {
6794 sys::SmartScopedLock<true> Lock(*VTMutex);
6795 return &(*EVTs->insert(VT).first);
6797 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
6798 "Value type out of range!");
6799 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
6803 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
6804 /// indicated value. This method ignores uses of other values defined by this
6806 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
6807 assert(Value < getNumValues() && "Bad value!");
6809 // TODO: Only iterate over uses of a given value of the node
6810 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
6811 if (UI.getUse().getResNo() == Value) {
6818 // Found exactly the right number of uses?
6823 /// hasAnyUseOfValue - Return true if there are any use of the indicated
6824 /// value. This method ignores uses of other values defined by this operation.
6825 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
6826 assert(Value < getNumValues() && "Bad value!");
6828 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
6829 if (UI.getUse().getResNo() == Value)
6836 /// isOnlyUserOf - Return true if this node is the only use of N.
6838 bool SDNode::isOnlyUserOf(const SDNode *N) const {
6840 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
6851 /// isOperand - Return true if this node is an operand of N.
6853 bool SDValue::isOperandOf(const SDNode *N) const {
6854 for (const SDValue &Op : N->op_values())
6860 bool SDNode::isOperandOf(const SDNode *N) const {
6861 for (const SDValue &Op : N->op_values())
6862 if (this == Op.getNode())
6867 /// reachesChainWithoutSideEffects - Return true if this operand (which must
6868 /// be a chain) reaches the specified operand without crossing any
6869 /// side-effecting instructions on any chain path. In practice, this looks
6870 /// through token factors and non-volatile loads. In order to remain efficient,
6871 /// this only looks a couple of nodes in, it does not do an exhaustive search.
6872 bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
6873 unsigned Depth) const {
6874 if (*this == Dest) return true;
6876 // Don't search too deeply, we just want to be able to see through
6877 // TokenFactor's etc.
6878 if (Depth == 0) return false;
6880 // If this is a token factor, all inputs to the TF happen in parallel. If any
6881 // of the operands of the TF does not reach dest, then we cannot do the xform.
6882 if (getOpcode() == ISD::TokenFactor) {
6883 for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
6884 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1))
6889 // Loads don't have side effects, look through them.
6890 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
6891 if (!Ld->isVolatile())
6892 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
6897 /// hasPredecessor - Return true if N is a predecessor of this node.
6898 /// N is either an operand of this node, or can be reached by recursively
6899 /// traversing up the operands.
6900 /// NOTE: This is an expensive method. Use it carefully.
6901 bool SDNode::hasPredecessor(const SDNode *N) const {
6902 SmallPtrSet<const SDNode *, 32> Visited;
6903 SmallVector<const SDNode *, 16> Worklist;
6904 return hasPredecessorHelper(N, Visited, Worklist);
6908 SDNode::hasPredecessorHelper(const SDNode *N,
6909 SmallPtrSetImpl<const SDNode *> &Visited,
6910 SmallVectorImpl<const SDNode *> &Worklist) const {
6911 if (Visited.empty()) {
6912 Worklist.push_back(this);
6914 // Take a look in the visited set. If we've already encountered this node
6915 // we needn't search further.
6916 if (Visited.count(N))
6920 // Haven't visited N yet. Continue the search.
6921 while (!Worklist.empty()) {
6922 const SDNode *M = Worklist.pop_back_val();
6923 for (const SDValue &OpV : M->op_values()) {
6924 SDNode *Op = OpV.getNode();
6925 if (Visited.insert(Op).second)
6926 Worklist.push_back(Op);
6935 uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
6936 assert(Num < NumOperands && "Invalid child # of SDNode!");
6937 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
6940 const SDNodeFlags *SDNode::getFlags() const {
6941 if (auto *FlagsNode = dyn_cast<BinaryWithFlagsSDNode>(this))
6942 return &FlagsNode->Flags;
6946 void SDNode::intersectFlagsWith(const SDNodeFlags *Flags) {
6947 if (auto *FlagsNode = dyn_cast<BinaryWithFlagsSDNode>(this))
6948 FlagsNode->Flags.intersectWith(Flags);
6951 SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
6952 assert(N->getNumValues() == 1 &&
6953 "Can't unroll a vector with multiple results!");
6955 EVT VT = N->getValueType(0);
6956 unsigned NE = VT.getVectorNumElements();
6957 EVT EltVT = VT.getVectorElementType();
6960 SmallVector<SDValue, 8> Scalars;
6961 SmallVector<SDValue, 4> Operands(N->getNumOperands());
6963 // If ResNE is 0, fully unroll the vector op.
6966 else if (NE > ResNE)
6970 for (i= 0; i != NE; ++i) {
6971 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
6972 SDValue Operand = N->getOperand(j);
6973 EVT OperandVT = Operand.getValueType();
6974 if (OperandVT.isVector()) {
6975 // A vector operand; extract a single element.
6976 EVT OperandEltVT = OperandVT.getVectorElementType();
6978 getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
6979 getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
6981 // A scalar operand; just use it as is.
6982 Operands[j] = Operand;
6986 switch (N->getOpcode()) {
6988 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
6993 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
7000 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
7001 getShiftAmountOperand(Operands[0].getValueType(),
7004 case ISD::SIGN_EXTEND_INREG:
7005 case ISD::FP_ROUND_INREG: {
7006 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
7007 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
7009 getValueType(ExtVT)));
7014 for (; i < ResNE; ++i)
7015 Scalars.push_back(getUNDEF(EltVT));
7017 return getNode(ISD::BUILD_VECTOR, dl,
7018 EVT::getVectorVT(*getContext(), EltVT, ResNE), Scalars);
7022 /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a
7023 /// location that is 'Dist' units away from the location that the 'Base' load
7024 /// is loading from.
7025 bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base,
7026 unsigned Bytes, int Dist) const {
7027 if (LD->getChain() != Base->getChain())
7029 EVT VT = LD->getValueType(0);
7030 if (VT.getSizeInBits() / 8 != Bytes)
7033 SDValue Loc = LD->getOperand(1);
7034 SDValue BaseLoc = Base->getOperand(1);
7035 if (Loc.getOpcode() == ISD::FrameIndex) {
7036 if (BaseLoc.getOpcode() != ISD::FrameIndex)
7038 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo();
7039 int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
7040 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
7041 int FS = MFI->getObjectSize(FI);
7042 int BFS = MFI->getObjectSize(BFI);
7043 if (FS != BFS || FS != (int)Bytes) return false;
7044 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
7048 if (isBaseWithConstantOffset(Loc)) {
7049 int64_t LocOffset = cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue();
7050 if (Loc.getOperand(0) == BaseLoc) {
7051 // If the base location is a simple address with no offset itself, then
7052 // the second load's first add operand should be the base address.
7053 if (LocOffset == Dist * (int)Bytes)
7055 } else if (isBaseWithConstantOffset(BaseLoc)) {
7056 // The base location itself has an offset, so subtract that value from the
7057 // second load's offset before comparing to distance * size.
7059 cast<ConstantSDNode>(BaseLoc.getOperand(1))->getSExtValue();
7060 if (Loc.getOperand(0) == BaseLoc.getOperand(0)) {
7061 if ((LocOffset - BOffset) == Dist * (int)Bytes)
7066 const GlobalValue *GV1 = nullptr;
7067 const GlobalValue *GV2 = nullptr;
7068 int64_t Offset1 = 0;
7069 int64_t Offset2 = 0;
7070 bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1);
7071 bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
7072 if (isGA1 && isGA2 && GV1 == GV2)
7073 return Offset1 == (Offset2 + Dist*Bytes);
7078 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
7079 /// it cannot be inferred.
7080 unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
7081 // If this is a GlobalAddress + cst, return the alignment.
7082 const GlobalValue *GV;
7083 int64_t GVOffset = 0;
7084 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
7085 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
7086 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
7087 llvm::computeKnownBits(const_cast<GlobalValue *>(GV), KnownZero, KnownOne,
7089 unsigned AlignBits = KnownZero.countTrailingOnes();
7090 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
7092 return MinAlign(Align, GVOffset);
7095 // If this is a direct reference to a stack slot, use information about the
7096 // stack slot's alignment.
7097 int FrameIdx = 1 << 31;
7098 int64_t FrameOffset = 0;
7099 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
7100 FrameIdx = FI->getIndex();
7101 } else if (isBaseWithConstantOffset(Ptr) &&
7102 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
7104 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
7105 FrameOffset = Ptr.getConstantOperandVal(1);
7108 if (FrameIdx != (1 << 31)) {
7109 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo();
7110 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
7118 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
7119 /// which is split (or expanded) into two not necessarily identical pieces.
7120 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
7121 // Currently all types are split in half.
7123 if (!VT.isVector()) {
7124 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
7126 unsigned NumElements = VT.getVectorNumElements();
7127 assert(!(NumElements & 1) && "Splitting vector, but not in half!");
7128 LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
7131 return std::make_pair(LoVT, HiVT);
7134 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
7136 std::pair<SDValue, SDValue>
7137 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
7139 assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
7140 N.getValueType().getVectorNumElements() &&
7141 "More vector elements requested than available!");
7143 Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
7144 getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
7145 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
7146 getConstant(LoVT.getVectorNumElements(), DL,
7147 TLI->getVectorIdxTy(getDataLayout())));
7148 return std::make_pair(Lo, Hi);
7151 void SelectionDAG::ExtractVectorElements(SDValue Op,
7152 SmallVectorImpl<SDValue> &Args,
7153 unsigned Start, unsigned Count) {
7154 EVT VT = Op.getValueType();
7156 Count = VT.getVectorNumElements();
7158 EVT EltVT = VT.getVectorElementType();
7159 EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
7161 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
7162 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
7163 Op, getConstant(i, SL, IdxTy)));
7167 // getAddressSpace - Return the address space this GlobalAddress belongs to.
7168 unsigned GlobalAddressSDNode::getAddressSpace() const {
7169 return getGlobal()->getType()->getAddressSpace();
7173 Type *ConstantPoolSDNode::getType() const {
7174 if (isMachineConstantPoolEntry())
7175 return Val.MachineCPVal->getType();
7176 return Val.ConstVal->getType();
7179 bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue,
7181 unsigned &SplatBitSize,
7183 unsigned MinSplatBits,
7184 bool isBigEndian) const {
7185 EVT VT = getValueType(0);
7186 assert(VT.isVector() && "Expected a vector type");
7187 unsigned sz = VT.getSizeInBits();
7188 if (MinSplatBits > sz)
7191 SplatValue = APInt(sz, 0);
7192 SplatUndef = APInt(sz, 0);
7194 // Get the bits. Bits with undefined values (when the corresponding element
7195 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
7196 // in SplatValue. If any of the values are not constant, give up and return
7198 unsigned int nOps = getNumOperands();
7199 assert(nOps > 0 && "isConstantSplat has 0-size build vector");
7200 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits();
7202 for (unsigned j = 0; j < nOps; ++j) {
7203 unsigned i = isBigEndian ? nOps-1-j : j;
7204 SDValue OpVal = getOperand(i);
7205 unsigned BitPos = j * EltBitSize;
7207 if (OpVal.getOpcode() == ISD::UNDEF)
7208 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize);
7209 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal))
7210 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize).
7211 zextOrTrunc(sz) << BitPos;
7212 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal))
7213 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos;
7218 // The build_vector is all constants or undefs. Find the smallest element
7219 // size that splats the vector.
7221 HasAnyUndefs = (SplatUndef != 0);
7224 unsigned HalfSize = sz / 2;
7225 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
7226 APInt LowValue = SplatValue.trunc(HalfSize);
7227 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
7228 APInt LowUndef = SplatUndef.trunc(HalfSize);
7230 // If the two halves do not match (ignoring undef bits), stop here.
7231 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
7232 MinSplatBits > HalfSize)
7235 SplatValue = HighValue | LowValue;
7236 SplatUndef = HighUndef & LowUndef;
7245 SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
7246 if (UndefElements) {
7247 UndefElements->clear();
7248 UndefElements->resize(getNumOperands());
7251 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
7252 SDValue Op = getOperand(i);
7253 if (Op.getOpcode() == ISD::UNDEF) {
7255 (*UndefElements)[i] = true;
7256 } else if (!Splatted) {
7258 } else if (Splatted != Op) {
7264 assert(getOperand(0).getOpcode() == ISD::UNDEF &&
7265 "Can only have a splat without a constant for all undefs.");
7266 return getOperand(0);
7273 BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
7274 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
7278 BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
7279 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
7283 BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
7284 uint32_t BitWidth) const {
7285 if (ConstantFPSDNode *CN =
7286 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
7288 APSInt IntVal(BitWidth);
7289 APFloat APF = CN->getValueAPF();
7290 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
7295 return IntVal.exactLogBase2();
7300 bool BuildVectorSDNode::isConstant() const {
7301 for (const SDValue &Op : op_values()) {
7302 unsigned Opc = Op.getOpcode();
7303 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
7309 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
7310 // Find the first non-undef value in the shuffle mask.
7312 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
7315 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
7317 // Make sure all remaining elements are either undef or the same as the first
7319 for (int Idx = Mask[i]; i != e; ++i)
7320 if (Mask[i] >= 0 && Mask[i] != Idx)
7326 static void checkForCyclesHelper(const SDNode *N,
7327 SmallPtrSetImpl<const SDNode*> &Visited,
7328 SmallPtrSetImpl<const SDNode*> &Checked,
7329 const llvm::SelectionDAG *DAG) {
7330 // If this node has already been checked, don't check it again.
7331 if (Checked.count(N))
7334 // If a node has already been visited on this depth-first walk, reject it as
7336 if (!Visited.insert(N).second) {
7337 errs() << "Detected cycle in SelectionDAG\n";
7338 dbgs() << "Offending node:\n";
7339 N->dumprFull(DAG); dbgs() << "\n";
7343 for (const SDValue &Op : N->op_values())
7344 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
7351 void llvm::checkForCycles(const llvm::SDNode *N,
7352 const llvm::SelectionDAG *DAG,
7360 assert(N && "Checking nonexistent SDNode");
7361 SmallPtrSet<const SDNode*, 32> visited;
7362 SmallPtrSet<const SDNode*, 32> checked;
7363 checkForCyclesHelper(N, visited, checked, DAG);
7368 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
7369 checkForCycles(DAG->getRoot().getNode(), DAG, force);