X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FCodeGen%2FSelectionDAG%2FSelectionDAG.cpp;h=abd72727e9c906b339be9e1fae9d16b982dcf217;hb=317adcc9c6b29fb491d615996ba70b5c415a9eb2;hp=8a2962f0c33af5f0b41a20f5c9fc4d83ee783936;hpb=4c71dfe356716e6bc1993ef5efdced08b68fe612;p=oota-llvm.git

diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8a2962f0c33..abd72727e9c 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -10,13 +10,14 @@
 // This implements the SelectionDAG class.
 //
 //===----------------------------------------------------------------------===//
-
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/Constants.h"
+#include "llvm/GlobalAlias.h"
 #include "llvm/GlobalVariable.h"
 #include "llvm/Intrinsics.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Assembly/Writer.h"
+#include "llvm/CallingConv.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -39,11 +40,22 @@ using namespace llvm;
 
 /// makeVTList - Return an instance of the SDVTList struct initialized with the
 /// specified members.
-static SDVTList makeVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
+static SDVTList makeVTList(const MVT *VTs, unsigned NumVTs) {
   SDVTList Res = {VTs, NumVTs};
   return Res;
 }
 
+static const fltSemantics *MVTToAPFloatSemantics(MVT VT) {
+  switch (VT.getSimpleVT()) {
+  default: assert(0 && "Unknown FP format");
+  case MVT::f32:     return &APFloat::IEEEsingle;
+  case MVT::f64:     return &APFloat::IEEEdouble;
+  case MVT::f80:     return &APFloat::x87DoubleExtended;
+  case MVT::f128:    return &APFloat::IEEEquad;
+  case MVT::ppcf128: return &APFloat::PPCDoubleDouble;
+  }
+}
+
 SelectionDAG::DAGUpdateListener::~DAGUpdateListener() {}
 
 //===----------------------------------------------------------------------===//
@@ -58,30 +70,19 @@ bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
   return Value.bitwiseIsEqual(V);
 }
 
-bool ConstantFPSDNode::isValueValidForType(MVT::ValueType VT, 
+bool ConstantFPSDNode::isValueValidForType(MVT VT,
                                            const APFloat& Val) {
+  assert(VT.isFloatingPoint() && "Can only convert between FP types");
+  
+  // PPC long double cannot be converted to any other type.
+  if (VT == MVT::ppcf128 ||
+      &Val.getSemantics() == &APFloat::PPCDoubleDouble)
+    return false;
+  
   // convert modifies in place, so make a copy.
   APFloat Val2 = APFloat(Val);
-  switch (VT) {
-  default:
-    return false;         // These can't be represented as floating point!
-
-  // FIXME rounding mode needs to be more flexible
-  case MVT::f32:
-    return &Val2.getSemantics() == &APFloat::IEEEsingle ||
-           Val2.convert(APFloat::IEEEsingle, APFloat::rmNearestTiesToEven) == 
-              APFloat::opOK;
-  case MVT::f64:
-    return &Val2.getSemantics() == &APFloat::IEEEsingle || 
-           &Val2.getSemantics() == &APFloat::IEEEdouble ||
-           Val2.convert(APFloat::IEEEdouble, APFloat::rmNearestTiesToEven) == 
-             APFloat::opOK;
-  // TODO: Figure out how to test if we can use a shorter type instead!
-  case MVT::f80:
-  case MVT::f128:
-  case MVT::ppcf128:
-    return true;
-  }
+  return Val2.convert(*MVTToAPFloatSemantics(VT),
+                      APFloat::rmNearestTiesToEven) == APFloat::opOK;
 }
 
 //===----------------------------------------------------------------------===//
@@ -113,17 +114,9 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) {
     if (!cast<ConstantSDNode>(NotZero)->isAllOnesValue())
       return false;
   } else if (isa<ConstantFPSDNode>(NotZero)) {
-    MVT::ValueType VT = NotZero.getValueType();
-    if (VT== MVT::f64) {
-      if (((cast<ConstantFPSDNode>(NotZero)->getValueAPF().
-                  convertToAPInt().getZExtValue())) != (uint64_t)-1)
-        return false;
-    } else {
-      if ((uint32_t)cast<ConstantFPSDNode>(NotZero)->
-                      getValueAPF().convertToAPInt().getZExtValue() != 
-          (uint32_t)-1)
-        return false;
-    }
+    if (!cast<ConstantFPSDNode>(NotZero)->getValueAPF().
+                convertToAPInt().isAllOnesValue())
+      return false;
   } else
     return false;
   
@@ -300,6 +293,7 @@ ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
     switch (Result) {
     default: break;
     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
+    case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
@@ -325,14 +319,14 @@ static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
 
 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
 /// solely with their pointer.
-void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
+static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
   ID.AddPointer(VTList.VTs);  
 }
 
 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
 ///
 static void AddNodeIDOperands(FoldingSetNodeID &ID,
-                              const SDOperand *Ops, unsigned NumOps) {
+                              SDOperandPtr Ops, unsigned NumOps) {
   for (; NumOps; --NumOps, ++Ops) {
     ID.AddPointer(Ops->Val);
     ID.AddInteger(Ops->ResNo);
@@ -341,12 +335,13 @@ static void AddNodeIDOperands(FoldingSetNodeID &ID,
 
 static void AddNodeIDNode(FoldingSetNodeID &ID,
                           unsigned short OpC, SDVTList VTList, 
-                          const SDOperand *OpList, unsigned N) {
+                          SDOperandPtr OpList, unsigned N) {
   AddNodeIDOpcode(ID, OpC);
   AddNodeIDValueTypes(ID, VTList);
   AddNodeIDOperands(ID, OpList, N);
 }
 
+
 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
 /// data.
 static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
@@ -359,6 +354,9 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
   // Handle SDNode leafs with special info.
   switch (N->getOpcode()) {
   default: break;  // Normal nodes don't need extra info.
+  case ISD::ARG_FLAGS:
+    ID.AddInteger(cast<ARG_FLAGSSDNode>(N)->getArgFlags().getRawBits());
+    break;
   case ISD::TargetConstant:
   case ISD::Constant:
     ID.Add(cast<ConstantSDNode>(N)->getAPIntValue());
@@ -387,7 +385,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
     break;
   case ISD::MEMOPERAND: {
-    const MemOperand &MO = cast<MemOperandSDNode>(N)->MO;
+    const MachineMemOperand &MO = cast<MemOperandSDNode>(N)->MO;
     ID.AddPointer(MO.getValue());
     ID.AddInteger(MO.getFlags());
     ID.AddInteger(MO.getOffset());
@@ -418,7 +416,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
     LoadSDNode *LD = cast<LoadSDNode>(N);
     ID.AddInteger(LD->getAddressingMode());
     ID.AddInteger(LD->getExtensionType());
-    ID.AddInteger((unsigned int)(LD->getMemoryVT()));
+    ID.AddInteger(LD->getMemoryVT().getRawBits());
     ID.AddInteger(LD->getAlignment());
     ID.AddInteger(LD->isVolatile());
     break;
@@ -427,7 +425,7 @@ static void AddNodeIDNode(FoldingSetNodeID &ID, SDNode *N) {
     StoreSDNode *ST = cast<StoreSDNode>(N);
     ID.AddInteger(ST->getAddressingMode());
     ID.AddInteger(ST->isTruncatingStore());
-    ID.AddInteger((unsigned int)(ST->getMemoryVT()));
+    ID.AddInteger(ST->getMemoryVT().getRawBits());
     ID.AddInteger(ST->getAlignment());
     ID.AddInteger(ST->isVolatile());
     break;
@@ -465,15 +463,16 @@ void SelectionDAG::RemoveDeadNodes() {
     // Next, brutally remove the operand list.  This is safe to do, as there are
     // no cycles in the graph.
     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
-      SDNode *Operand = I->Val;
-      Operand->removeUser(N);
+      SDNode *Operand = I->getVal();
+      Operand->removeUser(std::distance(N->op_begin(), I), N);
       
       // Now that we removed this operand, see if there are no uses of it left.
       if (Operand->use_empty())
         DeadNodes.push_back(Operand);
     }
-    if (N->OperandsNeedDelete)
+    if (N->OperandsNeedDelete) {
       delete[] N->OperandList;
+    }
     N->OperandList = 0;
     N->NumOperands = 0;
     
@@ -496,7 +495,7 @@ void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){
     DeadNodes.pop_back();
     
     if (UpdateListener)
-      UpdateListener->NodeDeleted(N);
+      UpdateListener->NodeDeleted(N, 0);
     
     // Take the node out of the appropriate CSE map.
     RemoveNodeFromCSEMaps(N);
@@ -504,15 +503,16 @@ void SelectionDAG::RemoveDeadNode(SDNode *N, DAGUpdateListener *UpdateListener){
     // Next, brutally remove the operand list.  This is safe to do, as there are
     // no cycles in the graph.
     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
-      SDNode *Operand = I->Val;
-      Operand->removeUser(N);
+      SDNode *Operand = I->getVal();
+      Operand->removeUser(std::distance(N->op_begin(), I), N);
       
       // Now that we removed this operand, see if there are no uses of it left.
       if (Operand->use_empty())
         DeadNodes.push_back(Operand);
     }
-    if (N->OperandsNeedDelete)
+    if (N->OperandsNeedDelete) {
       delete[] N->OperandList;
+    }
     N->OperandList = 0;
     N->NumOperands = 0;
     
@@ -539,9 +539,10 @@ void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
     
   // Drop all of the operands and decrement used nodes use counts.
   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
-    I->Val->removeUser(N);
-  if (N->OperandsNeedDelete)
+    I->getVal()->removeUser(std::distance(N->op_begin(), I), N);
+  if (N->OperandsNeedDelete) {
     delete[] N->OperandList;
+  }
   N->OperandList = 0;
   N->NumOperands = 0;
   
@@ -573,12 +574,12 @@ void SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
       TargetExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
     break;
   case ISD::VALUETYPE: {
-    MVT::ValueType VT = cast<VTSDNode>(N)->getVT();
-    if (MVT::isExtendedVT(VT)) {
+    MVT VT = cast<VTSDNode>(N)->getVT();
+    if (VT.isExtended()) {
       Erased = ExtendedValueTypeNodes.erase(VT);
     } else {
-      Erased = ValueTypeNodes[VT] != 0;
-      ValueTypeNodes[VT] = 0;
+      Erased = ValueTypeNodes[VT.getSimpleVT()] != 0;
+      ValueTypeNodes[VT.getSimpleVT()] = 0;
     }
     break;
   }
@@ -667,7 +668,7 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
 /// return null, otherwise return a pointer to the slot it would take.  If a
 /// node already exists with these operands, the slot will be non-null.
 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 
-                                           const SDOperand *Ops,unsigned NumOps,
+                                           SDOperandPtr Ops,unsigned NumOps,
                                            void *&InsertPos) {
   if (N->getOpcode() == ISD::HANDLENODE || N->getValueType(0) == MVT::Flag)
     return 0;    // Never add these nodes.
@@ -683,13 +684,13 @@ SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
   if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
     ID.AddInteger(LD->getAddressingMode());
     ID.AddInteger(LD->getExtensionType());
-    ID.AddInteger((unsigned int)(LD->getMemoryVT()));
+    ID.AddInteger(LD->getMemoryVT().getRawBits());
     ID.AddInteger(LD->getAlignment());
     ID.AddInteger(LD->isVolatile());
   } else if (const StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
     ID.AddInteger(ST->getAddressingMode());
     ID.AddInteger(ST->isTruncatingStore());
-    ID.AddInteger((unsigned int)(ST->getMemoryVT()));
+    ID.AddInteger(ST->getMemoryVT().getRawBits());
     ID.AddInteger(ST->getAlignment());
     ID.AddInteger(ST->isVolatile());
   }
@@ -702,17 +703,19 @@ SelectionDAG::~SelectionDAG() {
   while (!AllNodes.empty()) {
     SDNode *N = AllNodes.begin();
     N->SetNextInBucket(0);
-    if (N->OperandsNeedDelete)
+    if (N->OperandsNeedDelete) {
       delete [] N->OperandList;
+    }
     N->OperandList = 0;
     N->NumOperands = 0;
     AllNodes.pop_front();
   }
 }
 
-SDOperand SelectionDAG::getZeroExtendInReg(SDOperand Op, MVT::ValueType VT) {
+SDOperand SelectionDAG::getZeroExtendInReg(SDOperand Op, MVT VT) {
   if (Op.getValueType() == VT) return Op;
-  int64_t Imm = ~0ULL >> (64-MVT::getSizeInBits(VT));
+  APInt Imm = APInt::getLowBitsSet(Op.getValueSizeInBits(),
+                                   VT.getSizeInBits());
   return getNode(ISD::AND, Op.getValueType(), Op,
                  getConstant(Imm, Op.getValueType()));
 }
@@ -726,30 +729,30 @@ SDOperand SelectionDAG::getString(const std::string &Val) {
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getConstant(uint64_t Val, MVT::ValueType VT, bool isT) {
-  MVT::ValueType EltVT =
-    MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT;
+SDOperand SelectionDAG::getConstant(uint64_t Val, MVT VT, bool isT) {
+  MVT EltVT =
+    VT.isVector() ? VT.getVectorElementType() : VT;
 
-  return getConstant(APInt(MVT::getSizeInBits(EltVT), Val), VT, isT);
+  return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT);
 }
 
-SDOperand SelectionDAG::getConstant(const APInt &Val, MVT::ValueType VT, bool isT) {
-  assert(MVT::isInteger(VT) && "Cannot create FP integer constant!");
+SDOperand SelectionDAG::getConstant(const APInt &Val, MVT VT, bool isT) {
+  assert(VT.isInteger() && "Cannot create FP integer constant!");
 
-  MVT::ValueType EltVT =
-    MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT;
+  MVT EltVT =
+    VT.isVector() ? VT.getVectorElementType() : VT;
   
-  assert(Val.getBitWidth() == MVT::getSizeInBits(EltVT) &&
+  assert(Val.getBitWidth() == EltVT.getSizeInBits() &&
          "APInt size does not match type size!");
 
   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(EltVT), (SDOperand*)0, 0);
   ID.Add(Val);
   void *IP = 0;
   SDNode *N = NULL;
   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
-    if (!MVT::isVector(VT))
+    if (!VT.isVector())
       return SDOperand(N, 0);
   if (!N) {
     N = new ConstantSDNode(isT, Val, EltVT);
@@ -758,9 +761,9 @@ SDOperand SelectionDAG::getConstant(const APInt &Val, MVT::ValueType VT, bool is
   }
 
   SDOperand Result(N, 0);
-  if (MVT::isVector(VT)) {
+  if (VT.isVector()) {
     SmallVector<SDOperand, 8> Ops;
-    Ops.assign(MVT::getVectorNumElements(VT), Result);
+    Ops.assign(VT.getVectorNumElements(), Result);
     Result = getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
   }
   return Result;
@@ -771,24 +774,23 @@ SDOperand SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
 }
 
 
-SDOperand SelectionDAG::getConstantFP(const APFloat& V, MVT::ValueType VT,
-                                      bool isTarget) {
-  assert(MVT::isFloatingPoint(VT) && "Cannot create integer FP constant!");
+SDOperand SelectionDAG::getConstantFP(const APFloat& V, MVT VT, bool isTarget) {
+  assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
                                 
-  MVT::ValueType EltVT =
-    MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT;
+  MVT EltVT =
+    VT.isVector() ? VT.getVectorElementType() : VT;
 
   // Do the map lookup using the actual bit pattern for the floating point
   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
   // we don't have issues with SNANs.
   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(EltVT), (SDOperand*)0, 0);
   ID.Add(V);
   void *IP = 0;
   SDNode *N = NULL;
   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
-    if (!MVT::isVector(VT))
+    if (!VT.isVector())
       return SDOperand(N, 0);
   if (!N) {
     N = new ConstantFPSDNode(isTarget, V, EltVT);
@@ -797,18 +799,17 @@ SDOperand SelectionDAG::getConstantFP(const APFloat& V, MVT::ValueType VT,
   }
 
   SDOperand Result(N, 0);
-  if (MVT::isVector(VT)) {
+  if (VT.isVector()) {
     SmallVector<SDOperand, 8> Ops;
-    Ops.assign(MVT::getVectorNumElements(VT), Result);
+    Ops.assign(VT.getVectorNumElements(), Result);
     Result = getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
   }
   return Result;
 }
 
-SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT,
-                                      bool isTarget) {
-  MVT::ValueType EltVT =
-    MVT::isVector(VT) ? MVT::getVectorElementType(VT) : VT;
+SDOperand SelectionDAG::getConstantFP(double Val, MVT VT, bool isTarget) {
+  MVT EltVT =
+    VT.isVector() ? VT.getVectorElementType() : VT;
   if (EltVT==MVT::f32)
     return getConstantFP(APFloat((float)Val), VT, isTarget);
   else
@@ -816,16 +817,24 @@ SDOperand SelectionDAG::getConstantFP(double Val, MVT::ValueType VT,
 }
 
 SDOperand SelectionDAG::getGlobalAddress(const GlobalValue *GV,
-                                         MVT::ValueType VT, int Offset,
+                                         MVT VT, int Offset,
                                          bool isTargetGA) {
-  const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
   unsigned Opc;
+
+  const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
+  if (!GVar) {
+    // If GV is an alias then use the aliasee for determining thread-localness.
+    if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
+      GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal());
+  }
+
   if (GVar && GVar->isThreadLocal())
     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
   else
     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
+
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(VT), (SDOperand*)0, 0);
   ID.AddPointer(GV);
   ID.AddInteger(Offset);
   void *IP = 0;
@@ -837,11 +846,10 @@ SDOperand SelectionDAG::getGlobalAddress(const GlobalValue *GV,
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getFrameIndex(int FI, MVT::ValueType VT,
-                                      bool isTarget) {
+SDOperand SelectionDAG::getFrameIndex(int FI, MVT VT, bool isTarget) {
   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(VT), (SDOperand*)0, 0);
   ID.AddInteger(FI);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
@@ -852,10 +860,10 @@ SDOperand SelectionDAG::getFrameIndex(int FI, MVT::ValueType VT,
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getJumpTable(int JTI, MVT::ValueType VT, bool isTarget){
+SDOperand SelectionDAG::getJumpTable(int JTI, MVT VT, bool isTarget){
   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(VT), (SDOperand*)0, 0);
   ID.AddInteger(JTI);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
@@ -866,12 +874,12 @@ SDOperand SelectionDAG::getJumpTable(int JTI, MVT::ValueType VT, bool isTarget){
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getConstantPool(Constant *C, MVT::ValueType VT,
+SDOperand SelectionDAG::getConstantPool(Constant *C, MVT VT,
                                         unsigned Alignment, int Offset,
                                         bool isTarget) {
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(VT), (SDOperand*)0, 0);
   ID.AddInteger(Alignment);
   ID.AddInteger(Offset);
   ID.AddPointer(C);
@@ -885,13 +893,12 @@ SDOperand SelectionDAG::getConstantPool(Constant *C, MVT::ValueType VT,
 }
 
 
-SDOperand SelectionDAG::getConstantPool(MachineConstantPoolValue *C,
-                                        MVT::ValueType VT,
+SDOperand SelectionDAG::getConstantPool(MachineConstantPoolValue *C, MVT VT,
                                         unsigned Alignment, int Offset,
                                         bool isTarget) {
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opc, getVTList(VT), (SDOperand*)0, 0);
   ID.AddInteger(Alignment);
   ID.AddInteger(Offset);
   C->AddSelectionDAGCSEId(ID);
@@ -907,7 +914,7 @@ SDOperand SelectionDAG::getConstantPool(MachineConstantPoolValue *C,
 
 SDOperand SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0);
+  AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), (SDOperand*)0, 0);
   ID.AddPointer(MBB);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
@@ -918,12 +925,25 @@ SDOperand SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getValueType(MVT::ValueType VT) {
-  if (!MVT::isExtendedVT(VT) && (unsigned)VT >= ValueTypeNodes.size())
-    ValueTypeNodes.resize(VT+1);
+SDOperand SelectionDAG::getArgFlags(ISD::ArgFlagsTy Flags) {
+  FoldingSetNodeID ID;
+  AddNodeIDNode(ID, ISD::ARG_FLAGS, getVTList(MVT::Other), (SDOperand*)0, 0);
+  ID.AddInteger(Flags.getRawBits());
+  void *IP = 0;
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    return SDOperand(E, 0);
+  SDNode *N = new ARG_FLAGSSDNode(Flags);
+  CSEMap.InsertNode(N, IP);
+  AllNodes.push_back(N);
+  return SDOperand(N, 0);
+}
+
+SDOperand SelectionDAG::getValueType(MVT VT) {
+  if (VT.isSimple() && (unsigned)VT.getSimpleVT() >= ValueTypeNodes.size())
+    ValueTypeNodes.resize(VT.getSimpleVT()+1);
 
-  SDNode *&N = MVT::isExtendedVT(VT) ?
-    ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT];
+  SDNode *&N = VT.isExtended() ?
+    ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT()];
 
   if (N) return SDOperand(N, 0);
   N = new VTSDNode(VT);
@@ -931,7 +951,7 @@ SDOperand SelectionDAG::getValueType(MVT::ValueType VT) {
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getExternalSymbol(const char *Sym, MVT::ValueType VT) {
+SDOperand SelectionDAG::getExternalSymbol(const char *Sym, MVT VT) {
   SDNode *&N = ExternalSymbols[Sym];
   if (N) return SDOperand(N, 0);
   N = new ExternalSymbolSDNode(false, Sym, VT);
@@ -939,8 +959,7 @@ SDOperand SelectionDAG::getExternalSymbol(const char *Sym, MVT::ValueType VT) {
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getTargetExternalSymbol(const char *Sym,
-                                                MVT::ValueType VT) {
+SDOperand SelectionDAG::getTargetExternalSymbol(const char *Sym, MVT VT) {
   SDNode *&N = TargetExternalSymbols[Sym];
   if (N) return SDOperand(N, 0);
   N = new ExternalSymbolSDNode(true, Sym, VT);
@@ -951,7 +970,7 @@ SDOperand SelectionDAG::getTargetExternalSymbol(const char *Sym,
 SDOperand SelectionDAG::getCondCode(ISD::CondCode Cond) {
   if ((unsigned)Cond >= CondCodeNodes.size())
     CondCodeNodes.resize(Cond+1);
-  
+
   if (CondCodeNodes[Cond] == 0) {
     CondCodeNodes[Cond] = new CondCodeSDNode(Cond);
     AllNodes.push_back(CondCodeNodes[Cond]);
@@ -959,9 +978,9 @@ SDOperand SelectionDAG::getCondCode(ISD::CondCode Cond) {
   return SDOperand(CondCodeNodes[Cond], 0);
 }
 
-SDOperand SelectionDAG::getRegister(unsigned RegNo, MVT::ValueType VT) {
+SDOperand SelectionDAG::getRegister(unsigned RegNo, MVT VT) {
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, ISD::Register, getVTList(VT), (SDOperand*)0, 0);
   ID.AddInteger(RegNo);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
@@ -977,7 +996,7 @@ SDOperand SelectionDAG::getSrcValue(const Value *V) {
          "SrcValue is not a pointer?");
 
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0);
+  AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), (SDOperand*)0, 0);
   ID.AddPointer(V);
 
   void *IP = 0;
@@ -990,13 +1009,13 @@ SDOperand SelectionDAG::getSrcValue(const Value *V) {
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getMemOperand(const MemOperand &MO) {
+SDOperand SelectionDAG::getMemOperand(const MachineMemOperand &MO) {
   const Value *v = MO.getValue();
   assert((!v || isa<PointerType>(v->getType())) &&
          "SrcValue is not a pointer?");
 
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::MEMOPERAND, getVTList(MVT::Other), 0, 0);
+  AddNodeIDNode(ID, ISD::MEMOPERAND, getVTList(MVT::Other), (SDOperand*)0, 0);
   ID.AddPointer(v);
   ID.AddInteger(MO.getFlags());
   ID.AddInteger(MO.getOffset());
@@ -1015,17 +1034,17 @@ SDOperand SelectionDAG::getMemOperand(const MemOperand &MO) {
 
 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
 /// specified value type.
-SDOperand SelectionDAG::CreateStackTemporary(MVT::ValueType VT) {
+SDOperand SelectionDAG::CreateStackTemporary(MVT VT) {
   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
-  unsigned ByteSize = MVT::getSizeInBits(VT)/8;
-  const Type *Ty = MVT::getTypeForValueType(VT);
+  unsigned ByteSize = VT.getSizeInBits()/8;
+  const Type *Ty = VT.getTypeForMVT();
   unsigned StackAlign = (unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty);
   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign);
   return getFrameIndex(FrameIdx, TLI.getPointerTy());
 }
 
 
-SDOperand SelectionDAG::FoldSetCC(MVT::ValueType VT, SDOperand N1,
+SDOperand SelectionDAG::FoldSetCC(MVT VT, SDOperand N1,
                                   SDOperand N2, ISD::CondCode Cond) {
   // These setcc operations always fold.
   switch (Cond) {
@@ -1045,33 +1064,27 @@ SDOperand SelectionDAG::FoldSetCC(MVT::ValueType VT, SDOperand N1,
   case ISD::SETUO:
   case ISD::SETUEQ:
   case ISD::SETUNE:
-    assert(!MVT::isInteger(N1.getValueType()) && "Illegal setcc for integer!");
+    assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
     break;
   }
   
   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val)) {
-    uint64_t C2 = N2C->getValue();
+    const APInt &C2 = N2C->getAPIntValue();
     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val)) {
-      uint64_t C1 = N1C->getValue();
-      
-      // Sign extend the operands if required
-      if (ISD::isSignedIntSetCC(Cond)) {
-        C1 = N1C->getSignExtended();
-        C2 = N2C->getSignExtended();
-      }
+      const APInt &C1 = N1C->getAPIntValue();
       
       switch (Cond) {
       default: assert(0 && "Unknown integer setcc!");
       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
       case ISD::SETNE:  return getConstant(C1 != C2, VT);
-      case ISD::SETULT: return getConstant(C1 <  C2, VT);
-      case ISD::SETUGT: return getConstant(C1 >  C2, VT);
-      case ISD::SETULE: return getConstant(C1 <= C2, VT);
-      case ISD::SETUGE: return getConstant(C1 >= C2, VT);
-      case ISD::SETLT:  return getConstant((int64_t)C1 <  (int64_t)C2, VT);
-      case ISD::SETGT:  return getConstant((int64_t)C1 >  (int64_t)C2, VT);
-      case ISD::SETLE:  return getConstant((int64_t)C1 <= (int64_t)C2, VT);
-      case ISD::SETGE:  return getConstant((int64_t)C1 >= (int64_t)C2, VT);
+      case ISD::SETULT: return getConstant(C1.ult(C2), VT);
+      case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
+      case ISD::SETULE: return getConstant(C1.ule(C2), VT);
+      case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
+      case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
+      case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
+      case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
+      case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
       }
     }
   }
@@ -1133,16 +1146,19 @@ SDOperand SelectionDAG::FoldSetCC(MVT::ValueType VT, SDOperand N1,
   return SDOperand();
 }
 
+/// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
+/// use this predicate to simplify operations downstream.
+bool SelectionDAG::SignBitIsZero(SDOperand Op, unsigned Depth) const {
+  unsigned BitWidth = Op.getValueSizeInBits();
+  return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
+}
+
 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
 /// this predicate to simplify operations downstream.  Mask is known to be zero
 /// for bits that V cannot have.
-bool SelectionDAG::MaskedValueIsZero(SDOperand Op, uint64_t Mask, 
+bool SelectionDAG::MaskedValueIsZero(SDOperand Op, const APInt &Mask, 
                                      unsigned Depth) const {
-  // The masks are not wide enough to represent this type!  Should use APInt.
-  if (Op.getValueType() == MVT::i128)
-    return false;
-  
-  uint64_t KnownZero, KnownOne;
+  APInt KnownZero, KnownOne;
   ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth);
   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 
   return (KnownZero & Mask) == Mask;
@@ -1156,7 +1172,7 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
                                      APInt &KnownZero, APInt &KnownOne,
                                      unsigned Depth) const {
   unsigned BitWidth = Mask.getBitWidth();
-  assert(BitWidth == MVT::getSizeInBits(Op.getValueType()) &&
+  assert(BitWidth == Op.getValueType().getSizeInBits() &&
          "Mask size mismatches value type size!");
 
   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
@@ -1209,6 +1225,52 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     KnownZero = KnownZeroOut;
     return;
   }
+  case ISD::MUL: {
+    APInt Mask2 = APInt::getAllOnesValue(BitWidth);
+    ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero, KnownOne, Depth+1);
+    ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1);
+    assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
+    assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?");
+
+    // If low bits are zero in either operand, output low known-0 bits.
+    // Also compute a conserative estimate for high known-0 bits.
+    // More trickiness is possible, but this is sufficient for the
+    // interesting case of alignment computation.
+    KnownOne.clear();
+    unsigned TrailZ = KnownZero.countTrailingOnes() +
+                      KnownZero2.countTrailingOnes();
+    unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
+                               KnownZero2.countLeadingOnes(),
+                               BitWidth) - BitWidth;
+
+    TrailZ = std::min(TrailZ, BitWidth);
+    LeadZ = std::min(LeadZ, BitWidth);
+    KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
+                APInt::getHighBitsSet(BitWidth, LeadZ);
+    KnownZero &= Mask;
+    return;
+  }
+  case ISD::UDIV: {
+    // For the purposes of computing leading zeros we can conservatively
+    // treat a udiv as a logical right shift by the power of 2 known to
+    // be less than the denominator.
+    APInt AllOnes = APInt::getAllOnesValue(BitWidth);
+    ComputeMaskedBits(Op.getOperand(0),
+                      AllOnes, KnownZero2, KnownOne2, Depth+1);
+    unsigned LeadZ = KnownZero2.countLeadingOnes();
+
+    KnownOne2.clear();
+    KnownZero2.clear();
+    ComputeMaskedBits(Op.getOperand(1),
+                      AllOnes, KnownZero2, KnownOne2, Depth+1);
+    unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
+    if (RHSUnknownLeadingOnes != BitWidth)
+      LeadZ = std::min(BitWidth,
+                       LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
+
+    KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ) & Mask;
+    return;
+  }
   case ISD::SELECT:
     ComputeMaskedBits(Op.getOperand(2), Mask, KnownZero, KnownOne, Depth+1);
     ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero2, KnownOne2, Depth+1);
@@ -1238,13 +1300,19 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
   case ISD::SHL:
     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-      ComputeMaskedBits(Op.getOperand(0), Mask.lshr(SA->getValue()),
+      unsigned ShAmt = SA->getValue();
+
+      // If the shift count is an invalid immediate, don't do anything.
+      if (ShAmt >= BitWidth)
+        return;
+
+      ComputeMaskedBits(Op.getOperand(0), Mask.lshr(ShAmt),
                         KnownZero, KnownOne, Depth+1);
       assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 
-      KnownZero <<= SA->getValue();
-      KnownOne  <<= SA->getValue();
+      KnownZero <<= ShAmt;
+      KnownOne  <<= ShAmt;
       // low bits known zero.
-      KnownZero |= APInt::getLowBitsSet(BitWidth, SA->getValue());
+      KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
     }
     return;
   case ISD::SRL:
@@ -1252,6 +1320,10 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
       unsigned ShAmt = SA->getValue();
 
+      // If the shift count is an invalid immediate, don't do anything.
+      if (ShAmt >= BitWidth)
+        return;
+
       ComputeMaskedBits(Op.getOperand(0), (Mask << ShAmt),
                         KnownZero, KnownOne, Depth+1);
       assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 
@@ -1266,6 +1338,10 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
       unsigned ShAmt = SA->getValue();
 
+      // If the shift count is an invalid immediate, don't do anything.
+      if (ShAmt >= BitWidth)
+        return;
+
       APInt InDemandedMask = (Mask << ShAmt);
       // If any of the demanded bits are produced by the sign extension, we also
       // demand the input sign bit.
@@ -1283,16 +1359,16 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
       APInt SignBit = APInt::getSignBit(BitWidth);
       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
       
-      if (!!(KnownZero & SignBit)) {
+      if (KnownZero.intersects(SignBit)) {
         KnownZero |= HighBits;  // New bits are known zero.
-      } else if (!!(KnownOne & SignBit)) {
+      } else if (KnownOne.intersects(SignBit)) {
         KnownOne  |= HighBits;  // New bits are known one.
       }
     }
     return;
   case ISD::SIGN_EXTEND_INREG: {
-    MVT::ValueType EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
-    unsigned EBits = MVT::getSizeInBits(EVT);
+    MVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+    unsigned EBits = EVT.getSizeInBits();
     
     // Sign extension.  Compute the demanded bits in the result that are not 
     // present in the input.
@@ -1313,10 +1389,10 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     
     // If the sign bit of the input is known set or clear, then we know the
     // top bits of the result.
-    if (!!(KnownZero & InSignBit)) {          // Input sign bit known clear
+    if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
       KnownZero |= NewBits;
       KnownOne  &= ~NewBits;
-    } else if (!!(KnownOne & InSignBit)) {    // Input sign bit known set
+    } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
       KnownOne  |= NewBits;
       KnownZero &= ~NewBits;
     } else {                              // Input sign bit unknown
@@ -1330,21 +1406,21 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
   case ISD::CTPOP: {
     unsigned LowBits = Log2_32(BitWidth)+1;
     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
-    KnownOne  = APInt(BitWidth, 0);
+    KnownOne.clear();
     return;
   }
   case ISD::LOAD: {
     if (ISD::isZEXTLoad(Op.Val)) {
       LoadSDNode *LD = cast<LoadSDNode>(Op);
-      MVT::ValueType VT = LD->getMemoryVT();
-      unsigned MemBits = MVT::getSizeInBits(VT);
+      MVT VT = LD->getMemoryVT();
+      unsigned MemBits = VT.getSizeInBits();
       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits) & Mask;
     }
     return;
   }
   case ISD::ZERO_EXTEND: {
-    MVT::ValueType InVT = Op.getOperand(0).getValueType();
-    unsigned InBits = MVT::getSizeInBits(InVT);
+    MVT InVT = Op.getOperand(0).getValueType();
+    unsigned InBits = InVT.getSizeInBits();
     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask;
     APInt InMask    = Mask;
     InMask.trunc(InBits);
@@ -1357,8 +1433,8 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     return;
   }
   case ISD::SIGN_EXTEND: {
-    MVT::ValueType InVT = Op.getOperand(0).getValueType();
-    unsigned InBits = MVT::getSizeInBits(InVT);
+    MVT InVT = Op.getOperand(0).getValueType();
+    unsigned InBits = InVT.getSizeInBits();
     APInt InSignBit = APInt::getSignBit(InBits);
     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits) & Mask;
     APInt InMask = Mask;
@@ -1398,8 +1474,8 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     return;
   }
   case ISD::ANY_EXTEND: {
-    MVT::ValueType InVT = Op.getOperand(0).getValueType();
-    unsigned InBits = MVT::getSizeInBits(InVT);
+    MVT InVT = Op.getOperand(0).getValueType();
+    unsigned InBits = InVT.getSizeInBits();
     APInt InMask = Mask;
     InMask.trunc(InBits);
     KnownZero.trunc(InBits);
@@ -1410,8 +1486,8 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     return;
   }
   case ISD::TRUNCATE: {
-    MVT::ValueType InVT = Op.getOperand(0).getValueType();
-    unsigned InBits = MVT::getSizeInBits(InVT);
+    MVT InVT = Op.getOperand(0).getValueType();
+    unsigned InBits = InVT.getSizeInBits();
     APInt InMask = Mask;
     InMask.zext(InBits);
     KnownZero.zext(InBits);
@@ -1423,8 +1499,8 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     break;
   }
   case ISD::AssertZext: {
-    MVT::ValueType VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
-    APInt InMask = APInt::getLowBitsSet(BitWidth, MVT::getSizeInBits(VT));
+    MVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+    APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
     ComputeMaskedBits(Op.getOperand(0), Mask & InMask, KnownZero, 
                       KnownOne, Depth+1);
     KnownZero |= (~InMask) & Mask;
@@ -1435,47 +1511,94 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
     return;
   
+  case ISD::SUB: {
+    if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
+      // We know that the top bits of C-X are clear if X contains less bits
+      // than C (i.e. no wrap-around can happen).  For example, 20-X is
+      // positive if we can prove that X is >= 0 and < 16.
+      if (CLHS->getAPIntValue().isNonNegative()) {
+        unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
+        // NLZ can't be BitWidth with no sign bit
+        APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
+        ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero2, KnownOne2,
+                          Depth+1);
+
+        // If all of the MaskV bits are known to be zero, then we know the
+        // output top bits are zero, because we now know that the output is
+        // from [0-C].
+        if ((KnownZero2 & MaskV) == MaskV) {
+          unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
+          // Top bits known zero.
+          KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2) & Mask;
+        }
+      }
+    }
+  }
+  // fall through
   case ISD::ADD: {
-    // If either the LHS or the RHS are Zero, the result is zero.
-    ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
-    ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero2, KnownOne2, Depth+1);
-    assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 
-    assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 
-    
     // Output known-0 bits are known if clear or set in both the low clear bits
     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
     // low 3 bits clear.
-    unsigned KnownZeroOut = std::min(KnownZero.countTrailingOnes(), 
-                                     KnownZero2.countTrailingOnes());
-    
-    KnownZero = APInt::getLowBitsSet(BitWidth, KnownZeroOut);
-    KnownOne = APInt(BitWidth, 0);
+    APInt Mask2 = APInt::getLowBitsSet(BitWidth, Mask.countTrailingOnes());
+    ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero2, KnownOne2, Depth+1);
+    assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 
+    unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
+
+    ComputeMaskedBits(Op.getOperand(1), Mask2, KnownZero2, KnownOne2, Depth+1);
+    assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 
+    KnownZeroOut = std::min(KnownZeroOut,
+                            KnownZero2.countTrailingOnes());
+
+    KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
     return;
   }
-  case ISD::SUB: {
-    ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0));
-    if (!CLHS) return;
-
-    // We know that the top bits of C-X are clear if X contains less bits
-    // than C (i.e. no wrap-around can happen).  For example, 20-X is
-    // positive if we can prove that X is >= 0 and < 16.
-    if (CLHS->getAPIntValue().isNonNegative()) {
-      unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
-      // NLZ can't be BitWidth with no sign bit
-      APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
-      ComputeMaskedBits(Op.getOperand(1), MaskV, KnownZero, KnownOne, Depth+1);
-
-      // If all of the MaskV bits are known to be zero, then we know the output
-      // top bits are zero, because we now know that the output is from [0-C].
-      if ((KnownZero & MaskV) == MaskV) {
-        unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
-        // Top bits known zero.
-        KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2) & Mask;
-        KnownOne = APInt(BitWidth, 0);   // No one bits known.
-      } else {
-        KnownZero = KnownOne = APInt(BitWidth, 0);  // Otherwise, nothing known.
+  case ISD::SREM:
+    if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+      APInt RA = Rem->getAPIntValue();
+      if (RA.isPowerOf2() || (-RA).isPowerOf2()) {
+        APInt LowBits = RA.isStrictlyPositive() ? (RA - 1) : ~RA;
+        APInt Mask2 = LowBits | APInt::getSignBit(BitWidth);
+        ComputeMaskedBits(Op.getOperand(0), Mask2,KnownZero2,KnownOne2,Depth+1);
+
+        // The sign of a remainder is equal to the sign of the first
+        // operand (zero being positive).
+        if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
+          KnownZero2 |= ~LowBits;
+        else if (KnownOne2[BitWidth-1])
+          KnownOne2 |= ~LowBits;
+
+        KnownZero |= KnownZero2 & Mask;
+        KnownOne |= KnownOne2 & Mask;
+
+        assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
+      }
+    }
+    return;
+  case ISD::UREM: {
+    if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+      APInt RA = Rem->getAPIntValue();
+      if (RA.isPowerOf2()) {
+        APInt LowBits = (RA - 1);
+        APInt Mask2 = LowBits & Mask;
+        KnownZero |= ~LowBits & Mask;
+        ComputeMaskedBits(Op.getOperand(0), Mask2, KnownZero, KnownOne,Depth+1);
+        assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
+        break;
       }
     }
+
+    // Since the result is less than or equal to either operand, any leading
+    // zero bits in either operand must also exist in the result.
+    APInt AllOnes = APInt::getAllOnesValue(BitWidth);
+    ComputeMaskedBits(Op.getOperand(0), AllOnes, KnownZero, KnownOne,
+                      Depth+1);
+    ComputeMaskedBits(Op.getOperand(1), AllOnes, KnownZero2, KnownOne2,
+                      Depth+1);
+
+    uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
+                                KnownZero2.countLeadingOnes());
+    KnownOne.clear();
+    KnownZero = APInt::getHighBitsSet(BitWidth, Leaders) & Mask;
     return;
   }
   default:
@@ -1490,35 +1613,17 @@ void SelectionDAG::ComputeMaskedBits(SDOperand Op, const APInt &Mask,
   }
 }
 
-/// ComputeMaskedBits - This is a wrapper around the APInt-using
-/// form of ComputeMaskedBits for use by clients that haven't been converted
-/// to APInt yet.
-void SelectionDAG::ComputeMaskedBits(SDOperand Op, uint64_t Mask, 
-                                     uint64_t &KnownZero, uint64_t &KnownOne,
-                                     unsigned Depth) const {
-  // The masks are not wide enough to represent this type!  Should use APInt.
-  if (Op.getValueType() == MVT::i128)
-    return;
-  
-  unsigned NumBits = MVT::getSizeInBits(Op.getValueType());
-  APInt APIntMask(NumBits, Mask);
-  APInt APIntKnownZero(NumBits, 0);
-  APInt APIntKnownOne(NumBits, 0);
-  ComputeMaskedBits(Op, APIntMask, APIntKnownZero, APIntKnownOne, Depth);
-  KnownZero = APIntKnownZero.getZExtValue();
-  KnownOne = APIntKnownOne.getZExtValue();
-}
-
 /// ComputeNumSignBits - Return the number of times the sign bit of the
 /// register is replicated into the other bits.  We know that at least 1 bit
 /// is always equal to the sign bit (itself), but other cases can give us
 /// information.  For example, immediately after an "SRA X, 2", we know that
 /// the top 3 bits are all equal to each other, so we return 3.
 unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
-  MVT::ValueType VT = Op.getValueType();
-  assert(MVT::isInteger(VT) && "Invalid VT!");
-  unsigned VTBits = MVT::getSizeInBits(VT);
+  MVT VT = Op.getValueType();
+  assert(VT.isInteger() && "Invalid VT!");
+  unsigned VTBits = VT.getSizeInBits();
   unsigned Tmp, Tmp2;
+  unsigned FirstAnswer = 1;
   
   if (Depth == 6)
     return 1;  // Limit search depth.
@@ -1526,33 +1631,29 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
   switch (Op.getOpcode()) {
   default: break;
   case ISD::AssertSext:
-    Tmp = MVT::getSizeInBits(cast<VTSDNode>(Op.getOperand(1))->getVT());
+    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
     return VTBits-Tmp+1;
   case ISD::AssertZext:
-    Tmp = MVT::getSizeInBits(cast<VTSDNode>(Op.getOperand(1))->getVT());
+    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
     return VTBits-Tmp;
     
   case ISD::Constant: {
-    uint64_t Val = cast<ConstantSDNode>(Op)->getValue();
-    // If negative, invert the bits, then look at it.
-    if (Val & MVT::getIntVTSignBit(VT))
-      Val = ~Val;
-    
-    // Shift the bits so they are the leading bits in the int64_t.
-    Val <<= 64-VTBits;
+    const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
+    // If negative, return # leading ones.
+    if (Val.isNegative())
+      return Val.countLeadingOnes();
     
-    // Return # leading zeros.  We use 'min' here in case Val was zero before
-    // shifting.  We don't want to return '64' as for an i32 "0".
-    return std::min(VTBits, CountLeadingZeros_64(Val));
+    // Return # leading zeros.
+    return Val.countLeadingZeros();
   }
     
   case ISD::SIGN_EXTEND:
-    Tmp = VTBits-MVT::getSizeInBits(Op.getOperand(0).getValueType());
+    Tmp = VTBits-Op.getOperand(0).getValueType().getSizeInBits();
     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
     
   case ISD::SIGN_EXTEND_INREG:
     // Max of the input and what this extends.
-    Tmp = MVT::getSizeInBits(cast<VTSDNode>(Op.getOperand(1))->getVT());
+    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
     Tmp = VTBits-Tmp+1;
     
     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
@@ -1578,16 +1679,21 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
   case ISD::AND:
   case ISD::OR:
   case ISD::XOR:    // NOT is handled here.
-    // Logical binary ops preserve the number of sign bits.
+    // Logical binary ops preserve the number of sign bits at the worst.
     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
-    if (Tmp == 1) return 1;  // Early out.
-    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
-    return std::min(Tmp, Tmp2);
+    if (Tmp != 1) {
+      Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
+      FirstAnswer = std::min(Tmp, Tmp2);
+      // We computed what we know about the sign bits as our first
+      // answer. Now proceed to the generic code that uses
+      // ComputeMaskedBits, and pick whichever answer is better.
+    }
+    break;
 
   case ISD::SELECT:
-    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
+    Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
     if (Tmp == 1) return 1;  // Early out.
-    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
+    Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
     return std::min(Tmp, Tmp2);
     
   case ISD::SETCC:
@@ -1620,18 +1726,18 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
     // Special case decrementing a value (ADD X, -1):
     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
       if (CRHS->isAllOnesValue()) {
-        uint64_t KnownZero, KnownOne;
-        uint64_t Mask = MVT::getIntVTBitMask(VT);
+        APInt KnownZero, KnownOne;
+        APInt Mask = APInt::getAllOnesValue(VTBits);
         ComputeMaskedBits(Op.getOperand(0), Mask, KnownZero, KnownOne, Depth+1);
         
         // If the input is known to be 0 or 1, the output is 0/-1, which is all
         // sign bits set.
-        if ((KnownZero|1) == Mask)
+        if ((KnownZero | APInt(VTBits, 1)) == Mask)
           return VTBits;
         
         // If we are subtracting one from a positive number, there is no carry
         // out of the result.
-        if (KnownZero & MVT::getIntVTSignBit(VT))
+        if (KnownZero.isNegative())
           return Tmp;
       }
       
@@ -1646,18 +1752,18 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
       
     // Handle NEG.
     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
-      if (CLHS->getValue() == 0) {
-        uint64_t KnownZero, KnownOne;
-        uint64_t Mask = MVT::getIntVTBitMask(VT);
+      if (CLHS->isNullValue()) {
+        APInt KnownZero, KnownOne;
+        APInt Mask = APInt::getAllOnesValue(VTBits);
         ComputeMaskedBits(Op.getOperand(1), Mask, KnownZero, KnownOne, Depth+1);
         // If the input is known to be 0 or 1, the output is 0/-1, which is all
         // sign bits set.
-        if ((KnownZero|1) == Mask)
+        if ((KnownZero | APInt(VTBits, 1)) == Mask)
           return VTBits;
         
         // If the input is known to be positive (the sign bit is known clear),
         // the output of the NEG has the same number of sign bits as the input.
-        if (KnownZero & MVT::getIntVTSignBit(VT))
+        if (KnownZero.isNegative())
           return Tmp2;
         
         // Otherwise, we treat this like a SUB.
@@ -1682,10 +1788,10 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
     switch (ExtType) {
     default: break;
     case ISD::SEXTLOAD:    // '17' bits known
-      Tmp = MVT::getSizeInBits(LD->getMemoryVT());
+      Tmp = LD->getMemoryVT().getSizeInBits();
       return VTBits-Tmp+1;
     case ISD::ZEXTLOAD:    // '16' bits known
-      Tmp = MVT::getSizeInBits(LD->getMemoryVT());
+      Tmp = LD->getMemoryVT().getSizeInBits();
       return VTBits-Tmp;
     }
   }
@@ -1696,32 +1802,31 @@ unsigned SelectionDAG::ComputeNumSignBits(SDOperand Op, unsigned Depth) const{
       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
       Op.getOpcode() == ISD::INTRINSIC_VOID) {
     unsigned NumBits = TLI.ComputeNumSignBitsForTargetNode(Op, Depth);
-    if (NumBits > 1) return NumBits;
+    if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
   }
   
   // Finally, if we can prove that the top bits of the result are 0's or 1's,
   // use this information.
-  uint64_t KnownZero, KnownOne;
-  uint64_t Mask = MVT::getIntVTBitMask(VT);
+  APInt KnownZero, KnownOne;
+  APInt Mask = APInt::getAllOnesValue(VTBits);
   ComputeMaskedBits(Op, Mask, KnownZero, KnownOne, Depth);
   
-  uint64_t SignBit = MVT::getIntVTSignBit(VT);
-  if (KnownZero & SignBit) {        // SignBit is 0
+  if (KnownZero.isNegative()) {        // sign bit is 0
     Mask = KnownZero;
-  } else if (KnownOne & SignBit) {  // SignBit is 1;
+  } else if (KnownOne.isNegative()) {  // sign bit is 1;
     Mask = KnownOne;
   } else {
     // Nothing known.
-    return 1;
+    return FirstAnswer;
   }
   
   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
   // the number of identical bits in the top of the input value.
-  Mask ^= ~0ULL;
-  Mask <<= 64-VTBits;
+  Mask = ~Mask;
+  Mask <<= Mask.getBitWidth()-VTBits;
   // Return # leading zeros.  We use 'min' here in case Val was zero before
   // shifting.  We don't want to return '64' as for an i32 "0".
-  return std::min(VTBits, CountLeadingZeros_64(Mask));
+  return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
 }
 
 
@@ -1735,11 +1840,40 @@ bool SelectionDAG::isVerifiedDebugInfoDesc(SDOperand Op) const {
 }
 
 
+/// getShuffleScalarElt - Returns the scalar element that will make up the ith
+/// element of the result of the vector shuffle.
+SDOperand SelectionDAG::getShuffleScalarElt(const SDNode *N, unsigned Idx) {
+  MVT VT = N->getValueType(0);
+  SDOperand PermMask = N->getOperand(2);
+  unsigned NumElems = PermMask.getNumOperands();
+  SDOperand V = (Idx < NumElems) ? N->getOperand(0) : N->getOperand(1);
+  Idx %= NumElems;
+
+  if (V.getOpcode() == ISD::BIT_CONVERT) {
+    V = V.getOperand(0);
+    if (V.getValueType().getVectorNumElements() != NumElems)
+      return SDOperand();
+  }
+  if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
+    return (Idx == 0) ? V.getOperand(0)
+                      : getNode(ISD::UNDEF, VT.getVectorElementType());
+  if (V.getOpcode() == ISD::BUILD_VECTOR)
+    return V.getOperand(Idx);
+  if (V.getOpcode() == ISD::VECTOR_SHUFFLE) {
+    SDOperand Elt = PermMask.getOperand(Idx);
+    if (Elt.getOpcode() == ISD::UNDEF)
+      return getNode(ISD::UNDEF, VT.getVectorElementType());
+    return getShuffleScalarElt(V.Val,cast<ConstantSDNode>(Elt)->getValue());
+  }
+  return SDOperand();
+}
+
+
 /// getNode - Gets or creates the specified node.
 ///
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT) {
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT) {
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0);
+  AddNodeIDNode(ID, Opcode, getVTList(VT), (SDOperand*)0, 0);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDOperand(E, 0);
@@ -1750,90 +1884,45 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT) {
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
-                                SDOperand Operand) {
-  unsigned Tmp1;
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT, SDOperand Operand) {
   // Constant fold unary operations with an integer constant operand.
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.Val)) {
-    uint64_t Val = C->getValue();
+    const APInt &Val = C->getAPIntValue();
+    unsigned BitWidth = VT.getSizeInBits();
     switch (Opcode) {
     default: break;
-    case ISD::SIGN_EXTEND: return getConstant(C->getSignExtended(), VT);
+    case ISD::SIGN_EXTEND:
+      return getConstant(APInt(Val).sextOrTrunc(BitWidth), VT);
     case ISD::ANY_EXTEND:
-    case ISD::ZERO_EXTEND: return getConstant(Val, VT);
-    case ISD::TRUNCATE:    return getConstant(Val, VT);
+    case ISD::ZERO_EXTEND:
+    case ISD::TRUNCATE:
+      return getConstant(APInt(Val).zextOrTrunc(BitWidth), VT);
     case ISD::UINT_TO_FP:
     case ISD::SINT_TO_FP: {
       const uint64_t zero[] = {0, 0};
       // No compile time operations on this type.
       if (VT==MVT::ppcf128)
         break;
-      APFloat apf = APFloat(APInt(MVT::getSizeInBits(VT), 2, zero));
-      (void)apf.convertFromZeroExtendedInteger(&Val, 
-                               MVT::getSizeInBits(Operand.getValueType()), 
-                               Opcode==ISD::SINT_TO_FP,
-                               APFloat::rmNearestTiesToEven);
+      APFloat apf = APFloat(APInt(BitWidth, 2, zero));
+      (void)apf.convertFromAPInt(Val, 
+                                 Opcode==ISD::SINT_TO_FP,
+                                 APFloat::rmNearestTiesToEven);
       return getConstantFP(apf, VT);
     }
     case ISD::BIT_CONVERT:
       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
-        return getConstantFP(BitsToFloat(Val), VT);
+        return getConstantFP(Val.bitsToFloat(), VT);
       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
-        return getConstantFP(BitsToDouble(Val), VT);
+        return getConstantFP(Val.bitsToDouble(), VT);
       break;
     case ISD::BSWAP:
-      switch(VT) {
-      default: assert(0 && "Invalid bswap!"); break;
-      case MVT::i16: return getConstant(ByteSwap_16((unsigned short)Val), VT);
-      case MVT::i32: return getConstant(ByteSwap_32((unsigned)Val), VT);
-      case MVT::i64: return getConstant(ByteSwap_64(Val), VT);
-      }
-      break;
+      return getConstant(Val.byteSwap(), VT);
     case ISD::CTPOP:
-      switch(VT) {
-      default: assert(0 && "Invalid ctpop!"); break;
-      case MVT::i1: return getConstant(Val != 0, VT);
-      case MVT::i8: 
-        Tmp1 = (unsigned)Val & 0xFF;
-        return getConstant(CountPopulation_32(Tmp1), VT);
-      case MVT::i16:
-        Tmp1 = (unsigned)Val & 0xFFFF;
-        return getConstant(CountPopulation_32(Tmp1), VT);
-      case MVT::i32:
-        return getConstant(CountPopulation_32((unsigned)Val), VT);
-      case MVT::i64:
-        return getConstant(CountPopulation_64(Val), VT);
-      }
+      return getConstant(Val.countPopulation(), VT);
     case ISD::CTLZ:
-      switch(VT) {
-      default: assert(0 && "Invalid ctlz!"); break;
-      case MVT::i1: return getConstant(Val == 0, VT);
-      case MVT::i8: 
-        Tmp1 = (unsigned)Val & 0xFF;
-        return getConstant(CountLeadingZeros_32(Tmp1)-24, VT);
-      case MVT::i16:
-        Tmp1 = (unsigned)Val & 0xFFFF;
-        return getConstant(CountLeadingZeros_32(Tmp1)-16, VT);
-      case MVT::i32:
-        return getConstant(CountLeadingZeros_32((unsigned)Val), VT);
-      case MVT::i64:
-        return getConstant(CountLeadingZeros_64(Val), VT);
-      }
+      return getConstant(Val.countLeadingZeros(), VT);
     case ISD::CTTZ:
-      switch(VT) {
-      default: assert(0 && "Invalid cttz!"); break;
-      case MVT::i1: return getConstant(Val == 0, VT);
-      case MVT::i8: 
-        Tmp1 = (unsigned)Val | 0x100;
-        return getConstant(CountTrailingZeros_32(Tmp1), VT);
-      case MVT::i16:
-        Tmp1 = (unsigned)Val | 0x10000;
-        return getConstant(CountTrailingZeros_32(Tmp1), VT);
-      case MVT::i32:
-        return getConstant(CountTrailingZeros_32((unsigned)Val), VT);
-      case MVT::i64:
-        return getConstant(CountTrailingZeros_64(Val), VT);
-      }
+      return getConstant(Val.countTrailingZeros(), VT);
     }
   }
 
@@ -1852,12 +1941,8 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
       case ISD::FP_EXTEND:
         // This can return overflow, underflow, or inexact; we don't care.
         // FIXME need to be more flexible about rounding mode.
-        (void) V.convert(VT==MVT::f32 ? APFloat::IEEEsingle : 
-                         VT==MVT::f64 ? APFloat::IEEEdouble :
-                         VT==MVT::f80 ? APFloat::x87DoubleExtended :
-                         VT==MVT::f128 ? APFloat::IEEEquad :
-                         APFloat::Bogus,
-                         APFloat::rmNearestTiesToEven);
+        (void)V.convert(*MVTToAPFloatSemantics(VT),
+                        APFloat::rmNearestTiesToEven);
         return getConstantFP(V, VT);
       case ISD::FP_TO_SINT:
       case ISD::FP_TO_UINT: {
@@ -1884,57 +1969,58 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
   unsigned OpOpcode = Operand.Val->getOpcode();
   switch (Opcode) {
   case ISD::TokenFactor:
-    return Operand;         // Factor of one node?  No factor.
+  case ISD::MERGE_VALUES:
+    return Operand;         // Factor or merge of one node?  No need.
   case ISD::FP_ROUND: assert(0 && "Invalid method to make FP_ROUND node");
   case ISD::FP_EXTEND:
-    assert(MVT::isFloatingPoint(VT) &&
-           MVT::isFloatingPoint(Operand.getValueType()) && "Invalid FP cast!");
+    assert(VT.isFloatingPoint() &&
+           Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
+    if (Operand.getOpcode() == ISD::UNDEF)
+      return getNode(ISD::UNDEF, VT);
     break;
-    case ISD::SIGN_EXTEND:
-    assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+  case ISD::SIGN_EXTEND:
+    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
            "Invalid SIGN_EXTEND!");
     if (Operand.getValueType() == VT) return Operand;   // noop extension
-    assert(MVT::getSizeInBits(Operand.getValueType()) < MVT::getSizeInBits(VT)
+    assert(Operand.getValueType().bitsLT(VT)
            && "Invalid sext node, dst < src!");
     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
       return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
     break;
   case ISD::ZERO_EXTEND:
-    assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
            "Invalid ZERO_EXTEND!");
     if (Operand.getValueType() == VT) return Operand;   // noop extension
-    assert(MVT::getSizeInBits(Operand.getValueType()) < MVT::getSizeInBits(VT)
+    assert(Operand.getValueType().bitsLT(VT)
            && "Invalid zext node, dst < src!");
     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
       return getNode(ISD::ZERO_EXTEND, VT, Operand.Val->getOperand(0));
     break;
   case ISD::ANY_EXTEND:
-    assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
            "Invalid ANY_EXTEND!");
     if (Operand.getValueType() == VT) return Operand;   // noop extension
-    assert(MVT::getSizeInBits(Operand.getValueType()) < MVT::getSizeInBits(VT)
+    assert(Operand.getValueType().bitsLT(VT)
            && "Invalid anyext node, dst < src!");
     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND)
       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
       return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
     break;
   case ISD::TRUNCATE:
-    assert(MVT::isInteger(VT) && MVT::isInteger(Operand.getValueType()) &&
+    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
            "Invalid TRUNCATE!");
     if (Operand.getValueType() == VT) return Operand;   // noop truncate
-    assert(MVT::getSizeInBits(Operand.getValueType()) > MVT::getSizeInBits(VT)
+    assert(Operand.getValueType().bitsGT(VT)
            && "Invalid truncate node, src < dst!");
     if (OpOpcode == ISD::TRUNCATE)
       return getNode(ISD::TRUNCATE, VT, Operand.Val->getOperand(0));
     else if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
              OpOpcode == ISD::ANY_EXTEND) {
       // If the source is smaller than the dest, we still need an extend.
-      if (MVT::getSizeInBits(Operand.Val->getOperand(0).getValueType())
-          < MVT::getSizeInBits(VT))
+      if (Operand.Val->getOperand(0).getValueType().bitsLT(VT))
         return getNode(OpOpcode, VT, Operand.Val->getOperand(0));
-      else if (MVT::getSizeInBits(Operand.Val->getOperand(0).getValueType())
-               > MVT::getSizeInBits(VT))
+      else if (Operand.Val->getOperand(0).getValueType().bitsGT(VT))
         return getNode(ISD::TRUNCATE, VT, Operand.Val->getOperand(0));
       else
         return Operand.Val->getOperand(0);
@@ -1942,7 +2028,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     break;
   case ISD::BIT_CONVERT:
     // Basic sanity checking.
-    assert(MVT::getSizeInBits(VT) == MVT::getSizeInBits(Operand.getValueType())
+    assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
            && "Cannot BIT_CONVERT between types of different sizes!");
     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
     if (OpOpcode == ISD::BIT_CONVERT)  // bitconv(bitconv(x)) -> bitconv(x)
@@ -1951,9 +2037,17 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
       return getNode(ISD::UNDEF, VT);
     break;
   case ISD::SCALAR_TO_VECTOR:
-    assert(MVT::isVector(VT) && !MVT::isVector(Operand.getValueType()) &&
-           MVT::getVectorElementType(VT) == Operand.getValueType() &&
+    assert(VT.isVector() && !Operand.getValueType().isVector() &&
+           VT.getVectorElementType() == Operand.getValueType() &&
            "Illegal SCALAR_TO_VECTOR node!");
+    if (OpOpcode == ISD::UNDEF)
+      return getNode(ISD::UNDEF, VT);
+    // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
+    if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
+        isa<ConstantSDNode>(Operand.getOperand(1)) &&
+        Operand.getConstantOperandVal(1) == 0 &&
+        Operand.getOperand(0).getValueType() == VT)
+      return Operand.getOperand(0);
     break;
   case ISD::FNEG:
     if (OpOpcode == ISD::FSUB)   // -(X-Y) -> (Y-X)
@@ -1988,7 +2082,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
 
 
 
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT,
                                 SDOperand N1, SDOperand N2) {
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.Val);
@@ -2002,32 +2096,32 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     if (N2.getOpcode() == ISD::EntryToken) return N1;
     break;
   case ISD::AND:
-    assert(MVT::isInteger(VT) && N1.getValueType() == N2.getValueType() &&
+    assert(VT.isInteger() && N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
     // worth handling here.
-    if (N2C && N2C->getValue() == 0)
+    if (N2C && N2C->isNullValue())
       return N2;
     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
       return N1;
     break;
   case ISD::OR:
   case ISD::XOR:
-    assert(MVT::isInteger(VT) && N1.getValueType() == N2.getValueType() &&
+  case ISD::ADD:
+  case ISD::SUB:
+    assert(VT.isInteger() && N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
-    // (X ^| 0) -> X.  This commonly occurs when legalizing i64 values, so it's
-    // worth handling here.
-    if (N2C && N2C->getValue() == 0)
+    // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
+    // it's worth handling here.
+    if (N2C && N2C->isNullValue())
       return N1;
     break;
   case ISD::UDIV:
   case ISD::UREM:
   case ISD::MULHU:
   case ISD::MULHS:
-    assert(MVT::isInteger(VT) && "This operator does not apply to FP types!");
+    assert(VT.isInteger() && "This operator does not apply to FP types!");
     // fall through
-  case ISD::ADD:
-  case ISD::SUB:
   case ISD::MUL:
   case ISD::SDIV:
   case ISD::SREM:
@@ -2041,8 +2135,8 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     break;
   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
     assert(N1.getValueType() == VT &&
-           MVT::isFloatingPoint(N1.getValueType()) && 
-           MVT::isFloatingPoint(N2.getValueType()) &&
+           N1.getValueType().isFloatingPoint() &&
+           N2.getValueType().isFloatingPoint() &&
            "Invalid FCOPYSIGN!");
     break;
   case ISD::SHL:
@@ -2052,51 +2146,48 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
   case ISD::ROTR:
     assert(VT == N1.getValueType() &&
            "Shift operators return type must be the same as their first arg");
-    assert(MVT::isInteger(VT) && MVT::isInteger(N2.getValueType()) &&
+    assert(VT.isInteger() && N2.getValueType().isInteger() &&
            VT != MVT::i1 && "Shifts only work on integers");
     break;
   case ISD::FP_ROUND_INREG: {
-    MVT::ValueType EVT = cast<VTSDNode>(N2)->getVT();
+    MVT EVT = cast<VTSDNode>(N2)->getVT();
     assert(VT == N1.getValueType() && "Not an inreg round!");
-    assert(MVT::isFloatingPoint(VT) && MVT::isFloatingPoint(EVT) &&
+    assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
            "Cannot FP_ROUND_INREG integer types");
-    assert(MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(VT) &&
-           "Not rounding down!");
+    assert(EVT.bitsLE(VT) && "Not rounding down!");
     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
     break;
   }
   case ISD::FP_ROUND:
-    assert(MVT::isFloatingPoint(VT) &&
-           MVT::isFloatingPoint(N1.getValueType()) &&
-           MVT::getSizeInBits(VT) <= MVT::getSizeInBits(N1.getValueType()) &&
+    assert(VT.isFloatingPoint() &&
+           N1.getValueType().isFloatingPoint() &&
+           VT.bitsLE(N1.getValueType()) &&
            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
     if (N1.getValueType() == VT) return N1;  // noop conversion.
     break;
   case ISD::AssertSext:
   case ISD::AssertZext: {
-    MVT::ValueType EVT = cast<VTSDNode>(N2)->getVT();
+    MVT EVT = cast<VTSDNode>(N2)->getVT();
     assert(VT == N1.getValueType() && "Not an inreg extend!");
-    assert(MVT::isInteger(VT) && MVT::isInteger(EVT) &&
+    assert(VT.isInteger() && EVT.isInteger() &&
            "Cannot *_EXTEND_INREG FP types");
-    assert(MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(VT) &&
-           "Not extending!");
+    assert(EVT.bitsLE(VT) && "Not extending!");
     if (VT == EVT) return N1; // noop assertion.
     break;
   }
   case ISD::SIGN_EXTEND_INREG: {
-    MVT::ValueType EVT = cast<VTSDNode>(N2)->getVT();
+    MVT EVT = cast<VTSDNode>(N2)->getVT();
     assert(VT == N1.getValueType() && "Not an inreg extend!");
-    assert(MVT::isInteger(VT) && MVT::isInteger(EVT) &&
+    assert(VT.isInteger() && EVT.isInteger() &&
            "Cannot *_EXTEND_INREG FP types");
-    assert(MVT::getSizeInBits(EVT) <= MVT::getSizeInBits(VT) &&
-           "Not extending!");
+    assert(EVT.bitsLE(VT) && "Not extending!");
     if (EVT == VT) return N1;  // Not actually extending
 
     if (N1C) {
-      int64_t Val = N1C->getValue();
-      unsigned FromBits = MVT::getSizeInBits(cast<VTSDNode>(N2)->getVT());
-      Val <<= 64-FromBits;
-      Val >>= 64-FromBits;
+      APInt Val = N1C->getAPIntValue();
+      unsigned FromBits = cast<VTSDNode>(N2)->getVT().getSizeInBits();
+      Val <<= Val.getBitWidth()-FromBits;
+      Val = Val.ashr(Val.getBitWidth()-FromBits);
       return getConstant(Val, VT);
     }
     break;
@@ -2104,12 +2195,16 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
   case ISD::EXTRACT_VECTOR_ELT:
     assert(N2C && "Bad EXTRACT_VECTOR_ELT!");
 
+    // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
+    if (N1.getOpcode() == ISD::UNDEF)
+      return getNode(ISD::UNDEF, VT);
+      
     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
     // expanding copies of large vectors from registers.
     if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
         N1.getNumOperands() > 0) {
       unsigned Factor =
-        MVT::getVectorNumElements(N1.getOperand(0).getValueType());
+        N1.getOperand(0).getValueType().getVectorNumElements();
       return getNode(ISD::EXTRACT_VECTOR_ELT, VT,
                      N1.getOperand(N2C->getValue() / Factor),
                      getConstant(N2C->getValue() % Factor, N2.getValueType()));
@@ -2119,7 +2214,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     // expanding large vector constants.
     if (N1.getOpcode() == ISD::BUILD_VECTOR)
       return N1.getOperand(N2C->getValue());
-
+      
     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
     // operations are lowered to scalars.
     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT)
@@ -2132,54 +2227,58 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     break;
   case ISD::EXTRACT_ELEMENT:
     assert(N2C && (unsigned)N2C->getValue() < 2 && "Bad EXTRACT_ELEMENT!");
-    
+    assert(!N1.getValueType().isVector() &&
+           N1.getValueType().isInteger() &&
+           !VT.isVector() && VT.isInteger() &&
+           "EXTRACT_ELEMENT only applies to integers!");
+
     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
     // 64-bit integers into 32-bit parts.  Instead of building the extract of
     // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 
     if (N1.getOpcode() == ISD::BUILD_PAIR)
       return N1.getOperand(N2C->getValue());
-    
+
     // EXTRACT_ELEMENT of a constant int is also very common.
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
-      unsigned Shift = MVT::getSizeInBits(VT) * N2C->getValue();
-      return getConstant(C->getValue() >> Shift, VT);
+      unsigned ElementSize = VT.getSizeInBits();
+      unsigned Shift = ElementSize * N2C->getValue();
+      APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
+      return getConstant(ShiftedVal.trunc(ElementSize), VT);
     }
     break;
+  case ISD::EXTRACT_SUBVECTOR:
+    if (N1.getValueType() == VT) // Trivial extraction.
+      return N1;
+    break;
   }
 
   if (N1C) {
     if (N2C) {
-      uint64_t C1 = N1C->getValue(), C2 = N2C->getValue();
+      APInt C1 = N1C->getAPIntValue(), C2 = N2C->getAPIntValue();
       switch (Opcode) {
       case ISD::ADD: return getConstant(C1 + C2, VT);
       case ISD::SUB: return getConstant(C1 - C2, VT);
       case ISD::MUL: return getConstant(C1 * C2, VT);
       case ISD::UDIV:
-        if (C2) return getConstant(C1 / C2, VT);
+        if (C2.getBoolValue()) return getConstant(C1.udiv(C2), VT);
         break;
       case ISD::UREM :
-        if (C2) return getConstant(C1 % C2, VT);
+        if (C2.getBoolValue()) return getConstant(C1.urem(C2), VT);
         break;
       case ISD::SDIV :
-        if (C2) return getConstant(N1C->getSignExtended() /
-                                   N2C->getSignExtended(), VT);
+        if (C2.getBoolValue()) return getConstant(C1.sdiv(C2), VT);
         break;
       case ISD::SREM :
-        if (C2) return getConstant(N1C->getSignExtended() %
-                                   N2C->getSignExtended(), VT);
+        if (C2.getBoolValue()) return getConstant(C1.srem(C2), VT);
         break;
       case ISD::AND  : return getConstant(C1 & C2, VT);
       case ISD::OR   : return getConstant(C1 | C2, VT);
       case ISD::XOR  : return getConstant(C1 ^ C2, VT);
       case ISD::SHL  : return getConstant(C1 << C2, VT);
-      case ISD::SRL  : return getConstant(C1 >> C2, VT);
-      case ISD::SRA  : return getConstant(N1C->getSignExtended() >>(int)C2, VT);
-      case ISD::ROTL : 
-        return getConstant((C1 << C2) | (C1 >> (MVT::getSizeInBits(VT) - C2)),
-                           VT);
-      case ISD::ROTR : 
-        return getConstant((C1 >> C2) | (C1 << (MVT::getSizeInBits(VT) - C2)), 
-                           VT);
+      case ISD::SRL  : return getConstant(C1.lshr(C2), VT);
+      case ISD::SRA  : return getConstant(C1.ashr(C2), VT);
+      case ISD::ROTL : return getConstant(C1.rotl(C2), VT);
+      case ISD::ROTR : return getConstant(C1.rotr(C2), VT);
       default: break;
       }
     } else {      // Cannonicalize constant to RHS if commutative
@@ -2255,7 +2354,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
       case ISD::SREM:
       case ISD::SRL:
       case ISD::SHL:
-        if (!MVT::isVector(VT)) 
+        if (!VT.isVector())
           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
         // For vectors, we can't easily build an all zero vector, just return
         // the LHS.
@@ -2267,6 +2366,12 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
   // Fold a bunch of operators when the RHS is undef. 
   if (N2.getOpcode() == ISD::UNDEF) {
     switch (Opcode) {
+    case ISD::XOR:
+      if (N1.getOpcode() == ISD::UNDEF)
+        // Handle undef ^ undef -> 0 special case. This is a common
+        // idiom (misuse).
+        return getConstant(0, VT);
+      // fallthrough
     case ISD::ADD:
     case ISD::ADDC:
     case ISD::ADDE:
@@ -2280,20 +2385,19 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     case ISD::SDIV:
     case ISD::UREM:
     case ISD::SREM:
-    case ISD::XOR:
       return N2;       // fold op(arg1, undef) -> undef
     case ISD::MUL: 
     case ISD::AND:
     case ISD::SRL:
     case ISD::SHL:
-      if (!MVT::isVector(VT)) 
+      if (!VT.isVector())
         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
       // For vectors, we can't easily build an all zero vector, just return
       // the LHS.
       return N1;
     case ISD::OR:
-      if (!MVT::isVector(VT)) 
-        return getConstant(MVT::getIntVTBitMask(VT), VT);
+      if (!VT.isVector())
+        return getConstant(VT.getIntegerVTBitMask(), VT);
       // For vectors, we can't easily build an all one vector, just return
       // the LHS.
       return N1;
@@ -2322,7 +2426,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT,
                                 SDOperand N1, SDOperand N2, SDOperand N3) {
   // Perform various simplifications.
   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.Val);
@@ -2354,9 +2458,9 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
     break;
   case ISD::VECTOR_SHUFFLE:
     assert(VT == N1.getValueType() && VT == N2.getValueType() &&
-           MVT::isVector(VT) && MVT::isVector(N3.getValueType()) &&
+           VT.isVector() && N3.getValueType().isVector() &&
            N3.getOpcode() == ISD::BUILD_VECTOR &&
-           MVT::getVectorNumElements(VT) == N3.getNumOperands() &&
+           VT.getVectorNumElements() == N3.getNumOperands() &&
            "Illegal VECTOR_SHUFFLE node!");
     break;
   case ISD::BIT_CONVERT:
@@ -2385,170 +2489,630 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT,
                                 SDOperand N1, SDOperand N2, SDOperand N3,
                                 SDOperand N4) {
   SDOperand Ops[] = { N1, N2, N3, N4 };
   return getNode(Opcode, VT, Ops, 4);
 }
 
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT,
                                 SDOperand N1, SDOperand N2, SDOperand N3,
                                 SDOperand N4, SDOperand N5) {
   SDOperand Ops[] = { N1, N2, N3, N4, N5 };
   return getNode(Opcode, VT, Ops, 5);
 }
 
-SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dest,
-                                  SDOperand Src, SDOperand Size,
-                                  SDOperand Align,
-                                  SDOperand AlwaysInline) {
-  SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline };
-  return getNode(ISD::MEMCPY, MVT::Other, Ops, 6);
+/// getMemsetValue - Vectorized representation of the memset value
+/// operand.
+static SDOperand getMemsetValue(SDOperand Value, MVT VT, SelectionDAG &DAG) {
+  unsigned NumBits = VT.isVector() ?
+    VT.getVectorElementType().getSizeInBits() : VT.getSizeInBits();
+  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
+    APInt Val = APInt(NumBits, C->getValue() & 255);
+    unsigned Shift = 8;
+    for (unsigned i = NumBits; i > 8; i >>= 1) {
+      Val = (Val << Shift) | Val;
+      Shift <<= 1;
+    }
+    if (VT.isInteger())
+      return DAG.getConstant(Val, VT);
+    return DAG.getConstantFP(APFloat(Val), VT);
+  }
+
+  Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
+  unsigned Shift = 8;
+  for (unsigned i = NumBits; i > 8; i >>= 1) {
+    Value = DAG.getNode(ISD::OR, VT,
+                        DAG.getNode(ISD::SHL, VT, Value,
+                                    DAG.getConstant(Shift, MVT::i8)), Value);
+    Shift <<= 1;
+  }
+
+  return Value;
+}
+
+/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
+/// used when a memcpy is turned into a memset when the source is a constant
+/// string ptr.
+static SDOperand getMemsetStringVal(MVT VT, SelectionDAG &DAG,
+                                    const TargetLowering &TLI,
+                                    std::string &Str, unsigned Offset) {
+  assert(!VT.isVector() && "Can't handle vector type here!");
+  unsigned NumBits = VT.getSizeInBits();
+  unsigned MSB = NumBits / 8;
+  uint64_t Val = 0;
+  if (TLI.isLittleEndian())
+    Offset = Offset + MSB - 1;
+  for (unsigned i = 0; i != MSB; ++i) {
+    Val = (Val << 8) | (unsigned char)Str[Offset];
+    Offset += TLI.isLittleEndian() ? -1 : 1;
+  }
+  return DAG.getConstant(Val, VT);
 }
 
-SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dest,
-                                  SDOperand Src, SDOperand Size,
-                                  SDOperand Align,
-                                  SDOperand AlwaysInline) {
-  SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline };
-  return getNode(ISD::MEMMOVE, MVT::Other, Ops, 6);
+/// getMemBasePlusOffset - Returns base and offset node for the 
+///
+static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
+                                      SelectionDAG &DAG) {
+  MVT VT = Base.getValueType();
+  return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
 }
 
-SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dest,
-                                  SDOperand Src, SDOperand Size,
-                                  SDOperand Align,
-                                  SDOperand AlwaysInline) {
-  SDOperand Ops[] = { Chain, Dest, Src, Size, Align, AlwaysInline };
-  return getNode(ISD::MEMSET, MVT::Other, Ops, 6);
+/// isMemSrcFromString - Returns true if memcpy source is a string constant.
+///
+static bool isMemSrcFromString(SDOperand Src, std::string &Str,
+                               uint64_t &SrcOff) {
+  unsigned SrcDelta = 0;
+  GlobalAddressSDNode *G = NULL;
+  if (Src.getOpcode() == ISD::GlobalAddress)
+    G = cast<GlobalAddressSDNode>(Src);
+  else if (Src.getOpcode() == ISD::ADD &&
+           Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
+           Src.getOperand(1).getOpcode() == ISD::Constant) {
+    G = cast<GlobalAddressSDNode>(Src.getOperand(0));
+    SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
+  }
+  if (!G)
+    return false;
+
+  GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
+  if (GV && GV->isConstant()) {
+    Str = GV->getStringValue(false);
+    if (!Str.empty()) {
+      SrcOff += SrcDelta;
+      return true;
+    }
+  }
+
+  return false;
 }
 
-SDOperand SelectionDAG::getLoad(MVT::ValueType VT,
-                                SDOperand Chain, SDOperand Ptr,
-                                const Value *SV, int SVOffset,
-                                bool isVolatile, unsigned Alignment) {
-  if (Alignment == 0) { // Ensure that codegen never sees alignment 0
-    const Type *Ty = 0;
-    if (VT != MVT::iPTR) {
-      Ty = MVT::getTypeForValueType(VT);
-    } else if (SV) {
-      const PointerType *PT = dyn_cast<PointerType>(SV->getType());
-      assert(PT && "Value for load must be a pointer");
-      Ty = PT->getElementType();
-    }  
-    assert(Ty && "Could not get type information for load");
-    Alignment = TLI.getTargetData()->getABITypeAlignment(Ty);
+/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
+/// to replace the memset / memcpy is below the threshold. It also returns the
+/// types of the sequence of memory ops to perform memset / memcpy.
+static
+bool MeetsMaxMemopRequirement(std::vector<MVT> &MemOps,
+                              SDOperand Dst, SDOperand Src,
+                              unsigned Limit, uint64_t Size, unsigned &Align,
+                              SelectionDAG &DAG,
+                              const TargetLowering &TLI) {
+  bool AllowUnalign = TLI.allowsUnalignedMemoryAccesses();
+
+  std::string Str;
+  uint64_t SrcOff = 0;
+  bool isSrcStr = isMemSrcFromString(Src, Str, SrcOff);
+  bool isSrcConst = isa<ConstantSDNode>(Src);
+  MVT VT= TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr);
+  if (VT != MVT::iAny) {
+    unsigned NewAlign = (unsigned)
+      TLI.getTargetData()->getABITypeAlignment(VT.getTypeForMVT());
+    // If source is a string constant, this will require an unaligned load.
+    if (NewAlign > Align && (isSrcConst || AllowUnalign)) {
+      if (Dst.getOpcode() != ISD::FrameIndex) {
+        // Can't change destination alignment. It requires a unaligned store.
+        if (AllowUnalign)
+          VT = MVT::iAny;
+      } else {
+        int FI = cast<FrameIndexSDNode>(Dst)->getIndex();
+        MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+        if (MFI->isFixedObjectIndex(FI)) {
+          // Can't change destination alignment. It requires a unaligned store.
+          if (AllowUnalign)
+            VT = MVT::iAny;
+        } else {
+          // Give the stack frame object a larger alignment if needed.
+          if (MFI->getObjectAlignment(FI) < NewAlign)
+            MFI->setObjectAlignment(FI, NewAlign);
+          Align = NewAlign;
+        }
+      }
+    }
   }
-  SDVTList VTs = getVTList(VT, MVT::Other);
-  SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
-  SDOperand Ops[] = { Chain, Ptr, Undef };
+
+  if (VT == MVT::iAny) {
+    if (AllowUnalign) {
+      VT = MVT::i64;
+    } else {
+      switch (Align & 7) {
+      case 0:  VT = MVT::i64; break;
+      case 4:  VT = MVT::i32; break;
+      case 2:  VT = MVT::i16; break;
+      default: VT = MVT::i8;  break;
+      }
+    }
+
+    MVT LVT = MVT::i64;
+    while (!TLI.isTypeLegal(LVT))
+      LVT = (MVT::SimpleValueType)(LVT.getSimpleVT() - 1);
+    assert(LVT.isInteger());
+
+    if (VT.bitsGT(LVT))
+      VT = LVT;
+  }
+
+  unsigned NumMemOps = 0;
+  while (Size != 0) {
+    unsigned VTSize = VT.getSizeInBits() / 8;
+    while (VTSize > Size) {
+      // For now, only use non-vector load / store's for the left-over pieces.
+      if (VT.isVector()) {
+        VT = MVT::i64;
+        while (!TLI.isTypeLegal(VT))
+          VT = (MVT::SimpleValueType)(VT.getSimpleVT() - 1);
+        VTSize = VT.getSizeInBits() / 8;
+      } else {
+        VT = (MVT::SimpleValueType)(VT.getSimpleVT() - 1);
+        VTSize >>= 1;
+      }
+    }
+
+    if (++NumMemOps > Limit)
+      return false;
+    MemOps.push_back(VT);
+    Size -= VTSize;
+  }
+
+  return true;
+}
+
+static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
+                                         SDOperand Chain, SDOperand Dst,
+                                         SDOperand Src, uint64_t Size,
+                                         unsigned Align, bool AlwaysInline,
+                                         const Value *DstSV, uint64_t DstSVOff,
+                                         const Value *SrcSV, uint64_t SrcSVOff){
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // Expand memcpy to a series of load and store ops if the size operand falls
+  // below a certain threshold.
+  std::vector<MVT> MemOps;
+  uint64_t Limit = -1;
+  if (!AlwaysInline)
+    Limit = TLI.getMaxStoresPerMemcpy();
+  unsigned DstAlign = Align;  // Destination alignment can change.
+  if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign,
+                                DAG, TLI))
+    return SDOperand();
+
+  std::string Str;
+  uint64_t SrcOff = 0, DstOff = 0;
+  bool CopyFromStr = isMemSrcFromString(Src, Str, SrcOff);
+
+  SmallVector<SDOperand, 8> OutChains;
+  unsigned NumMemOps = MemOps.size();
+  for (unsigned i = 0; i < NumMemOps; i++) {
+    MVT VT = MemOps[i];
+    unsigned VTSize = VT.getSizeInBits() / 8;
+    SDOperand Value, Store;
+
+    if (CopyFromStr && !VT.isVector()) {
+      // It's unlikely a store of a vector immediate can be done in a single
+      // instruction. It would require a load from a constantpool first.
+      // FIXME: Handle cases where store of vector immediate is done in a
+      // single instruction.
+      Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
+      Store = DAG.getStore(Chain, Value,
+                           getMemBasePlusOffset(Dst, DstOff, DAG),
+                           DstSV, DstSVOff + DstOff);
+    } else {
+      Value = DAG.getLoad(VT, Chain,
+                          getMemBasePlusOffset(Src, SrcOff, DAG),
+                          SrcSV, SrcSVOff + SrcOff, false, Align);
+      Store = DAG.getStore(Chain, Value,
+                           getMemBasePlusOffset(Dst, DstOff, DAG),
+                           DstSV, DstSVOff + DstOff, false, DstAlign);
+    }
+    OutChains.push_back(Store);
+    SrcOff += VTSize;
+    DstOff += VTSize;
+  }
+
+  return DAG.getNode(ISD::TokenFactor, MVT::Other,
+                     &OutChains[0], OutChains.size());
+}
+
+static SDOperand getMemmoveLoadsAndStores(SelectionDAG &DAG,
+                                          SDOperand Chain, SDOperand Dst,
+                                          SDOperand Src, uint64_t Size,
+                                          unsigned Align, bool AlwaysInline,
+                                          const Value *DstSV, uint64_t DstSVOff,
+                                          const Value *SrcSV, uint64_t SrcSVOff){
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // Expand memmove to a series of load and store ops if the size operand falls
+  // below a certain threshold.
+  std::vector<MVT> MemOps;
+  uint64_t Limit = -1;
+  if (!AlwaysInline)
+    Limit = TLI.getMaxStoresPerMemmove();
+  unsigned DstAlign = Align;  // Destination alignment can change.
+  if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign,
+                                DAG, TLI))
+    return SDOperand();
+
+  uint64_t SrcOff = 0, DstOff = 0;
+
+  SmallVector<SDOperand, 8> LoadValues;
+  SmallVector<SDOperand, 8> LoadChains;
+  SmallVector<SDOperand, 8> OutChains;
+  unsigned NumMemOps = MemOps.size();
+  for (unsigned i = 0; i < NumMemOps; i++) {
+    MVT VT = MemOps[i];
+    unsigned VTSize = VT.getSizeInBits() / 8;
+    SDOperand Value, Store;
+
+    Value = DAG.getLoad(VT, Chain,
+                        getMemBasePlusOffset(Src, SrcOff, DAG),
+                        SrcSV, SrcSVOff + SrcOff, false, Align);
+    LoadValues.push_back(Value);
+    LoadChains.push_back(Value.getValue(1));
+    SrcOff += VTSize;
+  }
+  Chain = DAG.getNode(ISD::TokenFactor, MVT::Other,
+                      &LoadChains[0], LoadChains.size());
+  OutChains.clear();
+  for (unsigned i = 0; i < NumMemOps; i++) {
+    MVT VT = MemOps[i];
+    unsigned VTSize = VT.getSizeInBits() / 8;
+    SDOperand Value, Store;
+
+    Store = DAG.getStore(Chain, LoadValues[i],
+                         getMemBasePlusOffset(Dst, DstOff, DAG),
+                         DstSV, DstSVOff + DstOff, false, DstAlign);
+    OutChains.push_back(Store);
+    DstOff += VTSize;
+  }
+
+  return DAG.getNode(ISD::TokenFactor, MVT::Other,
+                     &OutChains[0], OutChains.size());
+}
+
+static SDOperand getMemsetStores(SelectionDAG &DAG,
+                                 SDOperand Chain, SDOperand Dst,
+                                 SDOperand Src, uint64_t Size,
+                                 unsigned Align,
+                                 const Value *DstSV, uint64_t DstSVOff) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // Expand memset to a series of load/store ops if the size operand
+  // falls below a certain threshold.
+  std::vector<MVT> MemOps;
+  if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, TLI.getMaxStoresPerMemset(),
+                                Size, Align, DAG, TLI))
+    return SDOperand();
+
+  SmallVector<SDOperand, 8> OutChains;
+  uint64_t DstOff = 0;
+
+  unsigned NumMemOps = MemOps.size();
+  for (unsigned i = 0; i < NumMemOps; i++) {
+    MVT VT = MemOps[i];
+    unsigned VTSize = VT.getSizeInBits() / 8;
+    SDOperand Value = getMemsetValue(Src, VT, DAG);
+    SDOperand Store = DAG.getStore(Chain, Value,
+                                   getMemBasePlusOffset(Dst, DstOff, DAG),
+                                   DstSV, DstSVOff + DstOff);
+    OutChains.push_back(Store);
+    DstOff += VTSize;
+  }
+
+  return DAG.getNode(ISD::TokenFactor, MVT::Other,
+                     &OutChains[0], OutChains.size());
+}
+
+SDOperand SelectionDAG::getMemcpy(SDOperand Chain, SDOperand Dst,
+                                  SDOperand Src, SDOperand Size,
+                                  unsigned Align, bool AlwaysInline,
+                                  const Value *DstSV, uint64_t DstSVOff,
+                                  const Value *SrcSV, uint64_t SrcSVOff) {
+
+  // Check to see if we should lower the memcpy to loads and stores first.
+  // For cases within the target-specified limits, this is the best choice.
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (ConstantSize) {
+    // Memcpy with size zero? Just return the original chain.
+    if (ConstantSize->isNullValue())
+      return Chain;
+
+    SDOperand Result =
+      getMemcpyLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(),
+                              Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
+    if (Result.Val)
+      return Result;
+  }
+
+  // Then check to see if we should lower the memcpy with target-specific
+  // code. If the target chooses to do this, this is the next best.
+  SDOperand Result =
+    TLI.EmitTargetCodeForMemcpy(*this, Chain, Dst, Src, Size, Align,
+                                AlwaysInline,
+                                DstSV, DstSVOff, SrcSV, SrcSVOff);
+  if (Result.Val)
+    return Result;
+
+  // If we really need inline code and the target declined to provide it,
+  // use a (potentially long) sequence of loads and stores.
+  if (AlwaysInline) {
+    assert(ConstantSize && "AlwaysInline requires a constant size!");
+    return getMemcpyLoadsAndStores(*this, Chain, Dst, Src,
+                                   ConstantSize->getValue(), Align, true,
+                                   DstSV, DstSVOff, SrcSV, SrcSVOff);
+  }
+
+  // Emit a library call.
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Ty = TLI.getTargetData()->getIntPtrType();
+  Entry.Node = Dst; Args.push_back(Entry);
+  Entry.Node = Src; Args.push_back(Entry);
+  Entry.Node = Size; Args.push_back(Entry);
+  std::pair<SDOperand,SDOperand> CallResult =
+    TLI.LowerCallTo(Chain, Type::VoidTy,
+                    false, false, false, CallingConv::C, false,
+                    getExternalSymbol("memcpy", TLI.getPointerTy()),
+                    Args, *this);
+  return CallResult.second;
+}
+
+SDOperand SelectionDAG::getMemmove(SDOperand Chain, SDOperand Dst,
+                                   SDOperand Src, SDOperand Size,
+                                   unsigned Align,
+                                   const Value *DstSV, uint64_t DstSVOff,
+                                   const Value *SrcSV, uint64_t SrcSVOff) {
+
+  // Check to see if we should lower the memmove to loads and stores first.
+  // For cases within the target-specified limits, this is the best choice.
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (ConstantSize) {
+    // Memmove with size zero? Just return the original chain.
+    if (ConstantSize->isNullValue())
+      return Chain;
+
+    SDOperand Result =
+      getMemmoveLoadsAndStores(*this, Chain, Dst, Src, ConstantSize->getValue(),
+                               Align, false, DstSV, DstSVOff, SrcSV, SrcSVOff);
+    if (Result.Val)
+      return Result;
+  }
+
+  // Then check to see if we should lower the memmove with target-specific
+  // code. If the target chooses to do this, this is the next best.
+  SDOperand Result =
+    TLI.EmitTargetCodeForMemmove(*this, Chain, Dst, Src, Size, Align,
+                                 DstSV, DstSVOff, SrcSV, SrcSVOff);
+  if (Result.Val)
+    return Result;
+
+  // Emit a library call.
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Ty = TLI.getTargetData()->getIntPtrType();
+  Entry.Node = Dst; Args.push_back(Entry);
+  Entry.Node = Src; Args.push_back(Entry);
+  Entry.Node = Size; Args.push_back(Entry);
+  std::pair<SDOperand,SDOperand> CallResult =
+    TLI.LowerCallTo(Chain, Type::VoidTy,
+                    false, false, false, CallingConv::C, false,
+                    getExternalSymbol("memmove", TLI.getPointerTy()),
+                    Args, *this);
+  return CallResult.second;
+}
+
+SDOperand SelectionDAG::getMemset(SDOperand Chain, SDOperand Dst,
+                                  SDOperand Src, SDOperand Size,
+                                  unsigned Align,
+                                  const Value *DstSV, uint64_t DstSVOff) {
+
+  // Check to see if we should lower the memset to stores first.
+  // For cases within the target-specified limits, this is the best choice.
+  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
+  if (ConstantSize) {
+    // Memset with size zero? Just return the original chain.
+    if (ConstantSize->isNullValue())
+      return Chain;
+
+    SDOperand Result =
+      getMemsetStores(*this, Chain, Dst, Src, ConstantSize->getValue(), Align,
+                      DstSV, DstSVOff);
+    if (Result.Val)
+      return Result;
+  }
+
+  // Then check to see if we should lower the memset with target-specific
+  // code. If the target chooses to do this, this is the next best.
+  SDOperand Result =
+    TLI.EmitTargetCodeForMemset(*this, Chain, Dst, Src, Size, Align,
+                                DstSV, DstSVOff);
+  if (Result.Val)
+    return Result;
+
+  // Emit a library call.
+  const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType();
+  TargetLowering::ArgListTy Args;
+  TargetLowering::ArgListEntry Entry;
+  Entry.Node = Dst; Entry.Ty = IntPtrTy;
+  Args.push_back(Entry);
+  // Extend or truncate the argument to be an i32 value for the call.
+  if (Src.getValueType().bitsGT(MVT::i32))
+    Src = getNode(ISD::TRUNCATE, MVT::i32, Src);
+  else
+    Src = getNode(ISD::ZERO_EXTEND, MVT::i32, Src);
+  Entry.Node = Src; Entry.Ty = Type::Int32Ty; Entry.isSExt = true;
+  Args.push_back(Entry);
+  Entry.Node = Size; Entry.Ty = IntPtrTy; Entry.isSExt = false;
+  Args.push_back(Entry);
+  std::pair<SDOperand,SDOperand> CallResult =
+    TLI.LowerCallTo(Chain, Type::VoidTy,
+                    false, false, false, CallingConv::C, false,
+                    getExternalSymbol("memset", TLI.getPointerTy()),
+                    Args, *this);
+  return CallResult.second;
+}
+
+SDOperand SelectionDAG::getAtomic(unsigned Opcode, SDOperand Chain, 
+                                  SDOperand Ptr, SDOperand Cmp, 
+                                  SDOperand Swp, MVT VT) {
+  assert(Opcode == ISD::ATOMIC_LCS && "Invalid Atomic Op");
+  assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
+  SDVTList VTs = getVTList(Cmp.getValueType(), MVT::Other);
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
-  ID.AddInteger(ISD::UNINDEXED);
-  ID.AddInteger(ISD::NON_EXTLOAD);
-  ID.AddInteger((unsigned int)VT);
-  ID.AddInteger(Alignment);
-  ID.AddInteger(isVolatile);
-  void *IP = 0;
+  SDOperand Ops[] = {Chain, Ptr, Cmp, Swp};
+  AddNodeIDNode(ID, Opcode, VTs, Ops, 4);
+  ID.AddInteger(VT.getRawBits());
+  void* IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDOperand(E, 0);
-  SDNode *N = new LoadSDNode(Ops, VTs, ISD::UNINDEXED,
-                             ISD::NON_EXTLOAD, VT, SV, SVOffset, Alignment,
-                             isVolatile);
+  SDNode* N = new AtomicSDNode(Opcode, VTs, Chain, Ptr, Cmp, Swp, VT);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT::ValueType VT,
-                                   SDOperand Chain, SDOperand Ptr,
-                                   const Value *SV,
-                                   int SVOffset, MVT::ValueType EVT,
-                                   bool isVolatile, unsigned Alignment) {
-  // If they are asking for an extending load from/to the same thing, return a
-  // normal load.
-  if (VT == EVT)
-    return getLoad(VT, Chain, Ptr, SV, SVOffset, isVolatile, Alignment);
-
-  if (MVT::isVector(VT))
-    assert(EVT == MVT::getVectorElementType(VT) && "Invalid vector extload!");
-  else
-    assert(MVT::getSizeInBits(EVT) < MVT::getSizeInBits(VT) &&
-           "Should only be an extending load, not truncating!");
-  assert((ExtType == ISD::EXTLOAD || MVT::isInteger(VT)) &&
-         "Cannot sign/zero extend a FP/Vector load!");
-  assert(MVT::isInteger(VT) == MVT::isInteger(EVT) &&
-         "Cannot convert from FP to Int or Int -> FP!");
+SDOperand SelectionDAG::getAtomic(unsigned Opcode, SDOperand Chain, 
+                                  SDOperand Ptr, SDOperand Val, 
+                                  MVT VT) {
+  assert((   Opcode == ISD::ATOMIC_LAS || Opcode == ISD::ATOMIC_LSS
+          || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_LOAD_AND
+          || Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR
+          || Opcode == ISD::ATOMIC_LOAD_NAND 
+          || Opcode == ISD::ATOMIC_LOAD_MIN || Opcode == ISD::ATOMIC_LOAD_MAX
+          || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX) 
+         && "Invalid Atomic Op");
+  SDVTList VTs = getVTList(Val.getValueType(), MVT::Other);
+  FoldingSetNodeID ID;
+  SDOperand Ops[] = {Chain, Ptr, Val};
+  AddNodeIDNode(ID, Opcode, VTs, Ops, 3);
+  ID.AddInteger(VT.getRawBits());
+  void* IP = 0;
+  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+    return SDOperand(E, 0);
+  SDNode* N = new AtomicSDNode(Opcode, VTs, Chain, Ptr, Val, VT);
+  CSEMap.InsertNode(N, IP);
+  AllNodes.push_back(N);
+  return SDOperand(N, 0);
+}
 
+SDOperand
+SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
+                      MVT VT, SDOperand Chain,
+                      SDOperand Ptr, SDOperand Offset,
+                      const Value *SV, int SVOffset, MVT EVT,
+                      bool isVolatile, unsigned Alignment) {
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
     if (VT != MVT::iPTR) {
-      Ty = MVT::getTypeForValueType(VT);
+      Ty = VT.getTypeForMVT();
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
       assert(PT && "Value for load must be a pointer");
       Ty = PT->getElementType();
-    }  
+    }
     assert(Ty && "Could not get type information for load");
     Alignment = TLI.getTargetData()->getABITypeAlignment(Ty);
   }
-  SDVTList VTs = getVTList(VT, MVT::Other);
-  SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
-  SDOperand Ops[] = { Chain, Ptr, Undef };
+
+  if (VT == EVT) {
+    ExtType = ISD::NON_EXTLOAD;
+  } else if (ExtType == ISD::NON_EXTLOAD) {
+    assert(VT == EVT && "Non-extending load from different memory type!");
+  } else {
+    // Extending load.
+    if (VT.isVector())
+      assert(EVT == VT.getVectorElementType() && "Invalid vector extload!");
+    else
+      assert(EVT.bitsLT(VT) &&
+             "Should only be an extending load, not truncating!");
+    assert((ExtType == ISD::EXTLOAD || VT.isInteger()) &&
+           "Cannot sign/zero extend a FP/Vector load!");
+    assert(VT.isInteger() == EVT.isInteger() &&
+           "Cannot convert from FP to Int or Int -> FP!");
+  }
+
+  bool Indexed = AM != ISD::UNINDEXED;
+  assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
+         "Unindexed load with an offset!");
+
+  SDVTList VTs = Indexed ?
+    getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
+  SDOperand Ops[] = { Chain, Ptr, Offset };
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
-  ID.AddInteger(ISD::UNINDEXED);
+  ID.AddInteger(AM);
   ID.AddInteger(ExtType);
-  ID.AddInteger((unsigned int)EVT);
+  ID.AddInteger(EVT.getRawBits());
   ID.AddInteger(Alignment);
   ID.AddInteger(isVolatile);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDOperand(E, 0);
-  SDNode *N = new LoadSDNode(Ops, VTs, ISD::UNINDEXED, ExtType, EVT,
-                             SV, SVOffset, Alignment, isVolatile);
+  SDNode *N = new LoadSDNode(Ops, VTs, AM, ExtType, EVT, SV, SVOffset,
+                             Alignment, isVolatile);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDOperand(N, 0);
 }
 
+SDOperand SelectionDAG::getLoad(MVT VT,
+                                SDOperand Chain, SDOperand Ptr,
+                                const Value *SV, int SVOffset,
+                                bool isVolatile, unsigned Alignment) {
+  SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
+  return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, Chain, Ptr, Undef,
+                 SV, SVOffset, VT, isVolatile, Alignment);
+}
+
+SDOperand SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, MVT VT,
+                                   SDOperand Chain, SDOperand Ptr,
+                                   const Value *SV,
+                                   int SVOffset, MVT EVT,
+                                   bool isVolatile, unsigned Alignment) {
+  SDOperand Undef = getNode(ISD::UNDEF, Ptr.getValueType());
+  return getLoad(ISD::UNINDEXED, ExtType, VT, Chain, Ptr, Undef,
+                 SV, SVOffset, EVT, isVolatile, Alignment);
+}
+
 SDOperand
 SelectionDAG::getIndexedLoad(SDOperand OrigLoad, SDOperand Base,
                              SDOperand Offset, ISD::MemIndexedMode AM) {
   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
          "Load is already a indexed load!");
-  MVT::ValueType VT = OrigLoad.getValueType();
-  SDVTList VTs = getVTList(VT, Base.getValueType(), MVT::Other);
-  SDOperand Ops[] = { LD->getChain(), Base, Offset };
-  FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
-  ID.AddInteger(AM);
-  ID.AddInteger(LD->getExtensionType());
-  ID.AddInteger((unsigned int)(LD->getMemoryVT()));
-  ID.AddInteger(LD->getAlignment());
-  ID.AddInteger(LD->isVolatile());
-  void *IP = 0;
-  if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
-    return SDOperand(E, 0);
-  SDNode *N = new LoadSDNode(Ops, VTs, AM,
-                             LD->getExtensionType(), LD->getMemoryVT(),
-                             LD->getSrcValue(), LD->getSrcValueOffset(),
-                             LD->getAlignment(), LD->isVolatile());
-  CSEMap.InsertNode(N, IP);
-  AllNodes.push_back(N);
-  return SDOperand(N, 0);
+  return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(),
+                 LD->getChain(), Base, Offset, LD->getSrcValue(),
+                 LD->getSrcValueOffset(), LD->getMemoryVT(),
+                 LD->isVolatile(), LD->getAlignment());
 }
 
 SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val,
                                  SDOperand Ptr, const Value *SV, int SVOffset,
                                  bool isVolatile, unsigned Alignment) {
-  MVT::ValueType VT = Val.getValueType();
+  MVT VT = Val.getValueType();
 
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
     if (VT != MVT::iPTR) {
-      Ty = MVT::getTypeForValueType(VT);
+      Ty = VT.getTypeForMVT();
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
       assert(PT && "Value for store must be a pointer");
@@ -2564,7 +3128,7 @@ SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val,
   AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
   ID.AddInteger(ISD::UNINDEXED);
   ID.AddInteger(false);
-  ID.AddInteger((unsigned int)VT);
+  ID.AddInteger(VT.getRawBits());
   ID.AddInteger(Alignment);
   ID.AddInteger(isVolatile);
   void *IP = 0;
@@ -2579,22 +3143,21 @@ SDOperand SelectionDAG::getStore(SDOperand Chain, SDOperand Val,
 
 SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val,
                                       SDOperand Ptr, const Value *SV,
-                                      int SVOffset, MVT::ValueType SVT,
+                                      int SVOffset, MVT SVT,
                                       bool isVolatile, unsigned Alignment) {
-  MVT::ValueType VT = Val.getValueType();
+  MVT VT = Val.getValueType();
 
   if (VT == SVT)
     return getStore(Chain, Val, Ptr, SV, SVOffset, isVolatile, Alignment);
 
-  assert(MVT::getSizeInBits(VT) > MVT::getSizeInBits(SVT) &&
-         "Not a truncation?");
-  assert(MVT::isInteger(VT) == MVT::isInteger(SVT) &&
+  assert(VT.bitsGT(SVT) && "Not a truncation?");
+  assert(VT.isInteger() == SVT.isInteger() &&
          "Can't do FP-INT conversion!");
 
   if (Alignment == 0) { // Ensure that codegen never sees alignment 0
     const Type *Ty = 0;
     if (VT != MVT::iPTR) {
-      Ty = MVT::getTypeForValueType(VT);
+      Ty = VT.getTypeForMVT();
     } else if (SV) {
       const PointerType *PT = dyn_cast<PointerType>(SV->getType());
       assert(PT && "Value for store must be a pointer");
@@ -2610,7 +3173,7 @@ SDOperand SelectionDAG::getTruncStore(SDOperand Chain, SDOperand Val,
   AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
   ID.AddInteger(ISD::UNINDEXED);
   ID.AddInteger(1);
-  ID.AddInteger((unsigned int)SVT);
+  ID.AddInteger(SVT.getRawBits());
   ID.AddInteger(Alignment);
   ID.AddInteger(isVolatile);
   void *IP = 0;
@@ -2635,7 +3198,7 @@ SelectionDAG::getIndexedStore(SDOperand OrigStore, SDOperand Base,
   AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4);
   ID.AddInteger(AM);
   ID.AddInteger(ST->isTruncatingStore());
-  ID.AddInteger((unsigned int)(ST->getMemoryVT()));
+  ID.AddInteger(ST->getMemoryVT().getRawBits());
   ID.AddInteger(ST->getAlignment());
   ID.AddInteger(ST->isVolatile());
   void *IP = 0;
@@ -2650,15 +3213,15 @@ SelectionDAG::getIndexedStore(SDOperand OrigStore, SDOperand Base,
   return SDOperand(N, 0);
 }
 
-SDOperand SelectionDAG::getVAArg(MVT::ValueType VT,
+SDOperand SelectionDAG::getVAArg(MVT VT,
                                  SDOperand Chain, SDOperand Ptr,
                                  SDOperand SV) {
   SDOperand Ops[] = { Chain, Ptr, SV };
   return getNode(ISD::VAARG, getVTList(VT, MVT::Other), Ops, 3);
 }
 
-SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
-                                const SDOperand *Ops, unsigned NumOps) {
+SDOperand SelectionDAG::getNode(unsigned Opcode, MVT VT,
+                                SDOperandPtr Ops, unsigned NumOps) {
   switch (NumOps) {
   case 0: return getNode(Opcode, VT);
   case 1: return getNode(Opcode, VT, Ops[0]);
@@ -2706,22 +3269,22 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
 }
 
 SDOperand SelectionDAG::getNode(unsigned Opcode,
-                                std::vector<MVT::ValueType> &ResultTys,
-                                const SDOperand *Ops, unsigned NumOps) {
+                                std::vector<MVT> &ResultTys,
+                                SDOperandPtr Ops, unsigned NumOps) {
   return getNode(Opcode, getNodeValueTypes(ResultTys), ResultTys.size(),
                  Ops, NumOps);
 }
 
 SDOperand SelectionDAG::getNode(unsigned Opcode,
-                                const MVT::ValueType *VTs, unsigned NumVTs,
-                                const SDOperand *Ops, unsigned NumOps) {
+                                const MVT *VTs, unsigned NumVTs,
+                                SDOperandPtr Ops, unsigned NumOps) {
   if (NumVTs == 1)
     return getNode(Opcode, VTs[0], Ops, NumOps);
   return getNode(Opcode, makeVTList(VTs, NumVTs), Ops, NumOps);
 }  
   
 SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
-                                const SDOperand *Ops, unsigned NumOps) {
+                                SDOperandPtr Ops, unsigned NumOps) {
   if (VTList.NumVTs == 1)
     return getNode(Opcode, VTList.VTs[0], Ops, NumOps);
 
@@ -2740,7 +3303,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
         // If the and is only masking out bits that cannot effect the shift,
         // eliminate the and.
-        unsigned NumBits = MVT::getSizeInBits(VT)*2;
+        unsigned NumBits = VT.getSizeInBits()*2;
         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
           return getNode(Opcode, VT, N1, N2, N3.getOperand(0));
       }
@@ -2780,7 +3343,7 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
 }
 
 SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList) {
-  return getNode(Opcode, VTList, 0, 0);
+  return getNode(Opcode, VTList, (SDOperand*)0, 0);
 }
 
 SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
@@ -2815,31 +3378,31 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, SDVTList VTList,
   return getNode(Opcode, VTList, Ops, 5);
 }
 
-SDVTList SelectionDAG::getVTList(MVT::ValueType VT) {
+SDVTList SelectionDAG::getVTList(MVT VT) {
   return makeVTList(SDNode::getValueTypeList(VT), 1);
 }
 
-SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2) {
-  for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2) {
+  for (std::list<std::vector<MVT> >::iterator I = VTList.begin(),
        E = VTList.end(); I != E; ++I) {
     if (I->size() == 2 && (*I)[0] == VT1 && (*I)[1] == VT2)
       return makeVTList(&(*I)[0], 2);
   }
-  std::vector<MVT::ValueType> V;
+  std::vector<MVT> V;
   V.push_back(VT1);
   V.push_back(VT2);
   VTList.push_front(V);
   return makeVTList(&(*VTList.begin())[0], 2);
 }
-SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2,
-                                 MVT::ValueType VT3) {
-  for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+SDVTList SelectionDAG::getVTList(MVT VT1, MVT VT2,
+                                 MVT VT3) {
+  for (std::list<std::vector<MVT> >::iterator I = VTList.begin(),
        E = VTList.end(); I != E; ++I) {
     if (I->size() == 3 && (*I)[0] == VT1 && (*I)[1] == VT2 &&
         (*I)[2] == VT3)
       return makeVTList(&(*I)[0], 3);
   }
-  std::vector<MVT::ValueType> V;
+  std::vector<MVT> V;
   V.push_back(VT1);
   V.push_back(VT2);
   V.push_back(VT3);
@@ -2847,7 +3410,7 @@ SDVTList SelectionDAG::getVTList(MVT::ValueType VT1, MVT::ValueType VT2,
   return makeVTList(&(*VTList.begin())[0], 3);
 }
 
-SDVTList SelectionDAG::getVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
+SDVTList SelectionDAG::getVTList(const MVT *VTs, unsigned NumVTs) {
   switch (NumVTs) {
     case 0: assert(0 && "Cannot have nodes without results!");
     case 1: return getVTList(VTs[0]);
@@ -2856,7 +3419,7 @@ SDVTList SelectionDAG::getVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
     default: break;
   }
 
-  for (std::list<std::vector<MVT::ValueType> >::iterator I = VTList.begin(),
+  for (std::list<std::vector<MVT> >::iterator I = VTList.begin(),
        E = VTList.end(); I != E; ++I) {
     if (I->size() != NumVTs || VTs[0] != (*I)[0] || VTs[1] != (*I)[1]) continue;
    
@@ -2870,7 +3433,7 @@ SDVTList SelectionDAG::getVTList(const MVT::ValueType *VTs, unsigned NumVTs) {
       return makeVTList(&*I->begin(), NumVTs);
   }
   
-  VTList.push_front(std::vector<MVT::ValueType>(VTs, VTs+NumVTs));
+  VTList.push_front(std::vector<MVT>(VTs, VTs+NumVTs));
   return makeVTList(&*VTList.begin()->begin(), NumVTs);
 }
 
@@ -2899,9 +3462,10 @@ UpdateNodeOperands(SDOperand InN, SDOperand Op) {
     RemoveNodeFromCSEMaps(N);
   
   // Now we update the operands.
-  N->OperandList[0].Val->removeUser(N);
-  Op.Val->addUser(N);
+  N->OperandList[0].getVal()->removeUser(0, N);
   N->OperandList[0] = Op;
+  N->OperandList[0].setUser(N);
+  Op.Val->addUser(0, N);
   
   // If this gets put into a CSE map, add it.
   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
@@ -2928,14 +3492,16 @@ UpdateNodeOperands(SDOperand InN, SDOperand Op1, SDOperand Op2) {
   
   // Now we update the operands.
   if (N->OperandList[0] != Op1) {
-    N->OperandList[0].Val->removeUser(N);
-    Op1.Val->addUser(N);
+    N->OperandList[0].getVal()->removeUser(0, N);
     N->OperandList[0] = Op1;
+    N->OperandList[0].setUser(N);
+    Op1.Val->addUser(0, N);
   }
   if (N->OperandList[1] != Op2) {
-    N->OperandList[1].Val->removeUser(N);
-    Op2.Val->addUser(N);
+    N->OperandList[1].getVal()->removeUser(1, N);
     N->OperandList[1] = Op2;
+    N->OperandList[1].setUser(N);
+    Op2.Val->addUser(1, N);
   }
   
   // If this gets put into a CSE map, add it.
@@ -2963,9 +3529,8 @@ UpdateNodeOperands(SDOperand N, SDOperand Op1, SDOperand Op2,
   return UpdateNodeOperands(N, Ops, 5);
 }
 
-
 SDOperand SelectionDAG::
-UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) {
+UpdateNodeOperands(SDOperand InN, SDOperandPtr Ops, unsigned NumOps) {
   SDNode *N = InN.Val;
   assert(N->getNumOperands() == NumOps &&
          "Update with wrong number of operands");
@@ -2987,16 +3552,17 @@ UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) {
   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos))
     return SDOperand(Existing, InN.ResNo);
   
-  // Nope it doesn't.  Remove the node from it's current place in the maps.
+  // Nope it doesn't.  Remove the node from its current place in the maps.
   if (InsertPos)
     RemoveNodeFromCSEMaps(N);
   
   // Now we update the operands.
   for (unsigned i = 0; i != NumOps; ++i) {
     if (N->OperandList[i] != Ops[i]) {
-      N->OperandList[i].Val->removeUser(N);
-      Ops[i].Val->addUser(N);
+      N->OperandList[i].getVal()->removeUser(i, N);
       N->OperandList[i] = Ops[i];
+      N->OperandList[i].setUser(N);
+      Ops[i].Val->addUser(i, N);
     }
   }
 
@@ -3005,12 +3571,11 @@ UpdateNodeOperands(SDOperand InN, SDOperand *Ops, unsigned NumOps) {
   return InN;
 }
 
-
 /// MorphNodeTo - This frees the operands of the current node, resets the
 /// opcode, types, and operands to the specified value.  This should only be
 /// used by the SelectionDAG class.
 void SDNode::MorphNodeTo(unsigned Opc, SDVTList L,
-                         const SDOperand *Ops, unsigned NumOps) {
+                         SDOperandPtr Ops, unsigned NumOps) {
   NodeType = Opc;
   ValueList = L.VTs;
   NumValues = L.NumVTs;
@@ -3018,14 +3583,15 @@ void SDNode::MorphNodeTo(unsigned Opc, SDVTList L,
   // Clear the operands list, updating used nodes to remove this from their
   // use list.
   for (op_iterator I = op_begin(), E = op_end(); I != E; ++I)
-    I->Val->removeUser(this);
+    I->getVal()->removeUser(std::distance(op_begin(), I), this);
   
   // If NumOps is larger than the # of operands we currently have, reallocate
   // the operand list.
   if (NumOps > NumOperands) {
-    if (OperandsNeedDelete)
+    if (OperandsNeedDelete) {
       delete [] OperandList;
-    OperandList = new SDOperand[NumOps];
+    }
+    OperandList = new SDUse[NumOps];
     OperandsNeedDelete = true;
   }
   
@@ -3034,8 +3600,10 @@ void SDNode::MorphNodeTo(unsigned Opc, SDVTList L,
   
   for (unsigned i = 0, e = NumOps; i != e; ++i) {
     OperandList[i] = Ops[i];
-    SDNode *N = OperandList[i].Val;
-    N->Uses.push_back(this);
+    OperandList[i].setUser(this);
+    SDNode *N = OperandList[i].getVal();
+    N->addUser(i, this);
+    ++N->UsesSize;
   }
 }
 
@@ -3048,24 +3616,24 @@ void SDNode::MorphNodeTo(unsigned Opc, SDVTList L,
 /// node of the specified opcode and operands, it returns that node instead of
 /// the current one.
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
-                                   MVT::ValueType VT) {
+                                   MVT VT) {
   SDVTList VTs = getVTList(VT);
   FoldingSetNodeID ID;
-  AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, 0, 0);
+  AddNodeIDNode(ID, ISD::BUILTIN_OP_END+TargetOpc, VTs, (SDOperand*)0, 0);
   void *IP = 0;
   if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
     return ON;
    
   RemoveNodeFromCSEMaps(N);
   
-  N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, 0, 0);
+  N->MorphNodeTo(ISD::BUILTIN_OP_END+TargetOpc, VTs, SDOperandPtr(), 0);
 
   CSEMap.InsertNode(N, IP);
   return N;
 }
 
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
-                                   MVT::ValueType VT, SDOperand Op1) {
+                                   MVT VT, SDOperand Op1) {
   // If an identical node already exists, use it.
   SDVTList VTs = getVTList(VT);
   SDOperand Ops[] = { Op1 };
@@ -3083,7 +3651,7 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
 }
 
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
-                                   MVT::ValueType VT, SDOperand Op1,
+                                   MVT VT, SDOperand Op1,
                                    SDOperand Op2) {
   // If an identical node already exists, use it.
   SDVTList VTs = getVTList(VT);
@@ -3104,7 +3672,7 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
 }
 
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
-                                   MVT::ValueType VT, SDOperand Op1,
+                                   MVT VT, SDOperand Op1,
                                    SDOperand Op2, SDOperand Op3) {
   // If an identical node already exists, use it.
   SDVTList VTs = getVTList(VT);
@@ -3124,7 +3692,7 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
 }
 
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
-                                   MVT::ValueType VT, const SDOperand *Ops,
+                                   MVT VT, SDOperandPtr Ops,
                                    unsigned NumOps) {
   // If an identical node already exists, use it.
   SDVTList VTs = getVTList(VT);
@@ -3142,7 +3710,7 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
 }
 
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc, 
-                                   MVT::ValueType VT1, MVT::ValueType VT2,
+                                   MVT VT1, MVT VT2,
                                    SDOperand Op1, SDOperand Op2) {
   SDVTList VTs = getVTList(VT1, VT2);
   FoldingSetNodeID ID;
@@ -3159,7 +3727,7 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
 }
 
 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
-                                   MVT::ValueType VT1, MVT::ValueType VT2,
+                                   MVT VT1, MVT VT2,
                                    SDOperand Op1, SDOperand Op2, 
                                    SDOperand Op3) {
   // If an identical node already exists, use it.
@@ -3185,98 +3753,105 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned TargetOpc,
 /// Note that getTargetNode returns the resultant node.  If there is already a
 /// node of the specified opcode and operands, it returns that node instead of
 /// the current one.
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT) {
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT) {
   return getNode(ISD::BUILTIN_OP_END+Opcode, VT).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
-                                    SDOperand Op1) {
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT, SDOperand Op1) {
   return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
                                     SDOperand Op1, SDOperand Op2) {
   return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1, Op2).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
                                     SDOperand Op1, SDOperand Op2,
                                     SDOperand Op3) {
   return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Op1, Op2, Op3).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT,
-                                    const SDOperand *Ops, unsigned NumOps) {
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT,
+                                    SDOperandPtr Ops, unsigned NumOps) {
   return getNode(ISD::BUILTIN_OP_END+Opcode, VT, Ops, NumOps).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
-                                    MVT::ValueType VT2) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2) {
+  const MVT *VTs = getNodeValueTypes(VT1, VT2);
   SDOperand Op;
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, &Op, 0).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
-                                    MVT::ValueType VT2, SDOperand Op1) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
+                                    MVT VT2, SDOperand Op1) {
+  const MVT *VTs = getNodeValueTypes(VT1, VT2);
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, &Op1, 1).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
-                                    MVT::ValueType VT2, SDOperand Op1,
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
+                                    MVT VT2, SDOperand Op1,
                                     SDOperand Op2) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+  const MVT *VTs = getNodeValueTypes(VT1, VT2);
   SDOperand Ops[] = { Op1, Op2 };
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, 2).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
-                                    MVT::ValueType VT2, SDOperand Op1,
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
+                                    MVT VT2, SDOperand Op1,
                                     SDOperand Op2, SDOperand Op3) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+  const MVT *VTs = getNodeValueTypes(VT1, VT2);
   SDOperand Ops[] = { Op1, Op2, Op3 };
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, 3).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1, 
-                                    MVT::ValueType VT2,
-                                    const SDOperand *Ops, unsigned NumOps) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2);
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2,
+                                    SDOperandPtr Ops, unsigned NumOps) {
+  const MVT *VTs = getNodeValueTypes(VT1, VT2);
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 2, Ops, NumOps).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
-                                    MVT::ValueType VT2, MVT::ValueType VT3,
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
                                     SDOperand Op1, SDOperand Op2) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2, VT3);
+  const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
   SDOperand Ops[] = { Op1, Op2 };
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, 2).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1,
-                                    MVT::ValueType VT2, MVT::ValueType VT3,
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
                                     SDOperand Op1, SDOperand Op2,
                                     SDOperand Op3) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2, VT3);
+  const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
   SDOperand Ops[] = { Op1, Op2, Op3 };
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, 3).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1, 
-                                    MVT::ValueType VT2, MVT::ValueType VT3,
-                                    const SDOperand *Ops, unsigned NumOps) {
-  const MVT::ValueType *VTs = getNodeValueTypes(VT1, VT2, VT3);
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1, MVT VT2, MVT VT3,
+                                    SDOperandPtr Ops, unsigned NumOps) {
+  const MVT *VTs = getNodeValueTypes(VT1, VT2, VT3);
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 3, Ops, NumOps).Val;
 }
-SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT::ValueType VT1, 
-                                    MVT::ValueType VT2, MVT::ValueType VT3,
-                                    MVT::ValueType VT4,
-                                    const SDOperand *Ops, unsigned NumOps) {
-  std::vector<MVT::ValueType> VTList;
+SDNode *SelectionDAG::getTargetNode(unsigned Opcode, MVT VT1,
+                                    MVT VT2, MVT VT3, MVT VT4,
+                                    SDOperandPtr Ops, unsigned NumOps) {
+  std::vector<MVT> VTList;
   VTList.push_back(VT1);
   VTList.push_back(VT2);
   VTList.push_back(VT3);
   VTList.push_back(VT4);
-  const MVT::ValueType *VTs = getNodeValueTypes(VTList);
+  const MVT *VTs = getNodeValueTypes(VTList);
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, 4, Ops, NumOps).Val;
 }
 SDNode *SelectionDAG::getTargetNode(unsigned Opcode,
-                                    std::vector<MVT::ValueType> &ResultTys,
-                                    const SDOperand *Ops, unsigned NumOps) {
-  const MVT::ValueType *VTs = getNodeValueTypes(ResultTys);
+                                    std::vector<MVT> &ResultTys,
+                                    SDOperandPtr Ops, unsigned NumOps) {
+  const MVT *VTs = getNodeValueTypes(ResultTys);
   return getNode(ISD::BUILTIN_OP_END+Opcode, VTs, ResultTys.size(),
                  Ops, NumOps).Val;
 }
 
+/// getNodeIfExists - Get the specified node if it's already available, or
+/// else return NULL.
+SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
+                                      SDOperandPtr Ops, unsigned NumOps) {
+  if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) {
+    FoldingSetNodeID ID;
+    AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps);
+    void *IP = 0;
+    if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
+      return E;
+  }
+  return NULL;
+}
+
 
 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
 /// This can cause recursive merging of nodes in the DAG.
@@ -3289,21 +3864,22 @@ void SelectionDAG::ReplaceAllUsesWith(SDOperand FromN, SDOperand To,
   assert(From->getNumValues() == 1 && FromN.ResNo == 0 && 
          "Cannot replace with this method!");
   assert(From != To.Val && "Cannot replace uses of with self");
-  
+
   while (!From->use_empty()) {
-    // Process users until they are all gone.
-    SDNode *U = *From->use_begin();
-    
+    SDNode::use_iterator UI = From->use_begin();
+    SDNode *U = UI->getUser();
+
     // This node is about to morph, remove its old self from the CSE maps.
     RemoveNodeFromCSEMaps(U);
-    
-    for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands;
-         I != E; ++I)
-      if (I->Val == From) {
-        From->removeUser(U);
+    int operandNum = 0;
+    for (SDNode::op_iterator I = U->op_begin(), E = U->op_end();
+         I != E; ++I, ++operandNum)
+      if (I->getVal() == From) {
+        From->removeUser(operandNum, U);
         *I = To;
-        To.Val->addUser(U);
-      }
+        I->setUser(U);
+        To.Val->addUser(operandNum, U);
+      }    
 
     // Now that we have modified U, add it back to the CSE maps.  If it already
     // exists there, recursively merge the results together.
@@ -3311,7 +3887,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDOperand FromN, SDOperand To,
       ReplaceAllUsesWith(U, Existing, UpdateListener);
       // U is now dead.  Inform the listener if it exists and delete it.
       if (UpdateListener) 
-        UpdateListener->NodeDeleted(U);
+        UpdateListener->NodeDeleted(U, Existing);
       DeleteNodeNotInCSEMaps(U);
     } else {
       // If the node doesn't already exist, we updated it.  Inform a listener if
@@ -3338,27 +3914,27 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To,
                               UpdateListener);
   
   while (!From->use_empty()) {
-    // Process users until they are all gone.
-    SDNode *U = *From->use_begin();
-    
+    SDNode::use_iterator UI = From->use_begin();
+    SDNode *U = UI->getUser();
+
     // This node is about to morph, remove its old self from the CSE maps.
     RemoveNodeFromCSEMaps(U);
-    
-    for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands;
-         I != E; ++I)
-      if (I->Val == From) {
-        From->removeUser(U);
-        I->Val = To;
-        To->addUser(U);
+    int operandNum = 0;
+    for (SDNode::op_iterator I = U->op_begin(), E = U->op_end();
+         I != E; ++I, ++operandNum)
+      if (I->getVal() == From) {
+        From->removeUser(operandNum, U);
+        I->getVal() = To;
+        To->addUser(operandNum, U);
       }
-        
+
     // Now that we have modified U, add it back to the CSE maps.  If it already
     // exists there, recursively merge the results together.
     if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) {
       ReplaceAllUsesWith(U, Existing, UpdateListener);
       // U is now dead.  Inform the listener if it exists and delete it.
       if (UpdateListener) 
-        UpdateListener->NodeDeleted(U);
+        UpdateListener->NodeDeleted(U, Existing);
       DeleteNodeNotInCSEMaps(U);
     } else {
       // If the node doesn't already exist, we updated it.  Inform a listener if
@@ -3375,34 +3951,35 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To,
 /// This version can replace From with any result values.  To must match the
 /// number and types of values returned by From.
 void SelectionDAG::ReplaceAllUsesWith(SDNode *From,
-                                      const SDOperand *To,
+                                      SDOperandPtr To,
                                       DAGUpdateListener *UpdateListener) {
   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
     return ReplaceAllUsesWith(SDOperand(From, 0), To[0], UpdateListener);
 
   while (!From->use_empty()) {
-    // Process users until they are all gone.
-    SDNode *U = *From->use_begin();
-    
+    SDNode::use_iterator UI = From->use_begin();
+    SDNode *U = UI->getUser();
+
     // This node is about to morph, remove its old self from the CSE maps.
     RemoveNodeFromCSEMaps(U);
-    
-    for (SDOperand *I = U->OperandList, *E = U->OperandList+U->NumOperands;
-         I != E; ++I)
-      if (I->Val == From) {
-        const SDOperand &ToOp = To[I->ResNo];
-        From->removeUser(U);
+    int operandNum = 0;
+    for (SDNode::op_iterator I = U->op_begin(), E = U->op_end();
+         I != E; ++I, ++operandNum)
+      if (I->getVal() == From) {
+        const SDOperand &ToOp = To[I->getSDOperand().ResNo];
+        From->removeUser(operandNum, U);
         *I = ToOp;
-        ToOp.Val->addUser(U);
+        I->setUser(U);
+        ToOp.Val->addUser(operandNum, U);
       }
-        
+
     // Now that we have modified U, add it back to the CSE maps.  If it already
     // exists there, recursively merge the results together.
     if (SDNode *Existing = AddNonLeafNodeToCSEMaps(U)) {
       ReplaceAllUsesWith(U, Existing, UpdateListener);
       // U is now dead.  Inform the listener if it exists and delete it.
       if (UpdateListener) 
-        UpdateListener->NodeDeleted(U);
+        UpdateListener->NodeDeleted(U, Existing);
       DeleteNodeNotInCSEMaps(U);
     } else {
       // If the node doesn't already exist, we updated it.  Inform a listener if
@@ -3425,10 +4002,10 @@ namespace {
     ChainedSetUpdaterListener(SmallSetVector<SDNode*, 16> &set,
                               SelectionDAG::DAGUpdateListener *chain)
       : Set(set), Chain(chain) {}
-    
-    virtual void NodeDeleted(SDNode *N) {
+ 
+    virtual void NodeDeleted(SDNode *N, SDNode *E) {
       Set.remove(N);
-      if (Chain) Chain->NodeDeleted(N);
+      if (Chain) Chain->NodeDeleted(N, E);
     }
     virtual void NodeUpdated(SDNode *N) {
       if (Chain) Chain->NodeUpdated(N);
@@ -3453,7 +4030,13 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
 
   // Get all of the users of From.Val.  We want these in a nice,
   // deterministically ordered and uniqued set, so we use a SmallSetVector.
-  SmallSetVector<SDNode*, 16> Users(From.Val->use_begin(), From.Val->use_end());
+  SmallSetVector<SDNode*, 16> Users;
+  for (SDNode::use_iterator UI = From.Val->use_begin(), 
+      E = From.Val->use_end(); UI != E; ++UI) {
+    SDNode *User = UI->getUser();
+    if (!Users.count(User))
+      Users.insert(User);
+  }
 
   // When one of the recursive merges deletes nodes from the graph, we need to
   // make sure that UpdateListener is notified *and* that the node is removed
@@ -3467,7 +4050,7 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
     Users.pop_back();
     
     // Scan for an operand that matches From.
-    SDOperand *Op = User->OperandList, *E = User->OperandList+User->NumOperands;
+    SDNode::op_iterator Op = User->op_begin(), E = User->op_end();
     for (; Op != E; ++Op)
       if (*Op == From) break;
     
@@ -3481,9 +4064,10 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
     // Update all operands that match "From" in case there are multiple uses.
     for (; Op != E; ++Op) {
       if (*Op == From) {
-        From.Val->removeUser(User);
+        From.Val->removeUser(Op-User->op_begin(), User);
         *Op = To;
-        To.Val->addUser(User);
+        Op->setUser(User);
+        To.Val->addUser(Op-User->op_begin(), User);
       }
     }
                
@@ -3503,12 +4087,11 @@ void SelectionDAG::ReplaceAllUsesOfValueWith(SDOperand From, SDOperand To,
     ReplaceAllUsesWith(User, Existing, &CSUL);
     
     // User is now dead.  Notify a listener if present.
-    if (UpdateListener) UpdateListener->NodeDeleted(User);
+    if (UpdateListener) UpdateListener->NodeDeleted(User, Existing);
     DeleteNodeNotInCSEMaps(User);
   }
 }
 
-
 /// AssignNodeIds - Assign a unique node id for each node in the DAG based on
 /// their allnodes order. It returns the maximum id.
 unsigned SelectionDAG::AssignNodeIds() {
@@ -3545,7 +4128,7 @@ unsigned SelectionDAG::AssignTopologicalOrder(std::vector<SDNode*> &TopOrder) {
     Sources.pop_back();
     TopOrder.push_back(N);
     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
-      SDNode *P = I->Val;
+      SDNode *P = I->getVal();
       unsigned Degree = --InDegree[P->getNodeId()];
       if (Degree == 0)
         Sources.push_back(P);
@@ -3586,17 +4169,19 @@ void MemOperandSDNode::ANCHOR() {}
 void RegisterSDNode::ANCHOR() {}
 void ExternalSymbolSDNode::ANCHOR() {}
 void CondCodeSDNode::ANCHOR() {}
+void ARG_FLAGSSDNode::ANCHOR() {}
 void VTSDNode::ANCHOR() {}
 void LoadSDNode::ANCHOR() {}
 void StoreSDNode::ANCHOR() {}
+void AtomicSDNode::ANCHOR() {}
 
 HandleSDNode::~HandleSDNode() {
   SDVTList VTs = { 0, 0 };
-  MorphNodeTo(ISD::HANDLENODE, VTs, 0, 0);  // Drops operand uses.
+  MorphNodeTo(ISD::HANDLENODE, VTs, SDOperandPtr(), 0);  // Drops operand uses.
 }
 
 GlobalAddressSDNode::GlobalAddressSDNode(bool isTarget, const GlobalValue *GA,
-                                         MVT::ValueType VT, int o)
+                                         MVT VT, int o)
   : SDNode(isa<GlobalVariable>(GA) &&
            cast<GlobalVariable>(GA)->isThreadLocal() ?
            // Thread Local
@@ -3607,24 +4192,25 @@ GlobalAddressSDNode::GlobalAddressSDNode(bool isTarget, const GlobalValue *GA,
   TheGlobal = const_cast<GlobalValue*>(GA);
 }
 
-/// getMemOperand - Return a MemOperand object describing the memory
+/// getMemOperand - Return a MachineMemOperand object describing the memory
 /// reference performed by this load or store.
-MemOperand LSBaseSDNode::getMemOperand() const {
-  int Size = (MVT::getSizeInBits(getMemoryVT()) + 7) >> 3;
+MachineMemOperand LSBaseSDNode::getMemOperand() const {
+  int Size = (getMemoryVT().getSizeInBits() + 7) >> 3;
   int Flags =
-    getOpcode() == ISD::LOAD ? MemOperand::MOLoad : MemOperand::MOStore;
-  if (IsVolatile) Flags |= MemOperand::MOVolatile;
+    getOpcode() == ISD::LOAD ? MachineMemOperand::MOLoad :
+                               MachineMemOperand::MOStore;
+  if (IsVolatile) Flags |= MachineMemOperand::MOVolatile;
 
   // Check if the load references a frame index, and does not have
   // an SV attached.
   const FrameIndexSDNode *FI =
     dyn_cast<const FrameIndexSDNode>(getBasePtr().Val);
   if (!getSrcValue() && FI)
-    return MemOperand(PseudoSourceValue::getFixedStack(), Flags,
-                      FI->getIndex(), Size, Alignment);
+    return MachineMemOperand(PseudoSourceValue::getFixedStack(), Flags,
+                             FI->getIndex(), Size, Alignment);
   else
-    return MemOperand(getSrcValue(), Flags,
-                      getSrcValueOffset(), Size, Alignment);
+    return MachineMemOperand(getSrcValue(), Flags,
+                             getSrcValueOffset(), Size, Alignment);
 }
 
 /// Profile - Gather unique data for the node.
@@ -3635,14 +4221,14 @@ void SDNode::Profile(FoldingSetNodeID &ID) {
 
 /// getValueTypeList - Return a pointer to the specified value type.
 ///
-const MVT::ValueType *SDNode::getValueTypeList(MVT::ValueType VT) {
-  if (MVT::isExtendedVT(VT)) {
-    static std::set<MVT::ValueType> EVTs;
+const MVT *SDNode::getValueTypeList(MVT VT) {
+  if (VT.isExtended()) {
+    static std::set<MVT, MVT::compareRawBits> EVTs;
     return &(*EVTs.insert(VT).first);
   } else {
-    static MVT::ValueType VTs[MVT::LAST_VALUETYPE];
-    VTs[VT] = VT;
-    return &VTs[VT];
+    static MVT VTs[MVT::LAST_VALUETYPE];
+    VTs[VT.getSimpleVT()] = VT;
+    return &VTs[VT.getSimpleVT()];
   }
 }
 
@@ -3661,16 +4247,13 @@ bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
 
   SmallPtrSet<SDNode*, 32> UsersHandled;
 
-  for (SDNode::use_iterator UI = Uses.begin(), E = Uses.end(); UI != E; ++UI) {
-    SDNode *User = *UI;
-    if (User->getNumOperands() == 1 ||
-        UsersHandled.insert(User))     // First time we've seen this?
-      for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
-        if (User->getOperand(i) == TheValue) {
-          if (NUses == 0)
-            return false;   // too many uses
-          --NUses;
-        }
+  // TODO: Only iterate over uses of a given value of the node
+  for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
+    if (*UI == TheValue) {
+      if (NUses == 0)
+        return false;
+      --NUses;
+    }
   }
 
   // Found exactly the right number of uses?
@@ -3689,8 +4272,8 @@ bool SDNode::hasAnyUseOfValue(unsigned Value) const {
 
   SmallPtrSet<SDNode*, 32> UsersHandled;
 
-  for (SDNode::use_iterator UI = Uses.begin(), E = Uses.end(); UI != E; ++UI) {
-    SDNode *User = *UI;
+  for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
+    SDNode *User = UI->getUser();
     if (User->getNumOperands() == 1 ||
         UsersHandled.insert(User))     // First time we've seen this?
       for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
@@ -3703,12 +4286,12 @@ bool SDNode::hasAnyUseOfValue(unsigned Value) const {
 }
 
 
-/// isOnlyUse - Return true if this node is the only use of N.
+/// isOnlyUseOf - Return true if this node is the only use of N.
 ///
-bool SDNode::isOnlyUse(SDNode *N) const {
+bool SDNode::isOnlyUseOf(SDNode *N) const {
   bool Seen = false;
   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
-    SDNode *User = *I;
+    SDNode *User = I->getUser();
     if (User == this)
       Seen = true;
     else
@@ -3720,16 +4303,16 @@ bool SDNode::isOnlyUse(SDNode *N) const {
 
 /// isOperand - Return true if this node is an operand of N.
 ///
-bool SDOperand::isOperand(SDNode *N) const {
+bool SDOperand::isOperandOf(SDNode *N) const {
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
     if (*this == N->getOperand(i))
       return true;
   return false;
 }
 
-bool SDNode::isOperand(SDNode *N) const {
+bool SDNode::isOperandOf(SDNode *N) const {
   for (unsigned i = 0, e = N->NumOperands; i != e; ++i)
-    if (this == N->OperandList[i].Val)
+    if (this == N->OperandList[i].getVal())
       return true;
   return false;
 }
@@ -3780,11 +4363,11 @@ static void findPredecessor(SDNode *N, const SDNode *P, bool &found,
   }
 }
 
-/// isPredecessor - Return true if this node is a predecessor of N. This node
+/// isPredecessorOf - Return true if this node is a predecessor of N. This node
 /// is either an operand of N or it can be reached by recursively traversing
 /// up the operands.
 /// NOTE: this is an expensive method. Use it carefully.
-bool SDNode::isPredecessor(SDNode *N) const {
+bool SDNode::isPredecessorOf(SDNode *N) const {
   SmallPtrSet<SDNode *, 32> Visited;
   bool found = false;
   findPredecessor(N, this, found, Visited);
@@ -3816,7 +4399,20 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
       return "<<Unknown Target Node>>";
     }
    
+  case ISD::PREFETCH:      return "Prefetch";
   case ISD::MEMBARRIER:    return "MemBarrier";
+  case ISD::ATOMIC_LCS:    return "AtomicLCS";
+  case ISD::ATOMIC_LAS:    return "AtomicLAS";
+  case ISD::ATOMIC_LSS:    return "AtomicLSS";
+  case ISD::ATOMIC_LOAD_AND:  return "AtomicLoadAnd";
+  case ISD::ATOMIC_LOAD_OR:   return "AtomicLoadOr";
+  case ISD::ATOMIC_LOAD_XOR:  return "AtomicLoadXor";
+  case ISD::ATOMIC_LOAD_NAND: return "AtomicLoadNand";
+  case ISD::ATOMIC_LOAD_MIN:  return "AtomicLoadMin";
+  case ISD::ATOMIC_LOAD_MAX:  return "AtomicLoadMax";
+  case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin";
+  case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax";
+  case ISD::ATOMIC_SWAP:   return "AtomicSWAP";
   case ISD::PCMARKER:      return "PCMarker";
   case ISD::READCYCLECOUNTER: return "ReadCycleCounter";
   case ISD::SRCVALUE:      return "SrcValue";
@@ -3828,6 +4424,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
 
   case ISD::STRING:        return "String";
   case ISD::BasicBlock:    return "BasicBlock";
+  case ISD::ARG_FLAGS:     return "ArgFlags";
   case ISD::VALUETYPE:     return "ValueType";
   case ISD::Register:      return "Register";
 
@@ -3917,6 +4514,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FGETSIGN:  return "fgetsign";
 
   case ISD::SETCC:       return "setcc";
+  case ISD::VSETCC:      return "vsetcc";
   case ISD::SELECT:      return "select";
   case ISD::SELECT_CC:   return "select_cc";
   case ISD::INSERT_VECTOR_ELT:   return "insert_vector_elt";
@@ -3978,11 +4576,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::STACKRESTORE:       return "stackrestore";
   case ISD::TRAP:               return "trap";
 
-  // Block memory operations.
-  case ISD::MEMSET:  return "memset";
-  case ISD::MEMCPY:  return "memcpy";
-  case ISD::MEMMOVE: return "memmove";
-
   // Bit manipulation
   case ISD::BSWAP:   return "bswap";
   case ISD::CTPOP:   return "ctpop";
@@ -4040,6 +4633,30 @@ const char *SDNode::getIndexedModeName(ISD::MemIndexedMode AM) {
   }
 }
 
+std::string ISD::ArgFlagsTy::getArgFlagsString() {
+  std::string S = "< ";
+
+  if (isZExt())
+    S += "zext ";
+  if (isSExt())
+    S += "sext ";
+  if (isInReg())
+    S += "inreg ";
+  if (isSRet())
+    S += "sret ";
+  if (isByVal())
+    S += "byval ";
+  if (isNest())
+    S += "nest ";
+  if (getByValAlign())
+    S += "byval-align:" + utostr(getByValAlign()) + " ";
+  if (getOrigAlign())
+    S += "orig-align:" + utostr(getOrigAlign()) + " ";
+  if (getByValSize())
+    S += "byval-size:" + utostr(getByValSize()) + " ";
+  return S + ">";
+}
+
 void SDNode::dump() const { dump(0); }
 void SDNode::dump(const SelectionDAG *G) const {
   cerr << (void*)this << ": ";
@@ -4049,7 +4666,7 @@ void SDNode::dump(const SelectionDAG *G) const {
     if (getValueType(i) == MVT::Other)
       cerr << "ch";
     else
-      cerr << MVT::getValueTypeString(getValueType(i));
+      cerr << getValueType(i).getMVTString();
   }
   cerr << " = " << getOperationName(G);
 
@@ -4118,7 +4735,7 @@ void SDNode::dump(const SelectionDAG *G) const {
   } else if (const RegisterSDNode *R = dyn_cast<RegisterSDNode>(this)) {
     if (G && R->getReg() &&
         TargetRegisterInfo::isPhysicalRegister(R->getReg())) {
-      cerr << " " <<G->getTarget().getRegisterInfo()->getName(R->getReg());
+      cerr << " " << G->getTarget().getRegisterInfo()->getName(R->getReg());
     } else {
       cerr << " #" << R->getReg();
     }
@@ -4135,8 +4752,10 @@ void SDNode::dump(const SelectionDAG *G) const {
       cerr << "<" << M->MO.getValue() << ":" << M->MO.getOffset() << ">";
     else
       cerr << "<null:" << M->MO.getOffset() << ">";
+  } else if (const ARG_FLAGSSDNode *N = dyn_cast<ARG_FLAGSSDNode>(this)) {
+    cerr << N->getArgFlags().getArgFlagsString();
   } else if (const VTSDNode *N = dyn_cast<VTSDNode>(this)) {
-    cerr << ":" << MVT::getValueTypeString(N->getVT());
+    cerr << ":" << N->getVT().getMVTString();
   } else if (const LoadSDNode *LD = dyn_cast<LoadSDNode>(this)) {
     const Value *SrcValue = LD->getSrcValue();
     int SrcOffset = LD->getSrcValueOffset();
@@ -4161,7 +4780,7 @@ void SDNode::dump(const SelectionDAG *G) const {
       break;
     }
     if (doExt)
-      cerr << MVT::getValueTypeString(LD->getMemoryVT()) << ">";
+      cerr << LD->getMemoryVT().getMVTString() << ">";
 
     const char *AM = getIndexedModeName(LD->getAddressingMode());
     if (*AM)
@@ -4181,7 +4800,7 @@ void SDNode::dump(const SelectionDAG *G) const {
 
     if (ST->isTruncatingStore())
       cerr << " <trunc "
-           << MVT::getValueTypeString(ST->getMemoryVT()) << ">";
+           << ST->getMemoryVT().getMVTString() << ">";
 
     const char *AM = getIndexedModeName(ST->getAddressingMode());
     if (*AM)