X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FVMCore%2FAutoUpgrade.cpp;h=9e551bb0604dff2e767e84d4963d34e816163799;hb=84025ba08ff23711b9d3d33c6c21819d63d30865;hp=4d06b66681694945d352ee40f5db03ea33f3bdcd;hpb=20adc9dc4650313f017b27d9818eb2176238113d;p=oota-llvm.git diff --git a/lib/VMCore/AutoUpgrade.cpp b/lib/VMCore/AutoUpgrade.cpp index 4d06b666816..9e551bb0604 100644 --- a/lib/VMCore/AutoUpgrade.cpp +++ b/lib/VMCore/AutoUpgrade.cpp @@ -18,7 +18,9 @@ #include "llvm/Module.h" #include "llvm/IntrinsicInst.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Support/CallSite.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/IRBuilder.h" #include using namespace llvm; @@ -76,6 +78,62 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { NewFn = F; return true; } + } else if (Name.compare(5, 9, "arm.neon.", 9) == 0) { + if (((Name.compare(14, 5, "vmovl", 5) == 0 || + Name.compare(14, 5, "vaddl", 5) == 0 || + Name.compare(14, 5, "vsubl", 5) == 0 || + Name.compare(14, 5, "vaddw", 5) == 0 || + Name.compare(14, 5, "vsubw", 5) == 0 || + Name.compare(14, 5, "vmlal", 5) == 0 || + Name.compare(14, 5, "vmlsl", 5) == 0 || + Name.compare(14, 5, "vabdl", 5) == 0 || + Name.compare(14, 5, "vabal", 5) == 0) && + (Name.compare(19, 2, "s.", 2) == 0 || + Name.compare(19, 2, "u.", 2) == 0)) || + + (Name.compare(14, 4, "vaba", 4) == 0 && + (Name.compare(18, 2, "s.", 2) == 0 || + Name.compare(18, 2, "u.", 2) == 0)) || + + (Name.compare(14, 6, "vmovn.", 6) == 0)) { + + // Calls to these are transformed into IR without intrinsics. + NewFn = 0; + return true; + } + // Old versions of NEON ld/st intrinsics are missing alignment arguments. + bool isVLd = (Name.compare(14, 3, "vld", 3) == 0); + bool isVSt = (Name.compare(14, 3, "vst", 3) == 0); + if (isVLd || isVSt) { + unsigned NumVecs = Name.at(17) - '0'; + if (NumVecs == 0 || NumVecs > 4) + return false; + bool isLaneOp = (Name.compare(18, 5, "lane.", 5) == 0); + if (!isLaneOp && Name.at(18) != '.') + return false; + unsigned ExpectedArgs = 2; // for the address and alignment + if (isVSt || isLaneOp) + ExpectedArgs += NumVecs; + if (isLaneOp) + ExpectedArgs += 1; // for the lane number + unsigned NumP = FTy->getNumParams(); + if (NumP != ExpectedArgs - 1) + return false; + + // Change the name of the old (bad) intrinsic, because + // its type is incorrect, but we cannot overload that name. + F->setName(""); + + // One argument is missing: add the alignment argument. + std::vector NewParams; + for (unsigned p = 0; p < NumP; ++p) + NewParams.push_back(FTy->getParamType(p)); + NewParams.push_back(Type::getInt32Ty(F->getContext())); + FunctionType *NewFTy = FunctionType::get(FTy->getReturnType(), + NewParams, false); + NewFn = cast(M->getOrInsertFunction(Name, NewFTy)); + return true; + } } break; case 'b': @@ -180,7 +238,6 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { NewFnName = "llvm.memset.p0i8.i64"; } if (NewFnName) { - const FunctionType *FTy = F->getFunctionType(); NewFn = cast(M->getOrInsertFunction(NewFnName, FTy->getReturnType(), FTy->getParamType(0), @@ -230,37 +287,224 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { break; case 'x': // This fixes all MMX shift intrinsic instructions to take a - // v1i64 instead of a v2i32 as the second parameter. - if (Name.compare(5,10,"x86.mmx.ps",10) == 0 && - (Name.compare(13,4,"psll", 4) == 0 || - Name.compare(13,4,"psra", 4) == 0 || - Name.compare(13,4,"psrl", 4) == 0) && Name[17] != 'i') { - - const llvm::Type *VT = - VectorType::get(IntegerType::get(FTy->getContext(), 64), 1); - - // We don't have to do anything if the parameter already has - // the correct type. - if (FTy->getParamType(1) == VT) + // x86_mmx instead of a v1i64, v2i32, v4i16, or v8i8. + if (Name.compare(5, 8, "x86.mmx.", 8) == 0) { + const Type *X86_MMXTy = VectorType::getX86_MMXTy(FTy->getContext()); + + if (Name.compare(13, 4, "padd", 4) == 0 || + Name.compare(13, 4, "psub", 4) == 0 || + Name.compare(13, 4, "pmul", 4) == 0 || + Name.compare(13, 5, "pmadd", 5) == 0 || + Name.compare(13, 4, "pand", 4) == 0 || + Name.compare(13, 3, "por", 3) == 0 || + Name.compare(13, 4, "pxor", 4) == 0 || + Name.compare(13, 4, "pavg", 4) == 0 || + Name.compare(13, 4, "pmax", 4) == 0 || + Name.compare(13, 4, "pmin", 4) == 0 || + Name.compare(13, 4, "psad", 4) == 0 || + Name.compare(13, 4, "psll", 4) == 0 || + Name.compare(13, 4, "psrl", 4) == 0 || + Name.compare(13, 4, "psra", 4) == 0 || + Name.compare(13, 4, "pack", 4) == 0 || + Name.compare(13, 6, "punpck", 6) == 0 || + Name.compare(13, 4, "pcmp", 4) == 0) { + assert(FTy->getNumParams() == 2 && "MMX intrinsic takes 2 args!"); + const Type *SecondParamTy = X86_MMXTy; + + if (Name.compare(13, 5, "pslli", 5) == 0 || + Name.compare(13, 5, "psrli", 5) == 0 || + Name.compare(13, 5, "psrai", 5) == 0) + SecondParamTy = FTy->getParamType(1); + + // Don't do anything if it has the correct types. + if (FTy->getReturnType() == X86_MMXTy && + FTy->getParamType(0) == X86_MMXTy && + FTy->getParamType(1) == SecondParamTy) + break; + + // We first need to change the name of the old (bad) intrinsic, because + // its type is incorrect, but we cannot overload that name. We + // arbitrarily unique it here allowing us to construct a correctly named + // and typed function below. + F->setName(""); + + // Now construct the new intrinsic with the correct name and type. We + // leave the old function around in order to query its type, whatever it + // may be, and correctly convert up to the new type. + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, X86_MMXTy, + SecondParamTy, (Type*)0)); + return true; + } + + if (Name.compare(13, 8, "maskmovq", 8) == 0) { + // Don't do anything if it has the correct types. + if (FTy->getParamType(0) == X86_MMXTy && + FTy->getParamType(1) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + FTy->getReturnType(), + X86_MMXTy, + X86_MMXTy, + FTy->getParamType(2), + (Type*)0)); + return true; + } + + if (Name.compare(13, 8, "pmovmskb", 8) == 0) { + if (FTy->getParamType(0) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + FTy->getReturnType(), + X86_MMXTy, + (Type*)0)); + return true; + } + + if (Name.compare(13, 5, "movnt", 5) == 0) { + if (FTy->getParamType(1) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + FTy->getReturnType(), + FTy->getParamType(0), + X86_MMXTy, + (Type*)0)); + return true; + } + + if (Name.compare(13, 7, "palignr", 7) == 0) { + if (FTy->getReturnType() == X86_MMXTy && + FTy->getParamType(0) == X86_MMXTy && + FTy->getParamType(1) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + X86_MMXTy, + X86_MMXTy, + FTy->getParamType(2), + (Type*)0)); + return true; + } + + if (Name.compare(13, 5, "pextr", 5) == 0) { + if (FTy->getParamType(0) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + FTy->getReturnType(), + X86_MMXTy, + FTy->getParamType(1), + (Type*)0)); + return true; + } + + if (Name.compare(13, 5, "pinsr", 5) == 0) { + if (FTy->getReturnType() == X86_MMXTy && + FTy->getParamType(0) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + X86_MMXTy, + FTy->getParamType(1), + FTy->getParamType(2), + (Type*)0)); + return true; + } + + if (Name.compare(13, 12, "cvtsi32.si64", 12) == 0) { + if (FTy->getReturnType() == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + FTy->getParamType(0), + (Type*)0)); + return true; + } + + if (Name.compare(13, 12, "cvtsi64.si32", 12) == 0) { + if (FTy->getParamType(0) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + FTy->getReturnType(), + X86_MMXTy, + (Type*)0)); + return true; + } + + if (Name.compare(13, 8, "vec.init", 8) == 0) { + if (FTy->getReturnType() == X86_MMXTy) + break; + + F->setName(""); + + if (Name.compare(21, 2, ".b", 2) == 0) + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + FTy->getParamType(0), + FTy->getParamType(1), + FTy->getParamType(2), + FTy->getParamType(3), + FTy->getParamType(4), + FTy->getParamType(5), + FTy->getParamType(6), + FTy->getParamType(7), + (Type*)0)); + else if (Name.compare(21, 2, ".w", 2) == 0) + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + FTy->getParamType(0), + FTy->getParamType(1), + FTy->getParamType(2), + FTy->getParamType(3), + (Type*)0)); + else if (Name.compare(21, 2, ".d", 2) == 0) + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + FTy->getParamType(0), + FTy->getParamType(1), + (Type*)0)); + return true; + } + + + if (Name.compare(13, 9, "vec.ext.d", 9) == 0) { + if (FTy->getReturnType() == X86_MMXTy && + FTy->getParamType(0) == X86_MMXTy) + break; + + F->setName(""); + NewFn = cast(M->getOrInsertFunction(Name, + X86_MMXTy, + X86_MMXTy, + FTy->getParamType(1), + (Type*)0)); + return true; + } + + if (Name.compare(13, 9, "emms", 4) == 0 || + Name.compare(13, 9, "femms", 5) == 0) { + NewFn = 0; break; - - // We first need to change the name of the old (bad) intrinsic, because - // its type is incorrect, but we cannot overload that name. We - // arbitrarily unique it here allowing us to construct a correctly named - // and typed function below. - F->setName(""); + } - assert(FTy->getNumParams() == 2 && "MMX shift intrinsics take 2 args!"); - - // Now construct the new intrinsic with the correct name and type. We - // leave the old function around in order to query its type, whatever it - // may be, and correctly convert up to the new type. - NewFn = cast(M->getOrInsertFunction(Name, - FTy->getReturnType(), - FTy->getParamType(0), - VT, - (Type *)0)); - return true; + // We really shouldn't get here ever. + assert(0 && "Invalid MMX intrinsic!"); + break; } else if (Name.compare(5,17,"x86.sse2.loadh.pd",17) == 0 || Name.compare(5,17,"x86.sse2.loadl.pd",17) == 0 || Name.compare(5,16,"x86.sse2.movl.dq",16) == 0 || @@ -277,8 +521,23 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { // Calls to these intrinsics are transformed into vector multiplies. NewFn = 0; return true; + } else if (Name.compare(5, 18, "x86.ssse3.palign.r", 18) == 0 || + Name.compare(5, 22, "x86.ssse3.palign.r.128", 22) == 0) { + // Calls to these intrinsics are transformed into vector shuffles, shifts, + // or 0. + NewFn = 0; + return true; + } else if (Name.compare(5, 17, "x86.ssse3.pshuf.w", 17) == 0) { + // This is an SSE/MMX instruction. + const Type *X86_MMXTy = VectorType::getX86_MMXTy(FTy->getContext()); + NewFn = + cast(M->getOrInsertFunction("llvm.x86.sse.pshuf.w", + X86_MMXTy, + X86_MMXTy, + Type::getInt8Ty(F->getContext()), + (Type*)0)); + return true; } - break; } @@ -302,16 +561,171 @@ bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { return Upgraded; } +bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { + StringRef Name(GV->getName()); + + // We are only upgrading one symbol here. + if (Name == ".llvm.eh.catch.all.value") { + GV->setName("llvm.eh.catch.all.value"); + return true; + } + + return false; +} + +/// ExtendNEONArgs - For NEON "long" and "wide" operations, where the results +/// have vector elements twice as big as one or both source operands, do the +/// sign- or zero-extension that used to be handled by intrinsics. The +/// extended values are returned via V0 and V1. +static void ExtendNEONArgs(CallInst *CI, Value *Arg0, Value *Arg1, + Value *&V0, Value *&V1) { + Function *F = CI->getCalledFunction(); + const std::string& Name = F->getName(); + bool isLong = (Name.at(18) == 'l'); + bool isSigned = (Name.at(19) == 's'); + + if (isSigned) { + if (isLong) + V0 = new SExtInst(Arg0, CI->getType(), "", CI); + else + V0 = Arg0; + V1 = new SExtInst(Arg1, CI->getType(), "", CI); + } else { + if (isLong) + V0 = new ZExtInst(Arg0, CI->getType(), "", CI); + else + V0 = Arg0; + V1 = new ZExtInst(Arg1, CI->getType(), "", CI); + } +} + +/// CallVABD - As part of expanding a call to one of the old NEON vabdl, vaba, +/// or vabal intrinsics, construct a call to a vabd intrinsic. Examine the +/// name of the old intrinsic to determine whether to use a signed or unsigned +/// vabd intrinsic. Get the type from the old call instruction, adjusted for +/// half-size vector elements if the old intrinsic was vabdl or vabal. +static Instruction *CallVABD(CallInst *CI, Value *Arg0, Value *Arg1) { + Function *F = CI->getCalledFunction(); + const std::string& Name = F->getName(); + bool isLong = (Name.at(18) == 'l'); + bool isSigned = (Name.at(isLong ? 19 : 18) == 's'); + + Intrinsic::ID intID; + if (isSigned) + intID = Intrinsic::arm_neon_vabds; + else + intID = Intrinsic::arm_neon_vabdu; + + const Type *Ty = CI->getType(); + if (isLong) + Ty = VectorType::getTruncatedElementVectorType(cast(Ty)); + + Function *VABD = Intrinsic::getDeclaration(F->getParent(), intID, &Ty, 1); + Value *Operands[2]; + Operands[0] = Arg0; + Operands[1] = Arg1; + return CallInst::Create(VABD, Operands, Operands+2, + "upgraded."+CI->getName(), CI); +} + +/// ConstructNewCallInst - Construct a new CallInst with the signature of NewFn. +static void ConstructNewCallInst(Function *NewFn, CallInst *OldCI, + Value **Operands, unsigned NumOps, + bool AssignName = true) { + // Construct a new CallInst. + CallInst *NewCI = + CallInst::Create(NewFn, Operands, Operands + NumOps, + AssignName ? "upgraded." + OldCI->getName() : "", OldCI); + + NewCI->setTailCall(OldCI->isTailCall()); + NewCI->setCallingConv(OldCI->getCallingConv()); + + // Handle any uses of the old CallInst. If the type has changed, add a cast. + if (!OldCI->use_empty()) { + if (OldCI->getType() != NewCI->getType()) { + Function *OldFn = OldCI->getCalledFunction(); + CastInst *RetCast = + CastInst::Create(CastInst::getCastOpcode(NewCI, true, + OldFn->getReturnType(), true), + NewCI, OldFn->getReturnType(), NewCI->getName(),OldCI); + + // Replace all uses of the old call with the new cast which has the + // correct type. + OldCI->replaceAllUsesWith(RetCast); + } else { + OldCI->replaceAllUsesWith(NewCI); + } + } + + // Clean up the old call now that it has been completely upgraded. + OldCI->eraseFromParent(); +} + // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the // upgraded intrinsic. All argument and return casting must be provided in // order to seamlessly integrate with existing context. void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Function *F = CI->getCalledFunction(); LLVMContext &C = CI->getContext(); - + ImmutableCallSite CS(CI); + assert(F && "CallInst has no function associated with it."); if (!NewFn) { + // Get the Function's name. + const std::string& Name = F->getName(); + + // Upgrade ARM NEON intrinsics. + if (Name.compare(5, 9, "arm.neon.", 9) == 0) { + Instruction *NewI; + Value *V0, *V1; + if (Name.compare(14, 7, "vmovls.", 7) == 0) { + NewI = new SExtInst(CI->getArgOperand(0), CI->getType(), + "upgraded." + CI->getName(), CI); + } else if (Name.compare(14, 7, "vmovlu.", 7) == 0) { + NewI = new ZExtInst(CI->getArgOperand(0), CI->getType(), + "upgraded." + CI->getName(), CI); + } else if (Name.compare(14, 4, "vadd", 4) == 0) { + ExtendNEONArgs(CI, CI->getArgOperand(0), CI->getArgOperand(1), V0, V1); + NewI = BinaryOperator::CreateAdd(V0, V1, "upgraded."+CI->getName(), CI); + } else if (Name.compare(14, 4, "vsub", 4) == 0) { + ExtendNEONArgs(CI, CI->getArgOperand(0), CI->getArgOperand(1), V0, V1); + NewI = BinaryOperator::CreateSub(V0, V1,"upgraded."+CI->getName(),CI); + } else if (Name.compare(14, 4, "vmul", 4) == 0) { + ExtendNEONArgs(CI, CI->getArgOperand(0), CI->getArgOperand(1), V0, V1); + NewI = BinaryOperator::CreateMul(V0, V1,"upgraded."+CI->getName(),CI); + } else if (Name.compare(14, 4, "vmla", 4) == 0) { + ExtendNEONArgs(CI, CI->getArgOperand(1), CI->getArgOperand(2), V0, V1); + Instruction *MulI = BinaryOperator::CreateMul(V0, V1, "", CI); + NewI = BinaryOperator::CreateAdd(CI->getArgOperand(0), MulI, + "upgraded."+CI->getName(), CI); + } else if (Name.compare(14, 4, "vmls", 4) == 0) { + ExtendNEONArgs(CI, CI->getArgOperand(1), CI->getArgOperand(2), V0, V1); + Instruction *MulI = BinaryOperator::CreateMul(V0, V1, "", CI); + NewI = BinaryOperator::CreateSub(CI->getArgOperand(0), MulI, + "upgraded."+CI->getName(), CI); + } else if (Name.compare(14, 4, "vabd", 4) == 0) { + NewI = CallVABD(CI, CI->getArgOperand(0), CI->getArgOperand(1)); + NewI = new ZExtInst(NewI, CI->getType(), "upgraded."+CI->getName(), CI); + } else if (Name.compare(14, 4, "vaba", 4) == 0) { + NewI = CallVABD(CI, CI->getArgOperand(1), CI->getArgOperand(2)); + if (Name.at(18) == 'l') + NewI = new ZExtInst(NewI, CI->getType(), "", CI); + NewI = BinaryOperator::CreateAdd(CI->getArgOperand(0), NewI, + "upgraded."+CI->getName(), CI); + } else if (Name.compare(14, 6, "vmovn.", 6) == 0) { + NewI = new TruncInst(CI->getArgOperand(0), CI->getType(), + "upgraded." + CI->getName(), CI); + } else { + llvm_unreachable("Unknown arm.neon function for CallInst upgrade."); + } + // Replace any uses of the old CallInst. + if (!CI->use_empty()) + CI->replaceAllUsesWith(NewI); + CI->eraseFromParent(); + return; + } + bool isLoadH = false, isLoadL = false, isMovL = false; bool isMovSD = false, isShufPD = false; bool isUnpckhPD = false, isUnpcklPD = false; @@ -338,11 +752,11 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { if (isLoadH || isLoadL || isMovL || isMovSD || isShufPD || isUnpckhPD || isUnpcklPD || isPunpckhQPD || isPunpcklQPD) { std::vector Idxs; - Value *Op0 = CI->getOperand(1); + Value *Op0 = CI->getArgOperand(0); ShuffleVectorInst *SI = NULL; if (isLoadH || isLoadL) { Value *Op1 = UndefValue::get(Op0->getType()); - Value *Addr = new BitCastInst(CI->getOperand(2), + Value *Addr = new BitCastInst(CI->getArgOperand(1), Type::getDoublePtrTy(C), "upgraded.", CI); Value *Load = new LoadInst(Addr, "upgraded.", false, 8, CI); @@ -375,7 +789,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { SI = new ShuffleVectorInst(ZeroV, Op0, Mask, "upgraded.", CI); } else if (isMovSD || isUnpckhPD || isUnpcklPD || isPunpckhQPD || isPunpcklQPD) { - Value *Op1 = CI->getOperand(2); + Value *Op1 = CI->getArgOperand(1); if (isMovSD) { Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 2)); Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), 1)); @@ -389,8 +803,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Mask = ConstantVector::get(Idxs); SI = new ShuffleVectorInst(Op0, Op1, Mask, "upgraded.", CI); } else if (isShufPD) { - Value *Op1 = CI->getOperand(2); - unsigned MaskVal = cast(CI->getOperand(3))->getZExtValue(); + Value *Op1 = CI->getArgOperand(1); + unsigned MaskVal = + cast(CI->getArgOperand(2))->getZExtValue(); Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), MaskVal & 1)); Idxs.push_back(ConstantInt::get(Type::getInt32Ty(C), ((MaskVal >> 1) & 1)+2)); @@ -410,8 +825,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->eraseFromParent(); } else if (F->getName() == "llvm.x86.sse41.pmulld") { // Upgrade this set of intrinsics into vector multiplies. - Instruction *Mul = BinaryOperator::CreateMul(CI->getOperand(1), - CI->getOperand(2), + Instruction *Mul = BinaryOperator::CreateMul(CI->getArgOperand(0), + CI->getArgOperand(1), CI->getName(), CI); // Fix up all the uses with our new multiply. @@ -420,6 +835,118 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Remove upgraded multiply. CI->eraseFromParent(); + } else if (F->getName() == "llvm.x86.ssse3.palign.r") { + Value *Op1 = CI->getArgOperand(0); + Value *Op2 = CI->getArgOperand(1); + Value *Op3 = CI->getArgOperand(2); + unsigned shiftVal = cast(Op3)->getZExtValue(); + Value *Rep; + IRBuilder<> Builder(C); + Builder.SetInsertPoint(CI->getParent(), CI); + + // If palignr is shifting the pair of input vectors less than 9 bytes, + // emit a shuffle instruction. + if (shiftVal <= 8) { + const Type *IntTy = Type::getInt32Ty(C); + const Type *EltTy = Type::getInt8Ty(C); + const Type *VecTy = VectorType::get(EltTy, 8); + + Op2 = Builder.CreateBitCast(Op2, VecTy); + Op1 = Builder.CreateBitCast(Op1, VecTy); + + llvm::SmallVector Indices; + for (unsigned i = 0; i != 8; ++i) + Indices.push_back(ConstantInt::get(IntTy, shiftVal + i)); + + Value *SV = ConstantVector::get(Indices); + Rep = Builder.CreateShuffleVector(Op2, Op1, SV, "palignr"); + Rep = Builder.CreateBitCast(Rep, F->getReturnType()); + } + + // If palignr is shifting the pair of input vectors more than 8 but less + // than 16 bytes, emit a logical right shift of the destination. + else if (shiftVal < 16) { + // MMX has these as 1 x i64 vectors for some odd optimization reasons. + const Type *EltTy = Type::getInt64Ty(C); + const Type *VecTy = VectorType::get(EltTy, 1); + + Op1 = Builder.CreateBitCast(Op1, VecTy, "cast"); + Op2 = ConstantInt::get(VecTy, (shiftVal-8) * 8); + + // create i32 constant + Function *I = + Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_mmx_psrl_q); + Rep = Builder.CreateCall2(I, Op1, Op2, "palignr"); + } + + // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. + else { + Rep = Constant::getNullValue(F->getReturnType()); + } + + // Replace any uses with our new instruction. + if (!CI->use_empty()) + CI->replaceAllUsesWith(Rep); + + // Remove upgraded instruction. + CI->eraseFromParent(); + + } else if (F->getName() == "llvm.x86.ssse3.palign.r.128") { + Value *Op1 = CI->getArgOperand(0); + Value *Op2 = CI->getArgOperand(1); + Value *Op3 = CI->getArgOperand(2); + unsigned shiftVal = cast(Op3)->getZExtValue(); + Value *Rep; + IRBuilder<> Builder(C); + Builder.SetInsertPoint(CI->getParent(), CI); + + // If palignr is shifting the pair of input vectors less than 17 bytes, + // emit a shuffle instruction. + if (shiftVal <= 16) { + const Type *IntTy = Type::getInt32Ty(C); + const Type *EltTy = Type::getInt8Ty(C); + const Type *VecTy = VectorType::get(EltTy, 16); + + Op2 = Builder.CreateBitCast(Op2, VecTy); + Op1 = Builder.CreateBitCast(Op1, VecTy); + + llvm::SmallVector Indices; + for (unsigned i = 0; i != 16; ++i) + Indices.push_back(ConstantInt::get(IntTy, shiftVal + i)); + + Value *SV = ConstantVector::get(Indices); + Rep = Builder.CreateShuffleVector(Op2, Op1, SV, "palignr"); + Rep = Builder.CreateBitCast(Rep, F->getReturnType()); + } + + // If palignr is shifting the pair of input vectors more than 16 but less + // than 32 bytes, emit a logical right shift of the destination. + else if (shiftVal < 32) { + const Type *EltTy = Type::getInt64Ty(C); + const Type *VecTy = VectorType::get(EltTy, 2); + const Type *IntTy = Type::getInt32Ty(C); + + Op1 = Builder.CreateBitCast(Op1, VecTy, "cast"); + Op2 = ConstantInt::get(IntTy, (shiftVal-16) * 8); + + // create i32 constant + Function *I = + Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_sse2_psrl_dq); + Rep = Builder.CreateCall2(I, Op1, Op2, "palignr"); + } + + // If palignr is shifting the pair of vectors more than 32 bytes, emit zero. + else { + Rep = Constant::getNullValue(F->getReturnType()); + } + + // Replace any uses with our new instruction. + if (!CI->use_empty()) + CI->replaceAllUsesWith(Rep); + + // Remove upgraded instruction. + CI->eraseFromParent(); + } else { llvm_unreachable("Unknown function for CallInst upgrade."); } @@ -427,31 +954,29 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { } switch (NewFn->getIntrinsicID()) { - default: llvm_unreachable("Unknown function for CallInst upgrade."); - case Intrinsic::x86_mmx_psll_d: - case Intrinsic::x86_mmx_psll_q: - case Intrinsic::x86_mmx_psll_w: - case Intrinsic::x86_mmx_psra_d: - case Intrinsic::x86_mmx_psra_w: - case Intrinsic::x86_mmx_psrl_d: - case Intrinsic::x86_mmx_psrl_q: - case Intrinsic::x86_mmx_psrl_w: { - Value *Operands[2]; - - Operands[0] = CI->getOperand(1); - - // Cast the second parameter to the correct type. - BitCastInst *BC = new BitCastInst(CI->getOperand(2), - NewFn->getFunctionType()->getParamType(1), - "upgraded.", CI); - Operands[1] = BC; - - // Construct a new CallInst - CallInst *NewCI = CallInst::Create(NewFn, Operands, Operands+2, - "upgraded."+CI->getName(), CI); + default: llvm_unreachable("Unknown function for CallInst upgrade."); + case Intrinsic::arm_neon_vld1: + case Intrinsic::arm_neon_vld2: + case Intrinsic::arm_neon_vld3: + case Intrinsic::arm_neon_vld4: + case Intrinsic::arm_neon_vst1: + case Intrinsic::arm_neon_vst2: + case Intrinsic::arm_neon_vst3: + case Intrinsic::arm_neon_vst4: + case Intrinsic::arm_neon_vld2lane: + case Intrinsic::arm_neon_vld3lane: + case Intrinsic::arm_neon_vld4lane: + case Intrinsic::arm_neon_vst2lane: + case Intrinsic::arm_neon_vst3lane: + case Intrinsic::arm_neon_vst4lane: { + // Add a default alignment argument of 1. + SmallVector Operands(CS.arg_begin(), CS.arg_end()); + Operands.push_back(ConstantInt::get(Type::getInt32Ty(C), 1)); + CallInst *NewCI = CallInst::Create(NewFn, Operands.begin(), Operands.end(), + CI->getName(), CI); NewCI->setTailCall(CI->isTailCall()); NewCI->setCallingConv(CI->getCallingConv()); - + // Handle any uses of the old CallInst. if (!CI->use_empty()) // Replace all uses of the old call with the new cast which has the @@ -462,12 +987,203 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->eraseFromParent(); break; } + + case Intrinsic::x86_mmx_padd_b: + case Intrinsic::x86_mmx_padd_w: + case Intrinsic::x86_mmx_padd_d: + case Intrinsic::x86_mmx_padd_q: + case Intrinsic::x86_mmx_padds_b: + case Intrinsic::x86_mmx_padds_w: + case Intrinsic::x86_mmx_paddus_b: + case Intrinsic::x86_mmx_paddus_w: + case Intrinsic::x86_mmx_psub_b: + case Intrinsic::x86_mmx_psub_w: + case Intrinsic::x86_mmx_psub_d: + case Intrinsic::x86_mmx_psub_q: + case Intrinsic::x86_mmx_psubs_b: + case Intrinsic::x86_mmx_psubs_w: + case Intrinsic::x86_mmx_psubus_b: + case Intrinsic::x86_mmx_psubus_w: + case Intrinsic::x86_mmx_pmulh_w: + case Intrinsic::x86_mmx_pmull_w: + case Intrinsic::x86_mmx_pmulhu_w: + case Intrinsic::x86_mmx_pmulu_dq: + case Intrinsic::x86_mmx_pmadd_wd: + case Intrinsic::x86_mmx_pand: + case Intrinsic::x86_mmx_pandn: + case Intrinsic::x86_mmx_por: + case Intrinsic::x86_mmx_pxor: + case Intrinsic::x86_mmx_pavg_b: + case Intrinsic::x86_mmx_pavg_w: + case Intrinsic::x86_mmx_pmaxu_b: + case Intrinsic::x86_mmx_pmaxs_w: + case Intrinsic::x86_mmx_pminu_b: + case Intrinsic::x86_mmx_pmins_w: + case Intrinsic::x86_mmx_psad_bw: + case Intrinsic::x86_mmx_psll_w: + case Intrinsic::x86_mmx_psll_d: + case Intrinsic::x86_mmx_psll_q: + case Intrinsic::x86_mmx_pslli_w: + case Intrinsic::x86_mmx_pslli_d: + case Intrinsic::x86_mmx_pslli_q: + case Intrinsic::x86_mmx_psrl_w: + case Intrinsic::x86_mmx_psrl_d: + case Intrinsic::x86_mmx_psrl_q: + case Intrinsic::x86_mmx_psrli_w: + case Intrinsic::x86_mmx_psrli_d: + case Intrinsic::x86_mmx_psrli_q: + case Intrinsic::x86_mmx_psra_w: + case Intrinsic::x86_mmx_psra_d: + case Intrinsic::x86_mmx_psrai_w: + case Intrinsic::x86_mmx_psrai_d: + case Intrinsic::x86_mmx_packsswb: + case Intrinsic::x86_mmx_packssdw: + case Intrinsic::x86_mmx_packuswb: + case Intrinsic::x86_mmx_punpckhbw: + case Intrinsic::x86_mmx_punpckhwd: + case Intrinsic::x86_mmx_punpckhdq: + case Intrinsic::x86_mmx_punpcklbw: + case Intrinsic::x86_mmx_punpcklwd: + case Intrinsic::x86_mmx_punpckldq: + case Intrinsic::x86_mmx_pcmpeq_b: + case Intrinsic::x86_mmx_pcmpeq_w: + case Intrinsic::x86_mmx_pcmpeq_d: + case Intrinsic::x86_mmx_pcmpgt_b: + case Intrinsic::x86_mmx_pcmpgt_w: + case Intrinsic::x86_mmx_pcmpgt_d: { + Value *Operands[2]; + + // Cast the operand to the X86 MMX type. + Operands[0] = new BitCastInst(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded.", CI); + + switch (NewFn->getIntrinsicID()) { + default: + // Cast to the X86 MMX type. + Operands[1] = new BitCastInst(CI->getArgOperand(1), + NewFn->getFunctionType()->getParamType(1), + "upgraded.", CI); + break; + case Intrinsic::x86_mmx_pslli_w: + case Intrinsic::x86_mmx_pslli_d: + case Intrinsic::x86_mmx_pslli_q: + case Intrinsic::x86_mmx_psrli_w: + case Intrinsic::x86_mmx_psrli_d: + case Intrinsic::x86_mmx_psrli_q: + case Intrinsic::x86_mmx_psrai_w: + case Intrinsic::x86_mmx_psrai_d: + // These take an i32 as their second parameter. + Operands[1] = CI->getArgOperand(1); + break; + } + + ConstructNewCallInst(NewFn, CI, Operands, 2); + break; + } + case Intrinsic::x86_mmx_maskmovq: { + Value *Operands[3]; + + // Cast the operands to the X86 MMX type. + Operands[0] = new BitCastInst(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded.", CI); + Operands[1] = new BitCastInst(CI->getArgOperand(1), + NewFn->getFunctionType()->getParamType(1), + "upgraded.", CI); + Operands[2] = CI->getArgOperand(2); + + ConstructNewCallInst(NewFn, CI, Operands, 3, false); + break; + } + case Intrinsic::x86_mmx_pmovmskb: { + Value *Operands[1]; + + // Cast the operand to the X86 MMX type. + Operands[0] = new BitCastInst(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded.", CI); + + ConstructNewCallInst(NewFn, CI, Operands, 1); + break; + } + case Intrinsic::x86_mmx_movnt_dq: { + Value *Operands[2]; + + Operands[0] = CI->getArgOperand(0); + + // Cast the operand to the X86 MMX type. + Operands[1] = new BitCastInst(CI->getArgOperand(1), + NewFn->getFunctionType()->getParamType(1), + "upgraded.", CI); + + ConstructNewCallInst(NewFn, CI, Operands, 2, false); + break; + } + case Intrinsic::x86_mmx_palignr_b: { + Value *Operands[3]; + + // Cast the operands to the X86 MMX type. + Operands[0] = new BitCastInst(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded.", CI); + Operands[1] = new BitCastInst(CI->getArgOperand(1), + NewFn->getFunctionType()->getParamType(1), + "upgraded.", CI); + Operands[2] = CI->getArgOperand(2); + + ConstructNewCallInst(NewFn, CI, Operands, 3); + break; + } + case Intrinsic::x86_mmx_pextr_w: { + Value *Operands[2]; + + // Cast the operands to the X86 MMX type. + Operands[0] = new BitCastInst(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded.", CI); + Operands[1] = CI->getArgOperand(1); + + ConstructNewCallInst(NewFn, CI, Operands, 2); + break; + } + case Intrinsic::x86_mmx_pinsr_w: { + Value *Operands[3]; + + // Cast the operands to the X86 MMX type. + Operands[0] = new BitCastInst(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded.", CI); + Operands[1] = CI->getArgOperand(1); + Operands[2] = CI->getArgOperand(2); + + ConstructNewCallInst(NewFn, CI, Operands, 3); + break; + } + case Intrinsic::x86_sse_pshuf_w: { + IRBuilder<> Builder(C); + Builder.SetInsertPoint(CI->getParent(), CI); + + // Cast the operand to the X86 MMX type. + Value *Operands[2]; + Operands[0] = + Builder.CreateBitCast(CI->getArgOperand(0), + NewFn->getFunctionType()->getParamType(0), + "upgraded."); + Operands[1] = + Builder.CreateTrunc(CI->getArgOperand(1), + Type::getInt8Ty(C), + "upgraded."); + + ConstructNewCallInst(NewFn, CI, Operands, 2); + break; + } + case Intrinsic::ctlz: case Intrinsic::ctpop: case Intrinsic::cttz: { - // Build a small vector of the 1..(N-1) operands, which are the - // parameters. - SmallVector Operands(CI->op_begin()+1, CI->op_end()); + // Build a small vector of the original arguments. + SmallVector Operands(CS.arg_begin(), CS.arg_end()); // Construct a new CallInst CallInst *NewCI = CallInst::Create(NewFn, Operands.begin(), Operands.end(), @@ -502,7 +1218,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { case Intrinsic::eh_selector: case Intrinsic::eh_typeid_for: { // Only the return type changed. - SmallVector Operands(CI->op_begin() + 1, CI->op_end()); + SmallVector Operands(CS.arg_begin(), CS.arg_end()); CallInst *NewCI = CallInst::Create(NewFn, Operands.begin(), Operands.end(), "upgraded." + CI->getName(), CI); NewCI->setTailCall(CI->isTailCall()); @@ -525,8 +1241,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { case Intrinsic::memset: { // Add isVolatile const llvm::Type *I1Ty = llvm::Type::getInt1Ty(CI->getContext()); - Value *Operands[5] = { CI->getOperand(1), CI->getOperand(2), - CI->getOperand(3), CI->getOperand(4), + Value *Operands[5] = { CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2), CI->getArgOperand(3), llvm::ConstantInt::get(I1Ty, 0) }; CallInst *NewCI = CallInst::Create(NewFn, Operands, Operands+5, CI->getName(), CI); @@ -608,7 +1324,8 @@ void llvm::CheckDebugInfoIntrinsics(Module *M) { if (Function *Declare = M->getFunction("llvm.dbg.declare")) { if (!Declare->use_empty()) { DbgDeclareInst *DDI = cast(Declare->use_back()); - if (!isa(DDI->getOperand(1)) ||!isa(DDI->getOperand(2))) { + if (!isa(DDI->getArgOperand(0)) || + !isa(DDI->getArgOperand(1))) { while (!Declare->use_empty()) { CallInst *CI = cast(Declare->use_back()); CI->eraseFromParent();