+ assert(F && "Intrinsic call is not direct?");
+
+ if (!NewFn) {
+ // Get the Function's name.
+ StringRef Name = F->getName();
+
+ Value *Rep;
+ // Upgrade packed integer vector compares intrinsics to compare instructions
+ if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
+ Name.startswith("llvm.x86.avx2.pcmpeq.")) {
+ Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
+ "pcmpeq");
+ // need to sign extend since icmp returns vector of i1
+ Rep = Builder.CreateSExt(Rep, CI->getType(), "");
+ } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
+ Name.startswith("llvm.x86.avx2.pcmpgt.")) {
+ Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
+ "pcmpgt");
+ // need to sign extend since icmp returns vector of i1
+ Rep = Builder.CreateSExt(Rep, CI->getType(), "");
+ } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
+ Name == "llvm.x86.avx.movnt.ps.256" ||
+ Name == "llvm.x86.avx.movnt.pd.256") {
+ IRBuilder<> Builder(C);
+ Builder.SetInsertPoint(CI->getParent(), CI);
+
+ Module *M = F->getParent();
+ SmallVector<Value *, 1> Elts;
+ Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
+ MDNode *Node = MDNode::get(C, Elts);
+
+ Value *Arg0 = CI->getArgOperand(0);
+ Value *Arg1 = CI->getArgOperand(1);
+
+ // Convert the type of the pointer to a pointer to the stored type.
+ Value *BC = Builder.CreateBitCast(Arg0,
+ PointerType::getUnqual(Arg1->getType()),
+ "cast");
+ StoreInst *SI = Builder.CreateStore(Arg1, BC);
+ SI->setMetadata(M->getMDKindID("nontemporal"), Node);
+ SI->setAlignment(16);
+
+ // Remove intrinsic.
+ CI->eraseFromParent();
+ return;
+ } else if (Name.startswith("llvm.x86.xop.vpcom")) {
+ Intrinsic::ID intID;
+ if (Name.endswith("ub"))
+ intID = Intrinsic::x86_xop_vpcomub;
+ else if (Name.endswith("uw"))
+ intID = Intrinsic::x86_xop_vpcomuw;
+ else if (Name.endswith("ud"))
+ intID = Intrinsic::x86_xop_vpcomud;
+ else if (Name.endswith("uq"))
+ intID = Intrinsic::x86_xop_vpcomuq;
+ else if (Name.endswith("b"))
+ intID = Intrinsic::x86_xop_vpcomb;
+ else if (Name.endswith("w"))
+ intID = Intrinsic::x86_xop_vpcomw;
+ else if (Name.endswith("d"))
+ intID = Intrinsic::x86_xop_vpcomd;
+ else if (Name.endswith("q"))
+ intID = Intrinsic::x86_xop_vpcomq;
+ else
+ llvm_unreachable("Unknown suffix");
+
+ Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
+ unsigned Imm;
+ if (Name.startswith("lt"))
+ Imm = 0;
+ else if (Name.startswith("le"))
+ Imm = 1;
+ else if (Name.startswith("gt"))
+ Imm = 2;
+ else if (Name.startswith("ge"))
+ Imm = 3;
+ else if (Name.startswith("eq"))
+ Imm = 4;
+ else if (Name.startswith("ne"))
+ Imm = 5;
+ else if (Name.startswith("true"))
+ Imm = 6;
+ else if (Name.startswith("false"))
+ Imm = 7;
+ else
+ llvm_unreachable("Unknown condition");
+
+ Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
+ Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
+ CI->getArgOperand(1), Builder.getInt8(Imm));
+ } else {
+ bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
+ if (Name == "llvm.x86.avx.vpermil.pd.256")
+ PD256 = true;
+ else if (Name == "llvm.x86.avx.vpermil.pd")
+ PD128 = true;
+ else if (Name == "llvm.x86.avx.vpermil.ps.256")
+ PS256 = true;
+ else if (Name == "llvm.x86.avx.vpermil.ps")
+ PS128 = true;
+
+ if (PD256 || PD128 || PS256 || PS128) {
+ Value *Op0 = CI->getArgOperand(0);
+ unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+ SmallVector<Constant*, 8> Idxs;
+
+ if (PD128)
+ for (unsigned i = 0; i != 2; ++i)
+ Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
+ else if (PD256)
+ for (unsigned l = 0; l != 4; l+=2)
+ for (unsigned i = 0; i != 2; ++i)
+ Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
+ else if (PS128)
+ for (unsigned i = 0; i != 4; ++i)
+ Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
+ else if (PS256)
+ for (unsigned l = 0; l != 8; l+=4)
+ for (unsigned i = 0; i != 4; ++i)
+ Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
+ else
+ llvm_unreachable("Unexpected function");
+
+ Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
+ } else {
+ llvm_unreachable("Unknown function for CallInst upgrade.");
+ }
+ }
+
+ CI->replaceAllUsesWith(Rep);
+ CI->eraseFromParent();
+ return;
+ }
+
+ std::string Name = CI->getName().str();
+ CI->setName(Name + ".old");
+