1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the auto-upgrade helper functions
12 //===----------------------------------------------------------------------===//
14 #include "llvm/IR/AutoUpgrade.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/CallSite.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DebugInfo.h"
19 #include "llvm/IR/DiagnosticInfo.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/Instruction.h"
23 #include "llvm/IR/IntrinsicInst.h"
24 #include "llvm/IR/LLVMContext.h"
25 #include "llvm/IR/Module.h"
26 #include "llvm/Support/ErrorHandling.h"
30 // Upgrade the declarations of the SSE4.1 functions whose arguments have
31 // changed their type from v4f32 to v2i64.
32 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID,
34 // Check whether this is an old version of the function, which received
36 Type *Arg0Type = F->getFunctionType()->getParamType(0);
37 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4))
40 // Yes, it's old, replace it with new version.
41 F->setName(F->getName() + ".old");
42 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
46 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
47 assert(F && "Illegal to upgrade a non-existent Function.");
49 // Quickly eliminate it, if it's not a candidate.
50 StringRef Name = F->getName();
51 if (Name.size() <= 8 || !Name.startswith("llvm."))
53 Name = Name.substr(5); // Strip off "llvm."
58 if (Name.startswith("arm.neon.vclz")) {
60 F->arg_begin()->getType(),
61 Type::getInt1Ty(F->getContext())
63 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
64 // the end of the name. Change name from llvm.arm.neon.vclz.* to
66 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
67 NewFn = Function::Create(fType, F->getLinkage(),
68 "llvm.ctlz." + Name.substr(14), F->getParent());
71 if (Name.startswith("arm.neon.vcnt")) {
72 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
73 F->arg_begin()->getType());
79 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
80 F->setName(Name + ".old");
81 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
82 F->arg_begin()->getType());
85 if (Name.startswith("cttz.") && F->arg_size() == 1) {
86 F->setName(Name + ".old");
87 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
88 F->arg_begin()->getType());
94 // We only need to change the name to match the mangling including the
96 if (F->arg_size() == 2 && Name.startswith("objectsize.")) {
97 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
98 if (F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
99 F->setName(Name + ".old");
100 NewFn = Intrinsic::getDeclaration(F->getParent(),
101 Intrinsic::objectsize, Tys);
108 if (Name.startswith("x86.sse2.pcmpeq.") ||
109 Name.startswith("x86.sse2.pcmpgt.") ||
110 Name.startswith("x86.avx2.pcmpeq.") ||
111 Name.startswith("x86.avx2.pcmpgt.") ||
112 Name.startswith("x86.avx.vpermil.") ||
113 Name == "x86.avx.movnt.dq.256" ||
114 Name == "x86.avx.movnt.pd.256" ||
115 Name == "x86.avx.movnt.ps.256" ||
116 Name == "x86.sse42.crc32.64.8" ||
117 Name == "x86.avx.vbroadcast.ss" ||
118 Name == "x86.avx.vbroadcast.ss.256" ||
119 Name == "x86.avx.vbroadcast.sd.256" ||
120 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) {
124 // SSE4.1 ptest functions may have an old signature.
125 if (Name.startswith("x86.sse41.ptest")) {
126 if (Name == "x86.sse41.ptestc")
127 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn);
128 if (Name == "x86.sse41.ptestz")
129 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn);
130 if (Name == "x86.sse41.ptestnzc")
131 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
133 // frcz.ss/sd may need to have an argument dropped
134 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) {
135 F->setName(Name + ".old");
136 NewFn = Intrinsic::getDeclaration(F->getParent(),
137 Intrinsic::x86_xop_vfrcz_ss);
140 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) {
141 F->setName(Name + ".old");
142 NewFn = Intrinsic::getDeclaration(F->getParent(),
143 Intrinsic::x86_xop_vfrcz_sd);
146 // Fix the FMA4 intrinsics to remove the 4
147 if (Name.startswith("x86.fma4.")) {
148 F->setName("llvm.x86.fma" + Name.substr(8));
156 // This may not belong here. This function is effectively being overloaded
157 // to both detect an intrinsic which needs upgrading, and to provide the
158 // upgraded form of the intrinsic. We should perhaps have two separate
159 // functions for this.
163 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
165 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
167 // Upgrade intrinsic attributes. This does not change the function.
170 if (unsigned id = F->getIntrinsicID())
171 F->setAttributes(Intrinsic::getAttributes(F->getContext(),
176 static bool UpgradeGlobalStructors(GlobalVariable *GV) {
177 ArrayType *ATy = dyn_cast<ArrayType>(GV->getType()->getElementType());
179 ATy ? dyn_cast<StructType>(ATy->getElementType()) : nullptr;
181 // Only upgrade an array of a two field struct with the appropriate field
183 if (!OldTy || OldTy->getNumElements() != 2)
186 // Get the upgraded 3 element type.
187 PointerType *VoidPtrTy = Type::getInt8Ty(GV->getContext())->getPointerTo();
189 OldTy->getElementType(0),
190 OldTy->getElementType(1),
194 StructType::get(GV->getContext(), Tys, /*isPacked=*/false);
196 // Build new constants with a null third field filled in.
197 Constant *OldInitC = GV->getInitializer();
198 ConstantArray *OldInit = dyn_cast<ConstantArray>(OldInitC);
199 if (!OldInit && !isa<ConstantAggregateZero>(OldInitC))
201 std::vector<Constant *> Initializers;
203 for (Use &U : OldInit->operands()) {
204 ConstantStruct *Init = cast<ConstantStruct>(&U);
206 ConstantStruct::get(NewTy, Init->getOperand(0), Init->getOperand(1),
207 Constant::getNullValue(VoidPtrTy), nullptr);
208 Initializers.push_back(NewInit);
211 assert(Initializers.size() == ATy->getNumElements());
213 // Replace the old GV with a new one.
214 ATy = ArrayType::get(NewTy, Initializers.size());
215 Constant *NewInit = ConstantArray::get(ATy, Initializers);
216 GlobalVariable *NewGV = new GlobalVariable(
217 *GV->getParent(), ATy, GV->isConstant(), GV->getLinkage(), NewInit, "",
218 GV, GV->getThreadLocalMode(), GV->getType()->getAddressSpace(),
219 GV->isExternallyInitialized());
220 NewGV->copyAttributesFrom(GV);
222 assert(GV->use_empty() && "program cannot use initializer list");
223 GV->eraseFromParent();
227 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
228 if (GV->getName() == "llvm.global_ctors" ||
229 GV->getName() == "llvm.global_dtors")
230 return UpgradeGlobalStructors(GV);
232 // Nothing to do yet.
236 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the
237 // upgraded intrinsic. All argument and return casting must be provided in
238 // order to seamlessly integrate with existing context.
239 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
240 Function *F = CI->getCalledFunction();
241 LLVMContext &C = CI->getContext();
242 IRBuilder<> Builder(C);
243 Builder.SetInsertPoint(CI->getParent(), CI);
245 assert(F && "Intrinsic call is not direct?");
248 // Get the Function's name.
249 StringRef Name = F->getName();
252 // Upgrade packed integer vector compares intrinsics to compare instructions
253 if (Name.startswith("llvm.x86.sse2.pcmpeq.") ||
254 Name.startswith("llvm.x86.avx2.pcmpeq.")) {
255 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1),
257 // need to sign extend since icmp returns vector of i1
258 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
259 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") ||
260 Name.startswith("llvm.x86.avx2.pcmpgt.")) {
261 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1),
263 // need to sign extend since icmp returns vector of i1
264 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
265 } else if (Name == "llvm.x86.avx.movnt.dq.256" ||
266 Name == "llvm.x86.avx.movnt.ps.256" ||
267 Name == "llvm.x86.avx.movnt.pd.256") {
268 IRBuilder<> Builder(C);
269 Builder.SetInsertPoint(CI->getParent(), CI);
271 Module *M = F->getParent();
272 SmallVector<Value *, 1> Elts;
273 Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1));
274 MDNode *Node = MDNode::get(C, Elts);
276 Value *Arg0 = CI->getArgOperand(0);
277 Value *Arg1 = CI->getArgOperand(1);
279 // Convert the type of the pointer to a pointer to the stored type.
280 Value *BC = Builder.CreateBitCast(Arg0,
281 PointerType::getUnqual(Arg1->getType()),
283 StoreInst *SI = Builder.CreateStore(Arg1, BC);
284 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
285 SI->setAlignment(16);
288 CI->eraseFromParent();
290 } else if (Name.startswith("llvm.x86.xop.vpcom")) {
292 if (Name.endswith("ub"))
293 intID = Intrinsic::x86_xop_vpcomub;
294 else if (Name.endswith("uw"))
295 intID = Intrinsic::x86_xop_vpcomuw;
296 else if (Name.endswith("ud"))
297 intID = Intrinsic::x86_xop_vpcomud;
298 else if (Name.endswith("uq"))
299 intID = Intrinsic::x86_xop_vpcomuq;
300 else if (Name.endswith("b"))
301 intID = Intrinsic::x86_xop_vpcomb;
302 else if (Name.endswith("w"))
303 intID = Intrinsic::x86_xop_vpcomw;
304 else if (Name.endswith("d"))
305 intID = Intrinsic::x86_xop_vpcomd;
306 else if (Name.endswith("q"))
307 intID = Intrinsic::x86_xop_vpcomq;
309 llvm_unreachable("Unknown suffix");
311 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom"
313 if (Name.startswith("lt"))
315 else if (Name.startswith("le"))
317 else if (Name.startswith("gt"))
319 else if (Name.startswith("ge"))
321 else if (Name.startswith("eq"))
323 else if (Name.startswith("ne"))
325 else if (Name.startswith("true"))
327 else if (Name.startswith("false"))
330 llvm_unreachable("Unknown condition");
332 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
333 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0),
334 CI->getArgOperand(1), Builder.getInt8(Imm));
335 } else if (Name == "llvm.x86.sse42.crc32.64.8") {
336 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
337 Intrinsic::x86_sse42_crc32_32_8);
338 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
339 Rep = Builder.CreateCall2(CRC32, Trunc0, CI->getArgOperand(1));
340 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
341 } else if (Name.startswith("llvm.x86.avx.vbroadcast")) {
342 // Replace broadcasts with a series of insertelements.
343 Type *VecTy = CI->getType();
344 Type *EltTy = VecTy->getVectorElementType();
345 unsigned EltNum = VecTy->getVectorNumElements();
346 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
347 EltTy->getPointerTo());
348 Value *Load = Builder.CreateLoad(Cast);
349 Type *I32Ty = Type::getInt32Ty(C);
350 Rep = UndefValue::get(VecTy);
351 for (unsigned I = 0; I < EltNum; ++I)
352 Rep = Builder.CreateInsertElement(Rep, Load,
353 ConstantInt::get(I32Ty, I));
355 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false;
356 if (Name == "llvm.x86.avx.vpermil.pd.256")
358 else if (Name == "llvm.x86.avx.vpermil.pd")
360 else if (Name == "llvm.x86.avx.vpermil.ps.256")
362 else if (Name == "llvm.x86.avx.vpermil.ps")
365 if (PD256 || PD128 || PS256 || PS128) {
366 Value *Op0 = CI->getArgOperand(0);
367 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
368 SmallVector<Constant*, 8> Idxs;
371 for (unsigned i = 0; i != 2; ++i)
372 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1));
374 for (unsigned l = 0; l != 4; l+=2)
375 for (unsigned i = 0; i != 2; ++i)
376 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l));
378 for (unsigned i = 0; i != 4; ++i)
379 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3));
381 for (unsigned l = 0; l != 8; l+=4)
382 for (unsigned i = 0; i != 4; ++i)
383 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l));
385 llvm_unreachable("Unexpected function");
387 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs));
389 llvm_unreachable("Unknown function for CallInst upgrade.");
393 CI->replaceAllUsesWith(Rep);
394 CI->eraseFromParent();
398 std::string Name = CI->getName().str();
399 CI->setName(Name + ".old");
401 switch (NewFn->getIntrinsicID()) {
403 llvm_unreachable("Unknown function for CallInst upgrade.");
405 case Intrinsic::ctlz:
406 case Intrinsic::cttz:
407 assert(CI->getNumArgOperands() == 1 &&
408 "Mismatch between function args and call args");
409 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
410 Builder.getFalse(), Name));
411 CI->eraseFromParent();
414 case Intrinsic::objectsize:
415 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn,
416 CI->getArgOperand(0),
417 CI->getArgOperand(1),
419 CI->eraseFromParent();
422 case Intrinsic::arm_neon_vclz: {
423 // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.*
424 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0),
426 "llvm.ctlz." + Name.substr(14)));
427 CI->eraseFromParent();
430 case Intrinsic::ctpop: {
431 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0)));
432 CI->eraseFromParent();
436 case Intrinsic::x86_xop_vfrcz_ss:
437 case Intrinsic::x86_xop_vfrcz_sd:
438 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1),
440 CI->eraseFromParent();
443 case Intrinsic::x86_sse41_ptestc:
444 case Intrinsic::x86_sse41_ptestz:
445 case Intrinsic::x86_sse41_ptestnzc: {
446 // The arguments for these intrinsics used to be v4f32, and changed
447 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
448 // So, the only thing required is a bitcast for both arguments.
449 // First, check the arguments have the old type.
450 Value *Arg0 = CI->getArgOperand(0);
451 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4))
454 // Old intrinsic, add bitcasts
455 Value *Arg1 = CI->getArgOperand(1);
458 Builder.CreateBitCast(Arg0,
459 VectorType::get(Type::getInt64Ty(C), 2),
462 Builder.CreateBitCast(Arg1,
463 VectorType::get(Type::getInt64Ty(C), 2),
466 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name);
467 CI->replaceAllUsesWith(NewCall);
468 CI->eraseFromParent();
474 // This tests each Function to determine if it needs upgrading. When we find
475 // one we are interested in, we then upgrade all calls to reflect the new
477 void llvm::UpgradeCallsToIntrinsic(Function* F) {
478 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
480 // Upgrade the function and check if it is a totaly new function.
482 if (UpgradeIntrinsicFunction(F, NewFn)) {
484 // Replace all uses to the old function with the new one if necessary.
485 for (Value::user_iterator UI = F->user_begin(), UE = F->user_end();
487 if (CallInst *CI = dyn_cast<CallInst>(*UI++))
488 UpgradeIntrinsicCall(CI, NewFn);
490 // Remove old function, no longer used, from the module.
491 F->eraseFromParent();
496 void llvm::UpgradeInstWithTBAATag(Instruction *I) {
497 MDNode *MD = I->getMetadata(LLVMContext::MD_tbaa);
498 assert(MD && "UpgradeInstWithTBAATag should have a TBAA tag");
499 // Check if the tag uses struct-path aware TBAA format.
500 if (isa<MDNode>(MD->getOperand(0)) && MD->getNumOperands() >= 3)
503 if (MD->getNumOperands() == 3) {
508 MDNode *ScalarType = MDNode::get(I->getContext(), Elts);
509 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
511 ScalarType, ScalarType,
512 Constant::getNullValue(Type::getInt64Ty(I->getContext())),
515 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts2));
517 // Create a MDNode <MD, MD, offset 0>
518 Value *Elts[] = {MD, MD,
519 Constant::getNullValue(Type::getInt64Ty(I->getContext()))};
520 I->setMetadata(LLVMContext::MD_tbaa, MDNode::get(I->getContext(), Elts));
524 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
525 Instruction *&Temp) {
526 if (Opc != Instruction::BitCast)
530 Type *SrcTy = V->getType();
531 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
532 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
533 LLVMContext &Context = V->getContext();
535 // We have no information about target data layout, so we assume that
536 // the maximum pointer size is 64bit.
537 Type *MidTy = Type::getInt64Ty(Context);
538 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
540 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
546 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
547 if (Opc != Instruction::BitCast)
550 Type *SrcTy = C->getType();
551 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
552 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
553 LLVMContext &Context = C->getContext();
555 // We have no information about target data layout, so we assume that
556 // the maximum pointer size is 64bit.
557 Type *MidTy = Type::getInt64Ty(Context);
559 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
566 /// Check the debug info version number, if it is out-dated, drop the debug
567 /// info. Return true if module is modified.
568 bool llvm::UpgradeDebugInfo(Module &M) {
569 unsigned Version = getDebugMetadataVersionFromModule(M);
570 if (Version == DEBUG_METADATA_VERSION)
573 bool RetCode = StripDebugInfo(M);
575 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
576 M.getContext().diagnose(DiagVersion);
581 void llvm::UpgradeMDStringConstant(std::string &String) {
582 const std::string OldPrefix = "llvm.vectorizer.";
583 if (String == "llvm.vectorizer.unroll") {
584 String = "llvm.loop.interleave.count";
585 } else if (String.find(OldPrefix) == 0) {
586 String.replace(0, OldPrefix.size(), "llvm.loop.vectorize.");