2 //===-- ARM64PromoteConstant.cpp --- Promote constant to global for ARM64 -===//
4 // The LLVM Compiler Infrastructure
6 // This file is distributed under the University of Illinois Open Source
7 // License. See LICENSE.TXT for details.
9 //===----------------------------------------------------------------------===//
11 // This file implements the ARM64PromoteConstant pass which promotes constants
12 // to global variables when this is likely to be more efficient. Currently only
13 // types related to constant vector (i.e., constant vector, array of constant
14 // vectors, constant structure with a constant vector field, etc.) are promoted
15 // to global variables. Constant vectors are likely to be lowered in target
16 // constant pool during instruction selection already; therefore, the access
17 // will remain the same (memory load), but the structure types are not split
18 // into different constant pool accesses for each field. A bonus side effect is
19 // that created globals may be merged by the global merge pass.
21 // FIXME: This pass may be useful for other targets too.
22 //===----------------------------------------------------------------------===//
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/ADT/DenseMap.h"
27 #include "llvm/ADT/SmallSet.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/IR/Constants.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalVariable.h"
33 #include "llvm/IR/InlineAsm.h"
34 #include "llvm/IR/Instructions.h"
35 #include "llvm/IR/IntrinsicInst.h"
36 #include "llvm/IR/IRBuilder.h"
37 #include "llvm/IR/Module.h"
38 #include "llvm/Pass.h"
39 #include "llvm/Support/CommandLine.h"
40 #include "llvm/Support/Debug.h"
44 #define DEBUG_TYPE "arm64-promote-const"
46 // Stress testing mode - disable heuristics.
47 static cl::opt<bool> Stress("arm64-stress-promote-const", cl::Hidden,
48 cl::desc("Promote all vector constants"));
50 STATISTIC(NumPromoted, "Number of promoted constants");
51 STATISTIC(NumPromotedUses, "Number of promoted constants uses");
53 //===----------------------------------------------------------------------===//
54 // ARM64PromoteConstant
55 //===----------------------------------------------------------------------===//
58 /// Promotes interesting constant into global variables.
59 /// The motivating example is:
60 /// static const uint16_t TableA[32] = {
61 /// 41944, 40330, 38837, 37450, 36158, 34953, 33826, 32768,
62 /// 31776, 30841, 29960, 29128, 28340, 27595, 26887, 26215,
63 /// 25576, 24967, 24386, 23832, 23302, 22796, 22311, 21846,
64 /// 21400, 20972, 20561, 20165, 19785, 19419, 19066, 18725,
67 /// uint8x16x4_t LoadStatic(void) {
69 /// ret.val[0] = vld1q_u16(TableA + 0);
70 /// ret.val[1] = vld1q_u16(TableA + 8);
71 /// ret.val[2] = vld1q_u16(TableA + 16);
72 /// ret.val[3] = vld1q_u16(TableA + 24);
76 /// The constants in this example are folded into the uses. Thus, 4 different
77 /// constants are created.
79 /// As their type is vector the cheapest way to create them is to load them
82 /// Therefore the final assembly final has 4 different loads. With this pass
83 /// enabled, only one load is issued for the constants.
84 class ARM64PromoteConstant : public ModulePass {
88 ARM64PromoteConstant() : ModulePass(ID) {}
90 virtual const char *getPassName() const { return "ARM64 Promote Constant"; }
92 /// Iterate over the functions and promote the interesting constants into
93 /// global variables with module scope.
94 bool runOnModule(Module &M) {
95 DEBUG(dbgs() << getPassName() << '\n');
98 Changed |= runOnFunction(MF);
104 /// Look for interesting constants used within the given function.
105 /// Promote them into global variables, load these global variables within
106 /// the related function, so that the number of inserted load is minimal.
107 bool runOnFunction(Function &F);
109 // This transformation requires dominator info
110 virtual void getAnalysisUsage(AnalysisUsage &AU) const {
111 AU.setPreservesCFG();
112 AU.addRequired<DominatorTreeWrapperPass>();
113 AU.addPreserved<DominatorTreeWrapperPass>();
116 /// Type to store a list of User.
117 typedef SmallVector<Value::user_iterator, 4> Users;
118 /// Map an insertion point to all the uses it dominates.
119 typedef DenseMap<Instruction *, Users> InsertionPoints;
120 /// Map a function to the required insertion point of load for a
122 typedef DenseMap<Function *, InsertionPoints> InsertionPointsPerFunc;
124 /// Find the closest point that dominates the given Use.
125 Instruction *findInsertionPoint(Value::user_iterator &Use);
127 /// Check if the given insertion point is dominated by an existing
129 /// If true, the given use is added to the list of dominated uses for
130 /// the related existing point.
131 /// \param NewPt the insertion point to be checked
132 /// \param UseIt the use to be added into the list of dominated uses
133 /// \param InsertPts existing insertion points
134 /// \pre NewPt and all instruction in InsertPts belong to the same function
135 /// \return true if one of the insertion point in InsertPts dominates NewPt,
137 bool isDominated(Instruction *NewPt, Value::user_iterator &UseIt,
138 InsertionPoints &InsertPts);
140 /// Check if the given insertion point can be merged with an existing
141 /// insertion point in a common dominator.
142 /// If true, the given use is added to the list of the created insertion
144 /// \param NewPt the insertion point to be checked
145 /// \param UseIt the use to be added into the list of dominated uses
146 /// \param InsertPts existing insertion points
147 /// \pre NewPt and all instruction in InsertPts belong to the same function
148 /// \pre isDominated returns false for the exact same parameters.
149 /// \return true if it exists an insertion point in InsertPts that could
150 /// have been merged with NewPt in a common dominator,
152 bool tryAndMerge(Instruction *NewPt, Value::user_iterator &UseIt,
153 InsertionPoints &InsertPts);
155 /// Compute the minimal insertion points to dominates all the interesting
157 /// Insertion points are group per function and each insertion point
158 /// contains a list of all the uses it dominates within the related function
159 /// \param Val constant to be examined
160 /// \param[out] InsPtsPerFunc output storage of the analysis
161 void computeInsertionPoints(Constant *Val,
162 InsertionPointsPerFunc &InsPtsPerFunc);
164 /// Insert a definition of a new global variable at each point contained in
165 /// InsPtsPerFunc and update the related uses (also contained in
167 bool insertDefinitions(Constant *Cst, InsertionPointsPerFunc &InsPtsPerFunc);
169 /// Compute the minimal insertion points to dominate all the interesting
170 /// uses of Val and insert a definition of a new global variable
172 /// Also update the uses of Val accordingly.
173 /// Currently a use of Val is considered interesting if:
174 /// - Val is not UndefValue
175 /// - Val is not zeroinitialized
176 /// - Replacing Val per a load of a global variable is valid.
177 /// \see shouldConvert for more details
178 bool computeAndInsertDefinitions(Constant *Val);
180 /// Promote the given constant into a global variable if it is expected to
182 /// \return true if Cst has been promoted
183 bool promoteConstant(Constant *Cst);
185 /// Transfer the list of dominated uses of IPI to NewPt in InsertPts.
186 /// Append UseIt to this list and delete the entry of IPI in InsertPts.
187 static void appendAndTransferDominatedUses(Instruction *NewPt,
188 Value::user_iterator &UseIt,
189 InsertionPoints::iterator &IPI,
190 InsertionPoints &InsertPts) {
191 // Record the dominated use.
192 IPI->second.push_back(UseIt);
193 // Transfer the dominated uses of IPI to NewPt
194 // Inserting into the DenseMap may invalidate existing iterator.
195 // Keep a copy of the key to find the iterator to erase.
196 Instruction *OldInstr = IPI->first;
197 InsertPts.insert(InsertionPoints::value_type(NewPt, IPI->second));
199 IPI = InsertPts.find(OldInstr);
200 InsertPts.erase(IPI);
203 } // end anonymous namespace
205 char ARM64PromoteConstant::ID = 0;
208 void initializeARM64PromoteConstantPass(PassRegistry &);
211 INITIALIZE_PASS_BEGIN(ARM64PromoteConstant, "arm64-promote-const",
212 "ARM64 Promote Constant Pass", false, false)
213 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
214 INITIALIZE_PASS_END(ARM64PromoteConstant, "arm64-promote-const",
215 "ARM64 Promote Constant Pass", false, false)
217 ModulePass *llvm::createARM64PromoteConstantPass() {
218 return new ARM64PromoteConstant();
221 /// Check if the given type uses a vector type.
222 static bool isConstantUsingVectorTy(const Type *CstTy) {
223 if (CstTy->isVectorTy())
225 if (CstTy->isStructTy()) {
226 for (unsigned EltIdx = 0, EndEltIdx = CstTy->getStructNumElements();
227 EltIdx < EndEltIdx; ++EltIdx)
228 if (isConstantUsingVectorTy(CstTy->getStructElementType(EltIdx)))
230 } else if (CstTy->isArrayTy())
231 return isConstantUsingVectorTy(CstTy->getArrayElementType());
235 /// Check if the given use (Instruction + OpIdx) of Cst should be converted into
236 /// a load of a global variable initialized with Cst.
237 /// A use should be converted if it is legal to do so.
238 /// For instance, it is not legal to turn the mask operand of a shuffle vector
239 /// into a load of a global variable.
240 static bool shouldConvertUse(const Constant *Cst, const Instruction *Instr,
242 // shufflevector instruction expects a const for the mask argument, i.e., the
243 // third argument. Do not promote this use in that case.
244 if (isa<const ShuffleVectorInst>(Instr) && OpIdx == 2)
247 // extractvalue instruction expects a const idx.
248 if (isa<const ExtractValueInst>(Instr) && OpIdx > 0)
251 // extractvalue instruction expects a const idx.
252 if (isa<const InsertValueInst>(Instr) && OpIdx > 1)
255 if (isa<const AllocaInst>(Instr) && OpIdx > 0)
258 // Alignment argument must be constant.
259 if (isa<const LoadInst>(Instr) && OpIdx > 0)
262 // Alignment argument must be constant.
263 if (isa<const StoreInst>(Instr) && OpIdx > 1)
266 // Index must be constant.
267 if (isa<const GetElementPtrInst>(Instr) && OpIdx > 0)
270 // Personality function and filters must be constant.
271 // Give up on that instruction.
272 if (isa<const LandingPadInst>(Instr))
275 // Switch instruction expects constants to compare to.
276 if (isa<const SwitchInst>(Instr))
279 // Expected address must be a constant.
280 if (isa<const IndirectBrInst>(Instr))
283 // Do not mess with intrinsics.
284 if (isa<const IntrinsicInst>(Instr))
287 // Do not mess with inline asm.
288 const CallInst *CI = dyn_cast<const CallInst>(Instr);
289 if (CI && isa<const InlineAsm>(CI->getCalledValue()))
295 /// Check if the given Cst should be converted into
296 /// a load of a global variable initialized with Cst.
297 /// A constant should be converted if it is likely that the materialization of
298 /// the constant will be tricky. Thus, we give up on zero or undef values.
300 /// \todo Currently, accept only vector related types.
301 /// Also we give up on all simple vector type to keep the existing
302 /// behavior. Otherwise, we should push here all the check of the lowering of
303 /// BUILD_VECTOR. By giving up, we lose the potential benefit of merging
304 /// constant via global merge and the fact that the same constant is stored
305 /// only once with this method (versus, as many function that uses the constant
306 /// for the regular approach, even for float).
307 /// Again, the simplest solution would be to promote every
308 /// constant and rematerialize them when they are actually cheap to create.
309 static bool shouldConvert(const Constant *Cst) {
310 if (isa<const UndefValue>(Cst))
313 // FIXME: In some cases, it may be interesting to promote in memory
314 // a zero initialized constant.
315 // E.g., when the type of Cst require more instructions than the
316 // adrp/add/load sequence or when this sequence can be shared by several
318 // Ideally, we could promote this into a global and rematerialize the constant
319 // when it was a bad idea.
320 if (Cst->isZeroValue())
326 // FIXME: see function \todo
327 if (Cst->getType()->isVectorTy())
329 return isConstantUsingVectorTy(Cst->getType());
333 ARM64PromoteConstant::findInsertionPoint(Value::user_iterator &Use) {
334 // If this user is a phi, the insertion point is in the related
335 // incoming basic block.
336 PHINode *PhiInst = dyn_cast<PHINode>(*Use);
337 Instruction *InsertionPoint;
340 PhiInst->getIncomingBlock(Use.getOperandNo())->getTerminator();
342 InsertionPoint = dyn_cast<Instruction>(*Use);
343 assert(InsertionPoint && "User is not an instruction!");
344 return InsertionPoint;
347 bool ARM64PromoteConstant::isDominated(Instruction *NewPt,
348 Value::user_iterator &UseIt,
349 InsertionPoints &InsertPts) {
351 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(
352 *NewPt->getParent()->getParent()).getDomTree();
354 // Traverse all the existing insertion points and check if one is dominating
355 // NewPt. If it is, remember that.
356 for (auto &IPI : InsertPts) {
357 if (NewPt == IPI.first || DT.dominates(IPI.first, NewPt) ||
358 // When IPI.first is a terminator instruction, DT may think that
359 // the result is defined on the edge.
360 // Here we are testing the insertion point, not the definition.
361 (IPI.first->getParent() != NewPt->getParent() &&
362 DT.dominates(IPI.first->getParent(), NewPt->getParent()))) {
363 // No need to insert this point. Just record the dominated use.
364 DEBUG(dbgs() << "Insertion point dominated by:\n");
365 DEBUG(IPI.first->print(dbgs()));
366 DEBUG(dbgs() << '\n');
367 IPI.second.push_back(UseIt);
374 bool ARM64PromoteConstant::tryAndMerge(Instruction *NewPt,
375 Value::user_iterator &UseIt,
376 InsertionPoints &InsertPts) {
377 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(
378 *NewPt->getParent()->getParent()).getDomTree();
379 BasicBlock *NewBB = NewPt->getParent();
381 // Traverse all the existing insertion point and check if one is dominated by
382 // NewPt and thus useless or can be combined with NewPt into a common
384 for (InsertionPoints::iterator IPI = InsertPts.begin(),
385 EndIPI = InsertPts.end();
386 IPI != EndIPI; ++IPI) {
387 BasicBlock *CurBB = IPI->first->getParent();
388 if (NewBB == CurBB) {
389 // Instructions are in the same block.
390 // By construction, NewPt is dominating the other.
391 // Indeed, isDominated returned false with the exact same arguments.
392 DEBUG(dbgs() << "Merge insertion point with:\n");
393 DEBUG(IPI->first->print(dbgs()));
394 DEBUG(dbgs() << "\nat considered insertion point.\n");
395 appendAndTransferDominatedUses(NewPt, UseIt, IPI, InsertPts);
399 // Look for a common dominator
400 BasicBlock *CommonDominator = DT.findNearestCommonDominator(NewBB, CurBB);
401 // If none exists, we cannot merge these two points.
402 if (!CommonDominator)
405 if (CommonDominator != NewBB) {
406 // By construction, the CommonDominator cannot be CurBB.
407 assert(CommonDominator != CurBB &&
408 "Instruction has not been rejected during isDominated check!");
409 // Take the last instruction of the CommonDominator as insertion point
410 NewPt = CommonDominator->getTerminator();
412 // else, CommonDominator is the block of NewBB, hence NewBB is the last
413 // possible insertion point in that block.
414 DEBUG(dbgs() << "Merge insertion point with:\n");
415 DEBUG(IPI->first->print(dbgs()));
416 DEBUG(dbgs() << '\n');
417 DEBUG(NewPt->print(dbgs()));
418 DEBUG(dbgs() << '\n');
419 appendAndTransferDominatedUses(NewPt, UseIt, IPI, InsertPts);
425 void ARM64PromoteConstant::computeInsertionPoints(
426 Constant *Val, InsertionPointsPerFunc &InsPtsPerFunc) {
427 DEBUG(dbgs() << "** Compute insertion points **\n");
428 for (Value::user_iterator UseIt = Val->user_begin(),
429 EndUseIt = Val->user_end();
430 UseIt != EndUseIt; ++UseIt) {
431 // If the user is not an Instruction, we cannot modify it.
432 if (!isa<Instruction>(*UseIt))
435 // Filter out uses that should not be converted.
436 if (!shouldConvertUse(Val, cast<Instruction>(*UseIt), UseIt.getOperandNo()))
439 DEBUG(dbgs() << "Considered use, opidx " << UseIt.getOperandNo() << ":\n");
440 DEBUG((*UseIt)->print(dbgs()));
441 DEBUG(dbgs() << '\n');
443 Instruction *InsertionPoint = findInsertionPoint(UseIt);
445 DEBUG(dbgs() << "Considered insertion point:\n");
446 DEBUG(InsertionPoint->print(dbgs()));
447 DEBUG(dbgs() << '\n');
449 // Check if the current insertion point is useless, i.e., it is dominated
451 InsertionPoints &InsertPts =
452 InsPtsPerFunc[InsertionPoint->getParent()->getParent()];
453 if (isDominated(InsertionPoint, UseIt, InsertPts))
455 // This insertion point is useful, check if we can merge some insertion
456 // point in a common dominator or if NewPt dominates an existing one.
457 if (tryAndMerge(InsertionPoint, UseIt, InsertPts))
460 DEBUG(dbgs() << "Keep considered insertion point\n");
462 // It is definitely useful by its own
463 InsertPts[InsertionPoint].push_back(UseIt);
468 ARM64PromoteConstant::insertDefinitions(Constant *Cst,
469 InsertionPointsPerFunc &InsPtsPerFunc) {
470 // We will create one global variable per Module.
471 DenseMap<Module *, GlobalVariable *> ModuleToMergedGV;
472 bool HasChanged = false;
474 // Traverse all insertion points in all the function.
475 for (InsertionPointsPerFunc::iterator FctToInstPtsIt = InsPtsPerFunc.begin(),
476 EndIt = InsPtsPerFunc.end();
477 FctToInstPtsIt != EndIt; ++FctToInstPtsIt) {
478 InsertionPoints &InsertPts = FctToInstPtsIt->second;
479 // Do more checking for debug purposes.
481 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(
482 *FctToInstPtsIt->first).getDomTree();
484 GlobalVariable *PromotedGV;
485 assert(!InsertPts.empty() && "Empty uses does not need a definition");
487 Module *M = FctToInstPtsIt->first->getParent();
488 DenseMap<Module *, GlobalVariable *>::iterator MapIt =
489 ModuleToMergedGV.find(M);
490 if (MapIt == ModuleToMergedGV.end()) {
491 PromotedGV = new GlobalVariable(
492 *M, Cst->getType(), true, GlobalValue::InternalLinkage, nullptr,
493 "_PromotedConst", nullptr, GlobalVariable::NotThreadLocal);
494 PromotedGV->setInitializer(Cst);
495 ModuleToMergedGV[M] = PromotedGV;
496 DEBUG(dbgs() << "Global replacement: ");
497 DEBUG(PromotedGV->print(dbgs()));
498 DEBUG(dbgs() << '\n');
502 PromotedGV = MapIt->second;
505 for (InsertionPoints::iterator IPI = InsertPts.begin(),
506 EndIPI = InsertPts.end();
507 IPI != EndIPI; ++IPI) {
508 // Create the load of the global variable.
509 IRBuilder<> Builder(IPI->first->getParent(), IPI->first);
510 LoadInst *LoadedCst = Builder.CreateLoad(PromotedGV);
511 DEBUG(dbgs() << "**********\n");
512 DEBUG(dbgs() << "New def: ");
513 DEBUG(LoadedCst->print(dbgs()));
514 DEBUG(dbgs() << '\n');
516 // Update the dominated uses.
517 Users &DominatedUsers = IPI->second;
518 for (Value::user_iterator Use : DominatedUsers) {
520 assert((DT.dominates(LoadedCst, cast<Instruction>(*Use)) ||
521 (isa<PHINode>(*Use) &&
522 DT.dominates(LoadedCst, findInsertionPoint(Use)))) &&
523 "Inserted definition does not dominate all its uses!");
525 DEBUG(dbgs() << "Use to update " << Use.getOperandNo() << ":");
526 DEBUG(Use->print(dbgs()));
527 DEBUG(dbgs() << '\n');
528 Use->setOperand(Use.getOperandNo(), LoadedCst);
536 bool ARM64PromoteConstant::computeAndInsertDefinitions(Constant *Val) {
537 InsertionPointsPerFunc InsertPtsPerFunc;
538 computeInsertionPoints(Val, InsertPtsPerFunc);
539 return insertDefinitions(Val, InsertPtsPerFunc);
542 bool ARM64PromoteConstant::promoteConstant(Constant *Cst) {
543 assert(Cst && "Given variable is not a valid constant.");
545 if (!shouldConvert(Cst))
548 DEBUG(dbgs() << "******************************\n");
549 DEBUG(dbgs() << "Candidate constant: ");
550 DEBUG(Cst->print(dbgs()));
551 DEBUG(dbgs() << '\n');
553 return computeAndInsertDefinitions(Cst);
556 bool ARM64PromoteConstant::runOnFunction(Function &F) {
557 // Look for instructions using constant vector. Promote that constant to a
558 // global variable. Create as few loads of this variable as possible and
559 // update the uses accordingly.
560 bool LocalChange = false;
561 SmallSet<Constant *, 8> AlreadyChecked;
563 for (auto &MBB : F) {
564 for (auto &MI : MBB) {
565 // Traverse the operand, looking for constant vectors. Replace them by a
566 // load of a global variable of constant vector type.
567 for (unsigned OpIdx = 0, EndOpIdx = MI.getNumOperands();
568 OpIdx != EndOpIdx; ++OpIdx) {
569 Constant *Cst = dyn_cast<Constant>(MI.getOperand(OpIdx));
570 // There is no point in promoting global values as they are already
571 // global. Do not promote constant expressions either, as they may
572 // require some code expansion.
573 if (Cst && !isa<GlobalValue>(Cst) && !isa<ConstantExpr>(Cst) &&
574 AlreadyChecked.insert(Cst))
575 LocalChange |= promoteConstant(Cst);