1 //===- InstCombineVectorOps.cpp -------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements instcombine for ExtractElement, InsertElement and
13 //===----------------------------------------------------------------------===//
15 #include "InstCombine.h"
18 /// CheapToScalarize - Return true if the value is cheaper to scalarize than it
19 /// is to leave as a vector operation.
20 static bool CheapToScalarize(Value *V, bool isConstant) {
21 if (isa<ConstantAggregateZero>(V))
23 if (ConstantVector *C = dyn_cast<ConstantVector>(V)) {
24 if (isConstant) return true;
25 // If all elts are the same, we can extract.
26 Constant *Op0 = C->getOperand(0);
27 for (unsigned i = 1; i < C->getNumOperands(); ++i)
28 if (C->getOperand(i) != Op0)
32 Instruction *I = dyn_cast<Instruction>(V);
35 // Insert element gets simplified to the inserted element or is deleted if
36 // this is constant idx extract element and its a constant idx insertelt.
37 if (I->getOpcode() == Instruction::InsertElement && isConstant &&
38 isa<ConstantInt>(I->getOperand(2)))
40 if (I->getOpcode() == Instruction::Load && I->hasOneUse())
42 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I))
43 if (BO->hasOneUse() &&
44 (CheapToScalarize(BO->getOperand(0), isConstant) ||
45 CheapToScalarize(BO->getOperand(1), isConstant)))
47 if (CmpInst *CI = dyn_cast<CmpInst>(I))
48 if (CI->hasOneUse() &&
49 (CheapToScalarize(CI->getOperand(0), isConstant) ||
50 CheapToScalarize(CI->getOperand(1), isConstant)))
56 /// getShuffleMask - Read and decode a shufflevector mask.
57 /// Turn undef elements into negative values.
58 static std::vector<int> getShuffleMask(const ShuffleVectorInst *SVI) {
59 unsigned NElts = SVI->getType()->getNumElements();
60 if (isa<ConstantAggregateZero>(SVI->getOperand(2)))
61 return std::vector<int>(NElts, 0);
62 if (isa<UndefValue>(SVI->getOperand(2)))
63 return std::vector<int>(NElts, -1);
65 std::vector<int> Result;
66 const ConstantVector *CP = cast<ConstantVector>(SVI->getOperand(2));
67 for (User::const_op_iterator i = CP->op_begin(), e = CP->op_end(); i!=e; ++i)
68 if (isa<UndefValue>(*i))
69 Result.push_back(-1); // undef
71 Result.push_back(cast<ConstantInt>(*i)->getZExtValue());
75 /// FindScalarElement - Given a vector and an element number, see if the scalar
76 /// value is already around as a register, for example if it were inserted then
77 /// extracted from the vector.
78 static Value *FindScalarElement(Value *V, unsigned EltNo) {
79 assert(V->getType()->isVectorTy() && "Not looking at a vector?");
80 const VectorType *PTy = cast<VectorType>(V->getType());
81 unsigned Width = PTy->getNumElements();
82 if (EltNo >= Width) // Out of range access.
83 return UndefValue::get(PTy->getElementType());
85 if (isa<UndefValue>(V))
86 return UndefValue::get(PTy->getElementType());
87 if (isa<ConstantAggregateZero>(V))
88 return Constant::getNullValue(PTy->getElementType());
89 if (ConstantVector *CP = dyn_cast<ConstantVector>(V))
90 return CP->getOperand(EltNo);
92 if (InsertElementInst *III = dyn_cast<InsertElementInst>(V)) {
93 // If this is an insert to a variable element, we don't know what it is.
94 if (!isa<ConstantInt>(III->getOperand(2)))
96 unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
98 // If this is an insert to the element we are looking for, return the
101 return III->getOperand(1);
103 // Otherwise, the insertelement doesn't modify the value, recurse on its
105 return FindScalarElement(III->getOperand(0), EltNo);
108 if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(V)) {
110 cast<VectorType>(SVI->getOperand(0)->getType())->getNumElements();
111 int InEl = getShuffleMask(SVI)[EltNo];
113 return UndefValue::get(PTy->getElementType());
114 if (InEl < (int)LHSWidth)
115 return FindScalarElement(SVI->getOperand(0), InEl);
116 return FindScalarElement(SVI->getOperand(1), InEl - LHSWidth);
119 // Otherwise, we don't know.
123 Instruction *InstCombiner::visitExtractElementInst(ExtractElementInst &EI) {
124 // If vector val is undef, replace extract with scalar undef.
125 if (isa<UndefValue>(EI.getOperand(0)))
126 return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType()));
128 // If vector val is constant 0, replace extract with scalar 0.
129 if (isa<ConstantAggregateZero>(EI.getOperand(0)))
130 return ReplaceInstUsesWith(EI, Constant::getNullValue(EI.getType()));
132 if (ConstantVector *C = dyn_cast<ConstantVector>(EI.getOperand(0))) {
133 // If vector val is constant with all elements the same, replace EI with
134 // that element. When the elements are not identical, we cannot replace yet
135 // (we do that below, but only when the index is constant).
136 Constant *op0 = C->getOperand(0);
137 for (unsigned i = 1; i != C->getNumOperands(); ++i)
138 if (C->getOperand(i) != op0) {
143 return ReplaceInstUsesWith(EI, op0);
146 // If extracting a specified index from the vector, see if we can recursively
147 // find a previously computed scalar that was inserted into the vector.
148 if (ConstantInt *IdxC = dyn_cast<ConstantInt>(EI.getOperand(1))) {
149 unsigned IndexVal = IdxC->getZExtValue();
150 unsigned VectorWidth = EI.getVectorOperandType()->getNumElements();
152 // If this is extracting an invalid index, turn this into undef, to avoid
153 // crashing the code below.
154 if (IndexVal >= VectorWidth)
155 return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType()));
157 // This instruction only demands the single element from the input vector.
158 // If the input vector has a single use, simplify it based on this use
160 if (EI.getOperand(0)->hasOneUse() && VectorWidth != 1) {
161 APInt UndefElts(VectorWidth, 0);
162 APInt DemandedMask(VectorWidth, 0);
163 DemandedMask.set(IndexVal);
164 if (Value *V = SimplifyDemandedVectorElts(EI.getOperand(0),
165 DemandedMask, UndefElts)) {
171 if (Value *Elt = FindScalarElement(EI.getOperand(0), IndexVal))
172 return ReplaceInstUsesWith(EI, Elt);
174 // If the this extractelement is directly using a bitcast from a vector of
175 // the same number of elements, see if we can find the source element from
176 // it. In this case, we will end up needing to bitcast the scalars.
177 if (BitCastInst *BCI = dyn_cast<BitCastInst>(EI.getOperand(0))) {
178 if (const VectorType *VT =
179 dyn_cast<VectorType>(BCI->getOperand(0)->getType()))
180 if (VT->getNumElements() == VectorWidth)
181 if (Value *Elt = FindScalarElement(BCI->getOperand(0), IndexVal))
182 return new BitCastInst(Elt, EI.getType());
186 if (Instruction *I = dyn_cast<Instruction>(EI.getOperand(0))) {
187 // Push extractelement into predecessor operation if legal and
188 // profitable to do so
189 if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
190 if (I->hasOneUse() &&
191 CheapToScalarize(BO, isa<ConstantInt>(EI.getOperand(1)))) {
193 Builder->CreateExtractElement(BO->getOperand(0), EI.getOperand(1),
194 EI.getName()+".lhs");
196 Builder->CreateExtractElement(BO->getOperand(1), EI.getOperand(1),
197 EI.getName()+".rhs");
198 return BinaryOperator::Create(BO->getOpcode(), newEI0, newEI1);
200 } else if (InsertElementInst *IE = dyn_cast<InsertElementInst>(I)) {
201 // Extracting the inserted element?
202 if (IE->getOperand(2) == EI.getOperand(1))
203 return ReplaceInstUsesWith(EI, IE->getOperand(1));
204 // If the inserted and extracted elements are constants, they must not
205 // be the same value, extract from the pre-inserted value instead.
206 if (isa<Constant>(IE->getOperand(2)) && isa<Constant>(EI.getOperand(1))) {
207 Worklist.AddValue(EI.getOperand(0));
208 EI.setOperand(0, IE->getOperand(0));
211 } else if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) {
212 // If this is extracting an element from a shufflevector, figure out where
213 // it came from and extract from the appropriate input element instead.
214 if (ConstantInt *Elt = dyn_cast<ConstantInt>(EI.getOperand(1))) {
215 int SrcIdx = getShuffleMask(SVI)[Elt->getZExtValue()];
218 cast<VectorType>(SVI->getOperand(0)->getType())->getNumElements();
221 return ReplaceInstUsesWith(EI, UndefValue::get(EI.getType()));
222 if (SrcIdx < (int)LHSWidth)
223 Src = SVI->getOperand(0);
226 Src = SVI->getOperand(1);
228 return ExtractElementInst::Create(Src,
229 ConstantInt::get(Type::getInt32Ty(EI.getContext()),
233 // FIXME: Canonicalize extractelement(bitcast) -> bitcast(extractelement)
238 /// CollectSingleShuffleElements - If V is a shuffle of values that ONLY returns
239 /// elements from either LHS or RHS, return the shuffle mask and true.
240 /// Otherwise, return false.
241 static bool CollectSingleShuffleElements(Value *V, Value *LHS, Value *RHS,
242 std::vector<Constant*> &Mask) {
243 assert(V->getType() == LHS->getType() && V->getType() == RHS->getType() &&
244 "Invalid CollectSingleShuffleElements");
245 unsigned NumElts = cast<VectorType>(V->getType())->getNumElements();
247 if (isa<UndefValue>(V)) {
248 Mask.assign(NumElts, UndefValue::get(Type::getInt32Ty(V->getContext())));
253 for (unsigned i = 0; i != NumElts; ++i)
254 Mask.push_back(ConstantInt::get(Type::getInt32Ty(V->getContext()), i));
259 for (unsigned i = 0; i != NumElts; ++i)
260 Mask.push_back(ConstantInt::get(Type::getInt32Ty(V->getContext()),
265 if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(V)) {
266 // If this is an insert of an extract from some other vector, include it.
267 Value *VecOp = IEI->getOperand(0);
268 Value *ScalarOp = IEI->getOperand(1);
269 Value *IdxOp = IEI->getOperand(2);
271 if (!isa<ConstantInt>(IdxOp))
273 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
275 if (isa<UndefValue>(ScalarOp)) { // inserting undef into vector.
276 // Okay, we can handle this if the vector we are insertinting into is
278 if (CollectSingleShuffleElements(VecOp, LHS, RHS, Mask)) {
279 // If so, update the mask to reflect the inserted undef.
280 Mask[InsertedIdx] = UndefValue::get(Type::getInt32Ty(V->getContext()));
283 } else if (ExtractElementInst *EI = dyn_cast<ExtractElementInst>(ScalarOp)){
284 if (isa<ConstantInt>(EI->getOperand(1)) &&
285 EI->getOperand(0)->getType() == V->getType()) {
286 unsigned ExtractedIdx =
287 cast<ConstantInt>(EI->getOperand(1))->getZExtValue();
289 // This must be extracting from either LHS or RHS.
290 if (EI->getOperand(0) == LHS || EI->getOperand(0) == RHS) {
291 // Okay, we can handle this if the vector we are insertinting into is
293 if (CollectSingleShuffleElements(VecOp, LHS, RHS, Mask)) {
294 // If so, update the mask to reflect the inserted value.
295 if (EI->getOperand(0) == LHS) {
296 Mask[InsertedIdx % NumElts] =
297 ConstantInt::get(Type::getInt32Ty(V->getContext()),
300 assert(EI->getOperand(0) == RHS);
301 Mask[InsertedIdx % NumElts] =
302 ConstantInt::get(Type::getInt32Ty(V->getContext()),
303 ExtractedIdx+NumElts);
312 // TODO: Handle shufflevector here!
317 /// CollectShuffleElements - We are building a shuffle of V, using RHS as the
318 /// RHS of the shuffle instruction, if it is not null. Return a shuffle mask
319 /// that computes V and the LHS value of the shuffle.
320 static Value *CollectShuffleElements(Value *V, std::vector<Constant*> &Mask,
322 assert(V->getType()->isVectorTy() &&
323 (RHS == 0 || V->getType() == RHS->getType()) &&
325 unsigned NumElts = cast<VectorType>(V->getType())->getNumElements();
327 if (isa<UndefValue>(V)) {
328 Mask.assign(NumElts, UndefValue::get(Type::getInt32Ty(V->getContext())));
330 } else if (isa<ConstantAggregateZero>(V)) {
331 Mask.assign(NumElts, ConstantInt::get(Type::getInt32Ty(V->getContext()),0));
333 } else if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(V)) {
334 // If this is an insert of an extract from some other vector, include it.
335 Value *VecOp = IEI->getOperand(0);
336 Value *ScalarOp = IEI->getOperand(1);
337 Value *IdxOp = IEI->getOperand(2);
339 if (ExtractElementInst *EI = dyn_cast<ExtractElementInst>(ScalarOp)) {
340 if (isa<ConstantInt>(EI->getOperand(1)) && isa<ConstantInt>(IdxOp) &&
341 EI->getOperand(0)->getType() == V->getType()) {
342 unsigned ExtractedIdx =
343 cast<ConstantInt>(EI->getOperand(1))->getZExtValue();
344 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
346 // Either the extracted from or inserted into vector must be RHSVec,
347 // otherwise we'd end up with a shuffle of three inputs.
348 if (EI->getOperand(0) == RHS || RHS == 0) {
349 RHS = EI->getOperand(0);
350 Value *V = CollectShuffleElements(VecOp, Mask, RHS);
351 Mask[InsertedIdx % NumElts] =
352 ConstantInt::get(Type::getInt32Ty(V->getContext()),
353 NumElts+ExtractedIdx);
358 Value *V = CollectShuffleElements(EI->getOperand(0), Mask, RHS);
359 // Everything but the extracted element is replaced with the RHS.
360 for (unsigned i = 0; i != NumElts; ++i) {
361 if (i != InsertedIdx)
362 Mask[i] = ConstantInt::get(Type::getInt32Ty(V->getContext()),
368 // If this insertelement is a chain that comes from exactly these two
369 // vectors, return the vector and the effective shuffle.
370 if (CollectSingleShuffleElements(IEI, EI->getOperand(0), RHS, Mask))
371 return EI->getOperand(0);
375 // TODO: Handle shufflevector here!
377 // Otherwise, can't do anything fancy. Return an identity vector.
378 for (unsigned i = 0; i != NumElts; ++i)
379 Mask.push_back(ConstantInt::get(Type::getInt32Ty(V->getContext()), i));
383 Instruction *InstCombiner::visitInsertElementInst(InsertElementInst &IE) {
384 Value *VecOp = IE.getOperand(0);
385 Value *ScalarOp = IE.getOperand(1);
386 Value *IdxOp = IE.getOperand(2);
388 // Inserting an undef or into an undefined place, remove this.
389 if (isa<UndefValue>(ScalarOp) || isa<UndefValue>(IdxOp))
390 ReplaceInstUsesWith(IE, VecOp);
392 // If the inserted element was extracted from some other vector, and if the
393 // indexes are constant, try to turn this into a shufflevector operation.
394 if (ExtractElementInst *EI = dyn_cast<ExtractElementInst>(ScalarOp)) {
395 if (isa<ConstantInt>(EI->getOperand(1)) && isa<ConstantInt>(IdxOp) &&
396 EI->getOperand(0)->getType() == IE.getType()) {
397 unsigned NumVectorElts = IE.getType()->getNumElements();
398 unsigned ExtractedIdx =
399 cast<ConstantInt>(EI->getOperand(1))->getZExtValue();
400 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
402 if (ExtractedIdx >= NumVectorElts) // Out of range extract.
403 return ReplaceInstUsesWith(IE, VecOp);
405 if (InsertedIdx >= NumVectorElts) // Out of range insert.
406 return ReplaceInstUsesWith(IE, UndefValue::get(IE.getType()));
408 // If we are extracting a value from a vector, then inserting it right
409 // back into the same place, just use the input vector.
410 if (EI->getOperand(0) == VecOp && ExtractedIdx == InsertedIdx)
411 return ReplaceInstUsesWith(IE, VecOp);
413 // If this insertelement isn't used by some other insertelement, turn it
414 // (and any insertelements it points to), into one big shuffle.
415 if (!IE.hasOneUse() || !isa<InsertElementInst>(IE.use_back())) {
416 std::vector<Constant*> Mask;
418 Value *LHS = CollectShuffleElements(&IE, Mask, RHS);
419 if (RHS == 0) RHS = UndefValue::get(LHS->getType());
420 // We now have a shuffle of LHS, RHS, Mask.
421 return new ShuffleVectorInst(LHS, RHS,
422 ConstantVector::get(Mask));
427 unsigned VWidth = cast<VectorType>(VecOp->getType())->getNumElements();
428 APInt UndefElts(VWidth, 0);
429 APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
430 if (SimplifyDemandedVectorElts(&IE, AllOnesEltMask, UndefElts))
437 Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
438 Value *LHS = SVI.getOperand(0);
439 Value *RHS = SVI.getOperand(1);
440 std::vector<int> Mask = getShuffleMask(&SVI);
442 bool MadeChange = false;
444 // Undefined shuffle mask -> undefined value.
445 if (isa<UndefValue>(SVI.getOperand(2)))
446 return ReplaceInstUsesWith(SVI, UndefValue::get(SVI.getType()));
448 unsigned VWidth = cast<VectorType>(SVI.getType())->getNumElements();
450 if (VWidth != cast<VectorType>(LHS->getType())->getNumElements())
453 APInt UndefElts(VWidth, 0);
454 APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
455 if (SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, UndefElts)) {
456 LHS = SVI.getOperand(0);
457 RHS = SVI.getOperand(1);
461 // Canonicalize shuffle(x ,x,mask) -> shuffle(x, undef,mask')
462 // Canonicalize shuffle(undef,x,mask) -> shuffle(x, undef,mask').
463 if (LHS == RHS || isa<UndefValue>(LHS)) {
464 if (isa<UndefValue>(LHS) && LHS == RHS) {
465 // shuffle(undef,undef,mask) -> undef.
466 return ReplaceInstUsesWith(SVI, LHS);
469 // Remap any references to RHS to use LHS.
470 std::vector<Constant*> Elts;
471 for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
473 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext())));
475 if ((Mask[i] >= (int)e && isa<UndefValue>(RHS)) ||
476 (Mask[i] < (int)e && isa<UndefValue>(LHS))) {
477 Mask[i] = -1; // Turn into undef.
478 Elts.push_back(UndefValue::get(Type::getInt32Ty(SVI.getContext())));
480 Mask[i] = Mask[i] % e; // Force to LHS.
481 Elts.push_back(ConstantInt::get(Type::getInt32Ty(SVI.getContext()),
486 SVI.setOperand(0, SVI.getOperand(1));
487 SVI.setOperand(1, UndefValue::get(RHS->getType()));
488 SVI.setOperand(2, ConstantVector::get(Elts));
489 LHS = SVI.getOperand(0);
490 RHS = SVI.getOperand(1);
494 // Analyze the shuffle, are the LHS or RHS and identity shuffles?
495 bool isLHSID = true, isRHSID = true;
497 for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
498 if (Mask[i] < 0) continue; // Ignore undef values.
499 // Is this an identity shuffle of the LHS value?
500 isLHSID &= (Mask[i] == (int)i);
502 // Is this an identity shuffle of the RHS value?
503 isRHSID &= (Mask[i]-e == i);
506 // Eliminate identity shuffles.
507 if (isLHSID) return ReplaceInstUsesWith(SVI, LHS);
508 if (isRHSID) return ReplaceInstUsesWith(SVI, RHS);
510 // If the LHS is a shufflevector itself, see if we can combine it with this
511 // one without producing an unusual shuffle. Here we are really conservative:
512 // we are absolutely afraid of producing a shuffle mask not in the input
513 // program, because the code gen may not be smart enough to turn a merged
514 // shuffle into two specific shuffles: it may produce worse code. As such,
515 // we only merge two shuffles if the result is either a splat or one of the
516 // two input shuffle masks. In this case, merging the shuffles just removes
517 // one instruction, which we know is safe. This is good for things like
518 // turning: (splat(splat)) -> splat.
519 if (ShuffleVectorInst *LHSSVI = dyn_cast<ShuffleVectorInst>(LHS)) {
520 if (isa<UndefValue>(RHS)) {
521 std::vector<int> LHSMask = getShuffleMask(LHSSVI);
523 if (LHSMask.size() == Mask.size()) {
524 std::vector<int> NewMask;
526 int SplatElt = -1; // undef
527 for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
529 if (Mask[i] < 0 || Mask[i] >= (int)e)
530 MaskElt = -1; // undef
532 MaskElt = LHSMask[Mask[i]];
533 // Check if this could still be a splat.
535 if (SplatElt >=0 && SplatElt != MaskElt)
539 NewMask.push_back(MaskElt);
542 // If the result mask is equal to the src shuffle or this
543 // shuffle mask, do the replacement.
544 if (isSplat || NewMask == LHSMask || NewMask == Mask) {
545 std::vector<Constant*> Elts;
546 const Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
547 for (unsigned i = 0, e = NewMask.size(); i != e; ++i) {
548 if (NewMask[i] < 0) {
549 Elts.push_back(UndefValue::get(Int32Ty));
551 Elts.push_back(ConstantInt::get(Int32Ty, NewMask[i]));
554 return new ShuffleVectorInst(LHSSVI->getOperand(0),
555 LHSSVI->getOperand(1),
556 ConstantVector::get(Elts));
562 return MadeChange ? &SVI : 0;