}
bool ScalarTargetTransformImpl::isLegalAddressingMode(const AddrMode &AM,
- Type *Ty) const {
+ Type *Ty) const {
return TLI->isLegalAddressingMode(AM, Ty);
}
return TLI->getJumpBufSize();
}
+bool ScalarTargetTransformImpl::shouldBuildLookupTables() const {
+ return TLI->supportJumpTables() &&
+ (TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+ TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
+}
+
//===----------------------------------------------------------------------===//
//
// Calls used by the vectorizers.
//
//===----------------------------------------------------------------------===//
-int InstructionOpcodeToISD(unsigned Opcode) {
- static const int OpToISDTbl[] = {
- /*Instruction::Ret */ 0, // Opcode numbering start at #1.
- /*Instruction::Br */ 0,
- /*Instruction::Switch */ 0,
- /*Instruction::IndirectBr */ 0,
- /*Instruction::Invoke */ 0,
- /*Instruction::Resume */ 0,
- /*Instruction::Unreachable */ 0,
- /*Instruction::Add */ ISD::ADD,
- /*Instruction::FAdd */ ISD::FADD,
- /*Instruction::Sub */ ISD::SUB,
- /*Instruction::FSub */ ISD::FSUB,
- /*Instruction::Mul */ ISD::MUL,
- /*Instruction::FMul */ ISD::FMUL,
- /*Instruction::UDiv */ ISD::UDIV,
- /*Instruction::SDiv */ ISD::UDIV,
- /*Instruction::FDiv */ ISD::FDIV,
- /*Instruction::URem */ ISD::UREM,
- /*Instruction::SRem */ ISD::SREM,
- /*Instruction::FRem */ ISD::FREM,
- /*Instruction::Shl */ ISD::SHL,
- /*Instruction::LShr */ ISD::SRL,
- /*Instruction::AShr */ ISD::SRA,
- /*Instruction::And */ ISD::AND,
- /*Instruction::Or */ ISD::OR,
- /*Instruction::Xor */ ISD::XOR,
- /*Instruction::Alloca */ 0,
- /*Instruction::Load */ ISD::LOAD,
- /*Instruction::Store */ ISD::STORE,
- /*Instruction::GetElementPtr */ 0,
- /*Instruction::Fence */ 0,
- /*Instruction::AtomicCmpXchg */ 0,
- /*Instruction::AtomicRMW */ 0,
- /*Instruction::Trunc */ ISD::TRUNCATE,
- /*Instruction::ZExt */ ISD::ZERO_EXTEND,
- /*Instruction::SExt */ ISD::SEXTLOAD,
- /*Instruction::FPToUI */ ISD::FP_TO_UINT,
- /*Instruction::FPToSI */ ISD::FP_TO_SINT,
- /*Instruction::UIToFP */ ISD::UINT_TO_FP,
- /*Instruction::SIToFP */ ISD::SINT_TO_FP,
- /*Instruction::FPTrunc */ ISD::FP_ROUND,
- /*Instruction::FPExt */ ISD::FP_EXTEND,
- /*Instruction::PtrToInt */ ISD::BITCAST,
- /*Instruction::IntToPtr */ ISD::BITCAST,
- /*Instruction::BitCast */ ISD::BITCAST,
- /*Instruction::ICmp */ ISD::SETCC,
- /*Instruction::FCmp */ ISD::SETCC,
- /*Instruction::PHI */ 0,
- /*Instruction::Call */ 0,
- /*Instruction::Select */ ISD::SELECT,
- /*Instruction::UserOp1 */ 0,
- /*Instruction::UserOp2 */ 0,
- /*Instruction::VAArg */ 0,
- /*Instruction::ExtractElement*/ ISD::EXTRACT_VECTOR_ELT,
- /*Instruction::InsertElement */ ISD::INSERT_VECTOR_ELT,
- /*Instruction::ShuffleVector */ ISD::VECTOR_SHUFFLE,
- /*Instruction::ExtractValue */ ISD::MERGE_VALUES,
- /*Instruction::InsertValue */ ISD::MERGE_VALUES,
- /*Instruction::LandingPad */ 0};
-
- assert((Instruction::Ret == 1) && (Instruction::LandingPad == 58) &&
- "Instruction order had changed");
-
- // Opcode numbering starts at #1 but the table starts at #0, so we subtract
- // one from the opcode number.
- return OpToISDTbl[Opcode - 1];
-}
-
-std::pair<unsigned, EVT>
-VectorTargetTransformImpl::getTypeLegalizationCost(LLVMContext &C,
- EVT Ty) const {
+int VectorTargetTransformImpl::InstructionOpcodeToISD(unsigned Opcode) const {
+ enum InstructionOpcodes {
+#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
+#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
+#include "llvm/Instruction.def"
+ };
+ switch (static_cast<InstructionOpcodes>(Opcode)) {
+ case Ret: return 0;
+ case Br: return 0;
+ case Switch: return 0;
+ case IndirectBr: return 0;
+ case Invoke: return 0;
+ case Resume: return 0;
+ case Unreachable: return 0;
+ case Add: return ISD::ADD;
+ case FAdd: return ISD::FADD;
+ case Sub: return ISD::SUB;
+ case FSub: return ISD::FSUB;
+ case Mul: return ISD::MUL;
+ case FMul: return ISD::FMUL;
+ case UDiv: return ISD::UDIV;
+ case SDiv: return ISD::UDIV;
+ case FDiv: return ISD::FDIV;
+ case URem: return ISD::UREM;
+ case SRem: return ISD::SREM;
+ case FRem: return ISD::FREM;
+ case Shl: return ISD::SHL;
+ case LShr: return ISD::SRL;
+ case AShr: return ISD::SRA;
+ case And: return ISD::AND;
+ case Or: return ISD::OR;
+ case Xor: return ISD::XOR;
+ case Alloca: return 0;
+ case Load: return ISD::LOAD;
+ case Store: return ISD::STORE;
+ case GetElementPtr: return 0;
+ case Fence: return 0;
+ case AtomicCmpXchg: return 0;
+ case AtomicRMW: return 0;
+ case Trunc: return ISD::TRUNCATE;
+ case ZExt: return ISD::ZERO_EXTEND;
+ case SExt: return ISD::SIGN_EXTEND;
+ case FPToUI: return ISD::FP_TO_UINT;
+ case FPToSI: return ISD::FP_TO_SINT;
+ case UIToFP: return ISD::UINT_TO_FP;
+ case SIToFP: return ISD::SINT_TO_FP;
+ case FPTrunc: return ISD::FP_ROUND;
+ case FPExt: return ISD::FP_EXTEND;
+ case PtrToInt: return ISD::BITCAST;
+ case IntToPtr: return ISD::BITCAST;
+ case BitCast: return ISD::BITCAST;
+ case ICmp: return ISD::SETCC;
+ case FCmp: return ISD::SETCC;
+ case PHI: return 0;
+ case Call: return 0;
+ case Select: return ISD::SELECT;
+ case UserOp1: return 0;
+ case UserOp2: return 0;
+ case VAArg: return 0;
+ case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
+ case InsertElement: return ISD::INSERT_VECTOR_ELT;
+ case ShuffleVector: return ISD::VECTOR_SHUFFLE;
+ case ExtractValue: return ISD::MERGE_VALUES;
+ case InsertValue: return ISD::MERGE_VALUES;
+ case LandingPad: return 0;
+ }
+
+ llvm_unreachable("Unknown instruction type encountered!");
+}
+
+std::pair<unsigned, MVT>
+VectorTargetTransformImpl::getTypeLegalizationCost(Type *Ty) const {
+
+ LLVMContext &C = Ty->getContext();
+ EVT MTy = TLI->getValueType(Ty);
+
unsigned Cost = 1;
// We keep legalizing the type until we find a legal kind. We assume that
// the only operation that costs anything is the split. After splitting
// we need to handle two types.
while (true) {
- TargetLowering::LegalizeKind LK = TLI->getTypeConversion(C, Ty);
+ TargetLowering::LegalizeKind LK = TLI->getTypeConversion(C, MTy);
if (LK.first == TargetLowering::TypeLegal)
- return std::make_pair(Cost, LK.second);
+ return std::make_pair(Cost, MTy.getSimpleVT());
- if (LK.first == TargetLowering::TypeSplitVector)
+ if (LK.first == TargetLowering::TypeSplitVector ||
+ LK.first == TargetLowering::TypeExpandInteger)
Cost *= 2;
// Keep legalizing the type.
- Ty = LK.second;
+ MTy = LK.second;
}
}
unsigned
-VectorTargetTransformImpl::getInstrCost(unsigned Opcode, Type *Ty1,
- Type *Ty2) const {
+VectorTargetTransformImpl::getScalarizationOverhead(Type *Ty,
+ bool Insert,
+ bool Extract) const {
+ assert (Ty->isVectorTy() && "Can only scalarize vectors");
+ unsigned Cost = 0;
+
+ for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
+ if (Insert)
+ Cost += getVectorInstrCost(Instruction::InsertElement, Ty, i);
+ if (Extract)
+ Cost += getVectorInstrCost(Instruction::ExtractElement, Ty, i);
+ }
+
+ return Cost;
+}
+
+unsigned VectorTargetTransformImpl::getArithmeticInstrCost(unsigned Opcode,
+ Type *Ty) const {
// Check if any of the operands are vector operands.
int ISD = InstructionOpcodeToISD(Opcode);
+ assert(ISD && "Invalid opcode");
- // Selects on vectors are actually vector selects.
- if (ISD == ISD::SELECT) {
- assert(Ty2 && "Ty2 must hold the select type");
- if (Ty2->isVectorTy())
- ISD = ISD::VSELECT;
+ std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Ty);
+
+ if (!TLI->isOperationExpand(ISD, LT.second)) {
+ // The operation is legal. Assume it costs 1. Multiply
+ // by the type-legalization overhead.
+ return LT.first * 1;
+ }
+
+ // Else, assume that we need to scalarize this op.
+ if (Ty->isVectorTy()) {
+ unsigned Num = Ty->getVectorNumElements();
+ unsigned Cost = getArithmeticInstrCost(Opcode, Ty->getScalarType());
+ // return the cost of multiple scalar invocation plus the cost of inserting
+ // and extracting the values.
+ return getScalarizationOverhead(Ty, true, true) + Num * Cost;
+ }
+
+ // We don't know anything about this scalar instruction.
+ return 1;
+}
+
+unsigned VectorTargetTransformImpl::getBroadcastCost(Type *Tp) const {
+ return 1;
+}
+
+unsigned VectorTargetTransformImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
+ Type *Src) const {
+ int ISD = InstructionOpcodeToISD(Opcode);
+ assert(ISD && "Invalid opcode");
+
+ std::pair<unsigned, MVT> SrcLT = getTypeLegalizationCost(Src);
+ std::pair<unsigned, MVT> DstLT = getTypeLegalizationCost(Dst);
+
+ // Handle scalar conversions.
+ if (!Src->isVectorTy() && !Dst->isVectorTy()) {
+
+ // Scalar bitcasts are usually free.
+ if (Opcode == Instruction::BitCast)
+ return 0;
+
+ if (Opcode == Instruction::Trunc &&
+ TLI->isTruncateFree(SrcLT.second, DstLT.second))
+ return 0;
+
+ if (Opcode == Instruction::ZExt &&
+ TLI->isZExtFree(SrcLT.second, DstLT.second))
+ return 0;
+
+ // Just check the op cost. If the operation is legal then assume it costs 1.
+ if (!TLI->isOperationExpand(ISD, DstLT.second))
+ return 1;
+
+ // Assume that illegal scalar instruction are expensive.
+ return 4;
}
- // If we don't have any information about this instruction assume it costs 1.
- if (ISD == 0)
- return 1;
+ // Check vector-to-vector casts.
+ if (Dst->isVectorTy() && Src->isVectorTy()) {
+
+ // If the cast is between same-sized registers, then the check is simple.
+ if (SrcLT.first == DstLT.first &&
+ SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) {
+
+ // Bitcast between types that are legalized to the same type are free.
+ if (Opcode == Instruction::BitCast || Opcode == Instruction::Trunc)
+ return 0;
+
+ // Assume that Zext is done using AND.
+ if (Opcode == Instruction::ZExt)
+ return 1;
+
+ // Assume that sext is done using SHL and SRA.
+ if (Opcode == Instruction::SExt)
+ return 2;
+
+ // Just check the op cost. If the operation is legal then assume it costs
+ // 1 and multiply by the type-legalization overhead.
+ if (!TLI->isOperationExpand(ISD, DstLT.second))
+ return SrcLT.first * 1;
+ }
+
+ // If we are converting vectors and the operation is illegal, or
+ // if the vectors are legalized to different types, estimate the
+ // scalarization costs.
+ unsigned Num = Dst->getVectorNumElements();
+ unsigned Cost = getCastInstrCost(Opcode, Dst->getScalarType(),
+ Src->getScalarType());
+
+ // Return the cost of multiple scalar invocation plus the cost of
+ // inserting and extracting the values.
+ return getScalarizationOverhead(Dst, true, true) + Num * Cost;
+ }
- assert(Ty1 && "We need to have at least one type");
+ // We already handled vector-to-vector and scalar-to-scalar conversions. This
+ // is where we handle bitcast between vectors and scalars. We need to assume
+ // that the conversion is scalarized in one way or another.
+ if (Opcode == Instruction::BitCast)
+ // Illegal bitcasts are done by storing and loading from a stack slot.
+ return (Src->isVectorTy()? getScalarizationOverhead(Src, false, true):0) +
+ (Dst->isVectorTy()? getScalarizationOverhead(Dst, true, false):0);
+
+ llvm_unreachable("Unhandled cast");
+ }
+
+unsigned VectorTargetTransformImpl::getCFInstrCost(unsigned Opcode) const {
+ return 0;
+}
+
+unsigned VectorTargetTransformImpl::getCmpSelInstrCost(unsigned Opcode,
+ Type *ValTy,
+ Type *CondTy) const {
+ int ISD = InstructionOpcodeToISD(Opcode);
+ assert(ISD && "Invalid opcode");
+
+ // Selects on vectors are actually vector selects.
+ if (ISD == ISD::SELECT) {
+ assert(CondTy && "CondTy must exist");
+ if (CondTy->isVectorTy())
+ ISD = ISD::VSELECT;
+ }
- // From this stage we look at the legalized type.
- std::pair<unsigned, EVT> LT =
- getTypeLegalizationCost(Ty1->getContext(), TLI->getValueType(Ty1));
+ std::pair<unsigned, MVT> LT = getTypeLegalizationCost(ValTy);
- if (TLI->isOperationLegalOrCustom(ISD, LT.second)) {
+ if (!TLI->isOperationExpand(ISD, LT.second)) {
// The operation is legal. Assume it costs 1. Multiply
// by the type-legalization overhead.
return LT.first * 1;
}
- unsigned NumElem =
- (LT.second.isVector() ? LT.second.getVectorNumElements() : 1);
+ // Otherwise, assume that the cast is scalarized.
+ if (ValTy->isVectorTy()) {
+ unsigned Num = ValTy->getVectorNumElements();
+ if (CondTy)
+ CondTy = CondTy->getScalarType();
+ unsigned Cost = getCmpSelInstrCost(Opcode, ValTy->getScalarType(),
+ CondTy);
- // We will probably scalarize this instruction. Assume that the cost is the
- // number of the vector elements.
- return LT.first * NumElem * 1;
+ // Return the cost of multiple scalar invocation plus the cost of inserting
+ // and extracting the values.
+ return getScalarizationOverhead(ValTy, true, false) + Num * Cost;
+ }
+
+ // Unknown scalar opcode.
+ return 1;
+}
+
+unsigned VectorTargetTransformImpl::getVectorInstrCost(unsigned Opcode,
+ Type *Val,
+ unsigned Index) const {
+ return 1;
}
unsigned
-VectorTargetTransformImpl::getBroadcastCost(Type *Tp) const {
+VectorTargetTransformImpl::getInstrCost(unsigned Opcode, Type *Ty1,
+ Type *Ty2) const {
return 1;
}
VectorTargetTransformImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
unsigned Alignment,
unsigned AddressSpace) const {
- // From this stage we look at the legalized type.
- std::pair<unsigned, EVT> LT =
- getTypeLegalizationCost(Src->getContext(), TLI->getValueType(Src));
+ assert(!Src->isVoidTy() && "Invalid type");
+ std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Src);
+
// Assume that all loads of legal types cost 1.
return LT.first;
}
+
+unsigned
+VectorTargetTransformImpl::getIntrinsicInstrCost(Intrinsic::ID, Type *RetTy,
+ ArrayRef<Type*> Tys) const {
+ // assume that we need to scalarize this intrinsic.
+ unsigned ScalarizationCost = 0;
+ unsigned ScalarCalls = 1;
+ if (RetTy->isVectorTy()) {
+ ScalarizationCost = getScalarizationOverhead(RetTy, true, false);
+ ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements());
+ }
+ for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
+ if (Tys[i]->isVectorTy()) {
+ ScalarizationCost += getScalarizationOverhead(Tys[i], false, true);
+ ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements());
+ }
+ }
+ return ScalarCalls + ScalarizationCost;
+}
+
+unsigned
+VectorTargetTransformImpl::getNumberOfParts(Type *Tp) const {
+ std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Tp);
+ return LT.first;
+}
projects / oota-llvm.git / blobdiff