From 8be39c81b0a90ec91a110b59dda07b9bacad093f Mon Sep 17 00:00:00 2001 From: Elena Demikhovsky Date: Sun, 8 Feb 2015 08:27:19 +0000 Subject: [PATCH] Masked Gather and Scatter Intrinsics. Gather and Scatter are new introduced intrinsics, comming after recently implemented masked load and store. This is the first patch for Gather and Scatter intrinsics. It includes only the syntax, parsing and verification. Gather and Scatter intrinsics allow to perform multiple memory accesses (read/write) in one vector instruction. The intrinsics are not target specific and will have the following syntax: Gather: declare <16 x i32> @llvm.masked.gather.v16i32(<16 x i32*> , i32 , <16 x i1> , <16 x i32> ) declare <8 x float> @llvm.masked.gather.v8f32(<8 x float*>, i32 , <8 x i1> , <8 x float>) Scatter: declare void @llvm.masked.scatter.v8i32(<8 x i32> , <8 x i32*> , i32 , <8 x i1> ) declare void @llvm.masked.scatter.v16i32(<16 x i32> , <16 x i32*> , i32 , <16 x i1> ) Vector of ptrs - a set of source/destination addresses, to load/store the value. Mask - switches on/off vector lanes to prevent memory access for switched-off lanes vector of ptrs, value and mask should have the same vector width. These are code examples where gather / scatter should be used and will allow function vectorization ;void foo1(int * restrict A, int * restrict B, int * restrict C) { ; for (int i=0; i> 3; } ArgKind getArgumentKind() const { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || - Kind == SameVecWidthArgument || Kind == PtrToArgument); + Kind == SameVecWidthArgument || Kind == PtrToArgument || + Kind == VecOfPtrsToElt); return (ArgKind)(Argument_Info & 7); } diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td index 5a304db09f2..1326a8330d5 100644 --- a/include/llvm/IR/Intrinsics.td +++ b/include/llvm/IR/Intrinsics.td @@ -117,6 +117,7 @@ class LLVMVectorSameWidth ValueType ElTy = elty.VT; } class LLVMPointerTo : LLVMMatchType; +class LLVMVectorOfPointersToElt : LLVMMatchType; // Match the type of another intrinsic parameter that is expected to be a // vector type, but change the element count to be half as many @@ -584,6 +585,19 @@ def int_masked_load : Intrinsic<[llvm_anyvector_ty], [LLVMPointerTo<0>, llvm_i32_ty, LLVMVectorSameWidth<0, llvm_i1_ty>, LLVMMatchType<0>], [IntrReadArgMem]>; + +def int_masked_gather: Intrinsic<[llvm_anyvector_ty], + [LLVMVectorOfPointersToElt<0>, llvm_i32_ty, + LLVMVectorSameWidth<0, llvm_i1_ty>, + LLVMMatchType<0>], + [IntrReadArgMem]>; + +def int_masked_scatter: Intrinsic<[], + [llvm_anyvector_ty, + LLVMVectorOfPointersToElt<0>, llvm_i32_ty, + LLVMVectorSameWidth<0, llvm_i1_ty>], + [IntrReadWriteArgMem]>; + //===----------------------------------------------------------------------===// // Target-specific intrinsics //===----------------------------------------------------------------------===// diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp index 070513edef2..fe44f17d686 100644 --- a/lib/IR/Function.cpp +++ b/lib/IR/Function.cpp @@ -542,7 +542,8 @@ enum IIT_Info { IIT_VARARG = 28, IIT_HALF_VEC_ARG = 29, IIT_SAME_VEC_WIDTH_ARG = 30, - IIT_PTR_TO_ARG = 31 + IIT_PTR_TO_ARG = 31, + IIT_VEC_OF_PTRS_TO_ELT = 32 }; @@ -662,6 +663,12 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef Infos, ArgInfo)); return; } + case IIT_VEC_OF_PTRS_TO_ELT: { + unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + OutputTable.push_back(IITDescriptor::get(IITDescriptor::VecOfPtrsToElt, + ArgInfo)); + return; + } case IIT_EMPTYSTRUCT: OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0)); return; @@ -781,6 +788,15 @@ static Type *DecodeFixedType(ArrayRef &Infos, Type *Ty = Tys[D.getArgumentNumber()]; return PointerType::getUnqual(Ty); } + case IITDescriptor::VecOfPtrsToElt: { + Type *Ty = Tys[D.getArgumentNumber()]; + VectorType *VTy = dyn_cast(Ty); + if (!VTy) + llvm_unreachable("Expected an argument of Vector Type"); + Type *EltTy = VTy->getVectorElementType(); + return VectorType::get(PointerType::getUnqual(EltTy), + VTy->getNumElements()); + } } llvm_unreachable("unhandled"); } diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index 29eddfe2541..a7b705410d7 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -2555,6 +2555,23 @@ bool Verifier::VerifyIntrinsicType(Type *Ty, PointerType *ThisArgType = dyn_cast(Ty); return (!ThisArgType || ThisArgType->getElementType() != ReferenceType); } + case IITDescriptor::VecOfPtrsToElt: { + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + VectorType * ReferenceType = + dyn_cast (ArgTys[D.getArgumentNumber()]); + VectorType *ThisArgVecTy = dyn_cast(Ty); + if (!ThisArgVecTy || !ReferenceType || + (ReferenceType->getVectorNumElements() != + ThisArgVecTy->getVectorNumElements())) + return true; + PointerType *ThisArgEltTy = + dyn_cast(ThisArgVecTy->getVectorElementType()); + if (!ThisArgEltTy) + return true; + return (!(ThisArgEltTy->getElementType() == + ReferenceType->getVectorElementType())); + } } llvm_unreachable("unhandled"); } diff --git a/utils/TableGen/IntrinsicEmitter.cpp b/utils/TableGen/IntrinsicEmitter.cpp index 87f9c90dc4a..e533d89326c 100644 --- a/utils/TableGen/IntrinsicEmitter.cpp +++ b/utils/TableGen/IntrinsicEmitter.cpp @@ -259,7 +259,8 @@ enum IIT_Info { IIT_VARARG = 28, IIT_HALF_VEC_ARG = 29, IIT_SAME_VEC_WIDTH_ARG = 30, - IIT_PTR_TO_ARG = 31 + IIT_PTR_TO_ARG = 31, + IIT_VEC_OF_PTRS_TO_ELT = 32 }; @@ -314,9 +315,10 @@ static void EncodeFixedType(Record *R, std::vector &ArgCodes, EncodeFixedValueType(VT, Sig); return; } - else if (R->isSubClassOf("LLVMPointerTo")) { + else if (R->isSubClassOf("LLVMPointerTo")) Sig.push_back(IIT_PTR_TO_ARG); - } + else if (R->isSubClassOf("LLVMVectorOfPointersToElt")) + Sig.push_back(IIT_VEC_OF_PTRS_TO_ELT); else Sig.push_back(IIT_ARG); return Sig.push_back((Number << 3) | ArgCodes[Number]); -- 2.34.1