+
+ switch (type) {
+ case 'c':
+ switch (ck) {
+ case ClassS: s += poly ? "_p8" : usgn ? "_u8" : "_s8"; break;
+ case ClassI: s += "_i8"; break;
+ case ClassW: s += "_8"; break;
+ default: break;
+ }
+ break;
+ case 's':
+ switch (ck) {
+ case ClassS: s += poly ? "_p16" : usgn ? "_u16" : "_s16"; break;
+ case ClassI: s += "_i16"; break;
+ case ClassW: s += "_16"; break;
+ default: break;
+ }
+ break;
+ case 'i':
+ switch (ck) {
+ case ClassS: s += usgn ? "_u32" : "_s32"; break;
+ case ClassI: s += "_i32"; break;
+ case ClassW: s += "_32"; break;
+ default: break;
+ }
+ break;
+ case 'l':
+ switch (ck) {
+ case ClassS: s += usgn ? "_u64" : "_s64"; break;
+ case ClassI: s += "_i64"; break;
+ case ClassW: s += "_64"; break;
+ default: break;
+ }
+ break;
+ case 'h':
+ switch (ck) {
+ case ClassS:
+ case ClassI: s += "_f16"; break;
+ case ClassW: s += "_16"; break;
+ default: break;
+ }
+ break;
+ case 'f':
+ switch (ck) {
+ case ClassS:
+ case ClassI: s += "_f32"; break;
+ case ClassW: s += "_32"; break;
+ default: break;
+ }
+ break;
+ default:
+ throw "unhandled type!";
+ break;
+ }
+ if (ck == ClassB)
+ s += "_v";
+
+ // Insert a 'q' before the first '_' character so that it ends up before
+ // _lane or _n on vector-scalar operations.
+ if (quad) {
+ size_t pos = s.find('_');
+ s = s.insert(pos, "q");
+ }
+ return s;
+}
+
+// Generate the string "(argtype a, argtype b, ...)"
+static std::string GenArgs(const std::string &proto, StringRef typestr) {
+ bool define = proto.find('i') != std::string::npos;
+ char arg = 'a';
+
+ std::string s;
+ s += "(";
+
+ for (unsigned i = 1, e = proto.size(); i != e; ++i, ++arg) {
+ if (define) {
+ // Immediate macro arguments are used directly instead of being assigned
+ // to local temporaries; prepend an underscore prefix to make their
+ // names consistent with the local temporaries.
+ if (proto[i] == 'i')
+ s += "__";
+ } else {
+ s += TypeString(proto[i], typestr) + " __";
+ }
+ s.push_back(arg);
+ if ((i + 1) < e)
+ s += ", ";
+ }
+
+ s += ")";
+ return s;
+}
+
+// Macro arguments are not type-checked like inline function arguments, so
+// assign them to local temporaries to get the right type checking.
+static std::string GenMacroLocals(const std::string &proto, StringRef typestr) {
+ char arg = 'a';
+ std::string s;
+
+ for (unsigned i = 1, e = proto.size(); i != e; ++i, ++arg) {
+ // Do not create a temporary for an immediate argument.
+ // That would defeat the whole point of using a macro!
+ if (proto[i] == 'i') continue;
+
+ s += TypeString(proto[i], typestr) + " __";
+ s.push_back(arg);
+ s += " = (";
+ s.push_back(arg);
+ s += "); ";
+ }
+
+ s += "\\\n ";
+ return s;
+}
+
+// Use the vmovl builtin to sign-extend or zero-extend a vector.
+static std::string Extend(StringRef typestr, const std::string &a) {
+ std::string s;
+ s = MangleName("vmovl", typestr, ClassS);
+ s += "(" + a + ")";
+ return s;
+}
+
+static std::string Duplicate(unsigned nElts, StringRef typestr,
+ const std::string &a) {
+ std::string s;
+
+ s = "(" + TypeString('d', typestr) + "){ ";
+ for (unsigned i = 0; i != nElts; ++i) {
+ s += a;
+ if ((i + 1) < nElts)
+ s += ", ";
+ }
+ s += " }";
+
+ return s;
+}
+
+static std::string SplatLane(unsigned nElts, const std::string &vec,
+ const std::string &lane) {
+ std::string s = "__builtin_shufflevector(" + vec + ", " + vec;
+ for (unsigned i = 0; i < nElts; ++i)
+ s += ", " + lane;
+ s += ")";
+ return s;
+}
+
+static unsigned GetNumElements(StringRef typestr, bool &quad) {
+ quad = false;
+ bool dummy = false;
+ char type = ClassifyType(typestr, quad, dummy, dummy);
+ unsigned nElts = 0;
+ switch (type) {
+ case 'c': nElts = 8; break;
+ case 's': nElts = 4; break;
+ case 'i': nElts = 2; break;
+ case 'l': nElts = 1; break;
+ case 'h': nElts = 4; break;
+ case 'f': nElts = 2; break;
+ default:
+ throw "unhandled type!";
+ break;
+ }
+ if (quad) nElts <<= 1;
+ return nElts;
+}
+
+// Generate the definition for this intrinsic, e.g. "a + b" for OpAdd.
+static std::string GenOpString(OpKind op, const std::string &proto,
+ StringRef typestr) {
+ bool quad;
+ unsigned nElts = GetNumElements(typestr, quad);
+
+ // If this builtin takes an immediate argument, we need to #define it rather
+ // than use a standard declaration, so that SemaChecking can range check
+ // the immediate passed by the user.
+ bool define = proto.find('i') != std::string::npos;
+
+ std::string ts = TypeString(proto[0], typestr);
+ std::string s;
+ if (!define) {
+ s = "return ";
+ }
+
+ switch(op) {
+ case OpAdd:
+ s += "__a + __b;";
+ break;
+ case OpAddl:
+ s += Extend(typestr, "__a") + " + " + Extend(typestr, "__b") + ";";
+ break;
+ case OpAddw:
+ s += "__a + " + Extend(typestr, "__b") + ";";
+ break;
+ case OpSub:
+ s += "__a - __b;";
+ break;
+ case OpSubl:
+ s += Extend(typestr, "__a") + " - " + Extend(typestr, "__b") + ";";
+ break;
+ case OpSubw:
+ s += "__a - " + Extend(typestr, "__b") + ";";
+ break;
+ case OpMulN:
+ s += "__a * " + Duplicate(nElts, typestr, "__b") + ";";
+ break;
+ case OpMulLane:
+ s += "__a * " + SplatLane(nElts, "__b", "__c") + ";";
+ break;
+ case OpMul:
+ s += "__a * __b;";
+ break;
+ case OpMullN:
+ s += Extend(typestr, "__a") + " * " +
+ Extend(typestr, Duplicate(nElts << (int)quad, typestr, "__b")) + ";";
+ break;
+ case OpMullLane:
+ s += Extend(typestr, "__a") + " * " +
+ Extend(typestr, SplatLane(nElts, "__b", "__c")) + ";";
+ break;
+ case OpMull:
+ s += Extend(typestr, "__a") + " * " + Extend(typestr, "__b") + ";";
+ break;
+ case OpMlaN:
+ s += "__a + (__b * " + Duplicate(nElts, typestr, "__c") + ");";
+ break;
+ case OpMlaLane:
+ s += "__a + (__b * " + SplatLane(nElts, "__c", "__d") + ");";
+ break;
+ case OpMla:
+ s += "__a + (__b * __c);";
+ break;
+ case OpMlalN:
+ s += "__a + (" + Extend(typestr, "__b") + " * " +
+ Extend(typestr, Duplicate(nElts, typestr, "__c")) + ");";
+ break;
+ case OpMlalLane:
+ s += "__a + (" + Extend(typestr, "__b") + " * " +
+ Extend(typestr, SplatLane(nElts, "__c", "__d")) + ");";
+ break;
+ case OpMlal:
+ s += "__a + (" + Extend(typestr, "__b") + " * " +
+ Extend(typestr, "__c") + ");";
+ break;
+ case OpMlsN:
+ s += "__a - (__b * " + Duplicate(nElts, typestr, "__c") + ");";
+ break;
+ case OpMlsLane:
+ s += "__a - (__b * " + SplatLane(nElts, "__c", "__d") + ");";
+ break;
+ case OpMls:
+ s += "__a - (__b * __c);";
+ break;
+ case OpMlslN:
+ s += "__a - (" + Extend(typestr, "__b") + " * " +
+ Extend(typestr, Duplicate(nElts, typestr, "__c")) + ");";
+ break;
+ case OpMlslLane:
+ s += "__a - (" + Extend(typestr, "__b") + " * " +
+ Extend(typestr, SplatLane(nElts, "__c", "__d")) + ");";
+ break;
+ case OpMlsl:
+ s += "__a - (" + Extend(typestr, "__b") + " * " +
+ Extend(typestr, "__c") + ");";
+ break;
+ case OpQDMullLane:
+ s += MangleName("vqdmull", typestr, ClassS) + "(__a, " +
+ SplatLane(nElts, "__b", "__c") + ");";
+ break;
+ case OpQDMlalLane:
+ s += MangleName("vqdmlal", typestr, ClassS) + "(__a, __b, " +
+ SplatLane(nElts, "__c", "__d") + ");";
+ break;
+ case OpQDMlslLane:
+ s += MangleName("vqdmlsl", typestr, ClassS) + "(__a, __b, " +
+ SplatLane(nElts, "__c", "__d") + ");";
+ break;
+ case OpQDMulhLane:
+ s += MangleName("vqdmulh", typestr, ClassS) + "(__a, " +
+ SplatLane(nElts, "__b", "__c") + ");";
+ break;
+ case OpQRDMulhLane:
+ s += MangleName("vqrdmulh", typestr, ClassS) + "(__a, " +
+ SplatLane(nElts, "__b", "__c") + ");";
+ break;
+ case OpEq:
+ s += "(" + ts + ")(__a == __b);";
+ break;
+ case OpGe:
+ s += "(" + ts + ")(__a >= __b);";
+ break;
+ case OpLe:
+ s += "(" + ts + ")(__a <= __b);";
+ break;
+ case OpGt:
+ s += "(" + ts + ")(__a > __b);";
+ break;
+ case OpLt:
+ s += "(" + ts + ")(__a < __b);";
+ break;
+ case OpNeg:
+ s += " -__a;";
+ break;
+ case OpNot:
+ s += " ~__a;";
+ break;
+ case OpAnd:
+ s += "__a & __b;";
+ break;
+ case OpOr:
+ s += "__a | __b;";
+ break;
+ case OpXor:
+ s += "__a ^ __b;";
+ break;
+ case OpAndNot:
+ s += "__a & ~__b;";
+ break;
+ case OpOrNot:
+ s += "__a | ~__b;";
+ break;
+ case OpCast:
+ s += "(" + ts + ")__a;";
+ break;
+ case OpConcat:
+ s += "(" + ts + ")__builtin_shufflevector((int64x1_t)__a";
+ s += ", (int64x1_t)__b, 0, 1);";
+ break;
+ case OpHi:
+ s += "(" + ts +
+ ")__builtin_shufflevector((int64x2_t)__a, (int64x2_t)__a, 1);";
+ break;
+ case OpLo:
+ s += "(" + ts +
+ ")__builtin_shufflevector((int64x2_t)__a, (int64x2_t)__a, 0);";
+ break;
+ case OpDup:
+ s += Duplicate(nElts, typestr, "__a") + ";";
+ break;
+ case OpDupLane:
+ s += SplatLane(nElts, "__a", "__b") + ";";
+ break;
+ case OpSelect:
+ // ((0 & 1) | (~0 & 2))
+ s += "(" + ts + ")";
+ ts = TypeString(proto[1], typestr);
+ s += "((__a & (" + ts + ")__b) | ";
+ s += "(~__a & (" + ts + ")__c));";
+ break;
+ case OpRev16:
+ s += "__builtin_shufflevector(__a, __a";
+ for (unsigned i = 2; i <= nElts; i += 2)
+ for (unsigned j = 0; j != 2; ++j)
+ s += ", " + utostr(i - j - 1);
+ s += ");";
+ break;
+ case OpRev32: {
+ unsigned WordElts = nElts >> (1 + (int)quad);
+ s += "__builtin_shufflevector(__a, __a";
+ for (unsigned i = WordElts; i <= nElts; i += WordElts)
+ for (unsigned j = 0; j != WordElts; ++j)
+ s += ", " + utostr(i - j - 1);
+ s += ");";
+ break;
+ }
+ case OpRev64: {
+ unsigned DblWordElts = nElts >> (int)quad;
+ s += "__builtin_shufflevector(__a, __a";
+ for (unsigned i = DblWordElts; i <= nElts; i += DblWordElts)
+ for (unsigned j = 0; j != DblWordElts; ++j)
+ s += ", " + utostr(i - j - 1);
+ s += ");";
+ break;
+ }
+ case OpAbdl: {
+ std::string abd = MangleName("vabd", typestr, ClassS) + "(__a, __b)";
+ if (typestr[0] != 'U') {
+ // vabd results are always unsigned and must be zero-extended.
+ std::string utype = "U" + typestr.str();
+ s += "(" + TypeString(proto[0], typestr) + ")";
+ abd = "(" + TypeString('d', utype) + ")" + abd;
+ s += Extend(utype, abd) + ";";
+ } else {
+ s += Extend(typestr, abd) + ";";
+ }
+ break;
+ }
+ case OpAba:
+ s += "__a + " + MangleName("vabd", typestr, ClassS) + "(__b, __c);";
+ break;
+ case OpAbal: {
+ s += "__a + ";
+ std::string abd = MangleName("vabd", typestr, ClassS) + "(__b, __c)";
+ if (typestr[0] != 'U') {
+ // vabd results are always unsigned and must be zero-extended.
+ std::string utype = "U" + typestr.str();
+ s += "(" + TypeString(proto[0], typestr) + ")";
+ abd = "(" + TypeString('d', utype) + ")" + abd;
+ s += Extend(utype, abd) + ";";
+ } else {
+ s += Extend(typestr, abd) + ";";
+ }
+ break;
+ }
+ default:
+ throw "unknown OpKind!";
+ break;
+ }
+ return s;
+}
+
+static unsigned GetNeonEnum(const std::string &proto, StringRef typestr) {
+ unsigned mod = proto[0];
+ unsigned ret = 0;
+
+ if (mod == 'v' || mod == 'f')
+ mod = proto[1];
+
+ bool quad = false;
+ bool poly = false;
+ bool usgn = false;
+ bool scal = false;
+ bool cnst = false;
+ bool pntr = false;
+
+ // Base type to get the type string for.
+ char type = ClassifyType(typestr, quad, poly, usgn);
+
+ // Based on the modifying character, change the type and width if necessary.
+ type = ModType(mod, type, quad, poly, usgn, scal, cnst, pntr);
+
+ if (usgn)
+ ret |= 0x08;
+ if (quad && proto[1] != 'g')
+ ret |= 0x10;
+