X-Git-Url: http://demsky.eecs.uci.edu/git/?a=blobdiff_plain;f=utils%2FTableGen%2FDAGISelEmitter.cpp;h=5e2b07d0d4c398506409b9e931ae559d73ae70a5;hb=d6c84720df0b63e34081e0c7890f3074d8b110b9;hp=bbb8a18eec2cda25246d27cbd1971626ff355fc4;hpb=11ee508d7bd37f2b007da735fbd2c1497ed51848;p=oota-llvm.git diff --git a/utils/TableGen/DAGISelEmitter.cpp b/utils/TableGen/DAGISelEmitter.cpp index bbb8a18eec2..5e2b07d0d4c 100644 --- a/utils/TableGen/DAGISelEmitter.cpp +++ b/utils/TableGen/DAGISelEmitter.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "DAGISelEmitter.h" +#include "DAGISelMatcher.h" #include "Record.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/CommandLine.h" @@ -23,6 +24,9 @@ #include using namespace llvm; +//#define ENABLE_NEW_ISEL + + static cl::opt GenDebug("gen-debug", cl::desc("Generate debug code"), cl::init(false)); @@ -30,27 +34,20 @@ GenDebug("gen-debug", cl::desc("Generate debug code"), cl::init(false)); // DAGISelEmitter Helper methods // -/// NodeIsComplexPattern - return true if N is a leaf node and a subclass of -/// ComplexPattern. -static bool NodeIsComplexPattern(TreePatternNode *N) { - return (N->isLeaf() && - dynamic_cast(N->getLeafValue()) && - static_cast(N->getLeafValue())->getDef()-> - isSubClassOf("ComplexPattern")); +/// getNodeName - The top level Select_* functions have an "SDNode* N" +/// argument. When expanding the pattern-matching code, the intermediate +/// variables have type SDValue. This function provides a uniform way to +/// reference the underlying "SDNode *" for both cases. +static std::string getNodeName(const std::string &S) { + if (S == "N") return S; + return S + ".getNode()"; } -/// NodeGetComplexPattern - return the pointer to the ComplexPattern if N -/// is a leaf node and a subclass of ComplexPattern, else it returns NULL. -static const ComplexPattern *NodeGetComplexPattern(TreePatternNode *N, - CodeGenDAGPatterns &CGP) { - if (N->isLeaf() && - dynamic_cast(N->getLeafValue()) && - static_cast(N->getLeafValue())->getDef()-> - isSubClassOf("ComplexPattern")) { - return &CGP.getComplexPattern(static_cast(N->getLeafValue()) - ->getDef()); - } - return NULL; +/// getNodeValue - Similar to getNodeName, except it provides a uniform +/// way to access the SDValue for both cases. +static std::string getValueName(const std::string &S) { + if (S == "N") return "SDValue(N, 0)"; + return S; } /// getPatternSize - Return the 'size' of this pattern. We want to match large @@ -75,7 +72,7 @@ static unsigned getPatternSize(TreePatternNode *P, CodeGenDAGPatterns &CGP) { // Later we can allow complexity / cost for each pattern to be (optionally) // specified. To get best possible pattern match we'll need to dynamically // calculate the complexity of all patterns a dag can potentially map to. - const ComplexPattern *AM = NodeGetComplexPattern(P, CGP); + const ComplexPattern *AM = P->getComplexPatternInfo(CGP); if (AM) Size += AM->getNumOperands() * 3; @@ -92,7 +89,7 @@ static unsigned getPatternSize(TreePatternNode *P, CodeGenDAGPatterns &CGP) { else if (Child->isLeaf()) { if (dynamic_cast(Child->getLeafValue())) Size += 5; // Matches a ConstantSDNode (+3) and a specific value (+2). - else if (NodeIsComplexPattern(Child)) + else if (Child->getComplexPatternInfo(CGP)) Size += getPatternSize(Child, CGP); else if (!Child->getPredicateFns().empty()) ++Size; @@ -114,7 +111,7 @@ static unsigned getResultPatternCost(TreePatternNode *P, if (Op->isSubClassOf("Instruction")) { Cost++; CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(Op->getName()); - if (II.usesCustomDAGSchedInserter) + if (II.usesCustomInserter) Cost += 10; } for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) @@ -147,7 +144,6 @@ struct PatternSortingPredicate { typedef std::pair CodeLine; typedef std::vector CodeList; - typedef std::vector > PatternList; bool operator()(const std::pair &LHSPair, const std::pair &RHSPair) { @@ -175,7 +171,8 @@ struct PatternSortingPredicate { /// getRegisterValueType - Look up and return the ValueType of the specified /// register. If the register is a member of multiple register classes which /// have different associated types, return MVT::Other. -static MVT::SimpleValueType getRegisterValueType(Record *R, const CodeGenTarget &T) { +static MVT::SimpleValueType getRegisterValueType(Record *R, + const CodeGenTarget &T) { bool FoundRC = false; MVT::SimpleValueType VT = MVT::Other; const std::vector &RCs = T.getRegisterClasses(); @@ -201,68 +198,10 @@ static MVT::SimpleValueType getRegisterValueType(Record *R, const CodeGenTarget return VT; } - -/// RemoveAllTypes - A quick recursive walk over a pattern which removes all -/// type information from it. -static void RemoveAllTypes(TreePatternNode *N) { - N->removeTypes(); - if (!N->isLeaf()) - for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) - RemoveAllTypes(N->getChild(i)); -} - -/// NodeHasProperty - return true if TreePatternNode has the specified -/// property. -static bool NodeHasProperty(TreePatternNode *N, SDNP Property, - CodeGenDAGPatterns &CGP) { - if (N->isLeaf()) { - const ComplexPattern *CP = NodeGetComplexPattern(N, CGP); - if (CP) - return CP->hasProperty(Property); - return false; - } - Record *Operator = N->getOperator(); - if (!Operator->isSubClassOf("SDNode")) return false; - - return CGP.getSDNodeInfo(Operator).hasProperty(Property); -} - -static bool PatternHasProperty(TreePatternNode *N, SDNP Property, - CodeGenDAGPatterns &CGP) { - if (NodeHasProperty(N, Property, CGP)) - return true; - - for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { - TreePatternNode *Child = N->getChild(i); - if (PatternHasProperty(Child, Property, CGP)) - return true; - } - - return false; -} - static std::string getOpcodeName(Record *Op, CodeGenDAGPatterns &CGP) { return CGP.getSDNodeInfo(Op).getEnumName(); } -static -bool DisablePatternForFastISel(TreePatternNode *N, CodeGenDAGPatterns &CGP) { - bool isStore = !N->isLeaf() && - getOpcodeName(N->getOperator(), CGP) == "ISD::STORE"; - if (!isStore && NodeHasProperty(N, SDNPHasChain, CGP)) - return false; - - bool HasChain = false; - for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { - TreePatternNode *Child = N->getChild(i); - if (PatternHasProperty(Child, SDNPHasChain, CGP)) { - HasChain = true; - break; - } - } - return HasChain; -} - //===----------------------------------------------------------------------===// // Node Transformation emitter implementation. // @@ -324,14 +263,14 @@ void DAGISelEmitter::EmitPredicateFunctions(raw_ostream &OS) { if (P->getOnlyTree()->isLeaf()) OS << "inline bool Predicate_" << PatFragRecord->getName() - << "(SDNode *N) {\n"; + << "(SDNode *N) const {\n"; else { std::string ClassName = CGP.getSDNodeInfo(P->getOnlyTree()->getOperator()).getSDClassName(); const char *C2 = ClassName == "SDNode" ? "N" : "inN"; OS << "inline bool Predicate_" << PatFragRecord->getName() - << "(SDNode *" << C2 << ") {\n"; + << "(SDNode *" << C2 << ") const {\n"; if (ClassName != "SDNode") OS << " " << ClassName << " *N = cast<" << ClassName << ">(inN);\n"; } @@ -360,8 +299,6 @@ private: // Node to name mapping std::map VariableMap; - // Node to operator mapping - std::map OperatorMap; // Name of the folded node which produces a flag. std::pair FoldedFlag; // Names of all the folded nodes which produce chains. @@ -447,758 +384,800 @@ public: /// matches, and the SDNode for the result has the RootName specified name. void EmitMatchCode(TreePatternNode *N, TreePatternNode *P, const std::string &RootName, const std::string &ChainSuffix, - bool &FoundChain) { + bool &FoundChain); - // Save loads/stores matched by a pattern. - if (!N->isLeaf() && N->getName().empty()) { - if (NodeHasProperty(N, SDNPMemOperand, CGP)) - LSI.push_back(RootName); - } - - bool isRoot = (P == NULL); - // Emit instruction predicates. Each predicate is just a string for now. - if (isRoot) { - // Record input varargs info. - NumInputRootOps = N->getNumChildren(); - - if (DisablePatternForFastISel(N, CGP)) - emitCheck("OptLevel != CodeGenOpt::None"); + void EmitChildMatchCode(TreePatternNode *Child, TreePatternNode *Parent, + const std::string &RootName, + const std::string &ChainSuffix, bool &FoundChain); - emitCheck(PredicateCheck); - } + /// EmitResultCode - Emit the action for a pattern. Now that it has matched + /// we actually have to build a DAG! + std::vector + EmitResultCode(TreePatternNode *N, std::vector DstRegs, + bool InFlagDecled, bool ResNodeDecled, + bool LikeLeaf = false, bool isRoot = false); - if (N->isLeaf()) { - if (IntInit *II = dynamic_cast(N->getLeafValue())) { - emitCheck("cast(" + RootName + - ")->getSExtValue() == INT64_C(" + - itostr(II->getValue()) + ")"); - return; - } else if (!NodeIsComplexPattern(N)) { - assert(0 && "Cannot match this as a leaf value!"); - abort(); - } + /// InsertOneTypeCheck - Insert a type-check for an unresolved type in 'Pat' + /// and add it to the tree. 'Pat' and 'Other' are isomorphic trees except that + /// 'Pat' may be missing types. If we find an unresolved type to add a check + /// for, this returns true otherwise false if Pat has all types. + bool InsertOneTypeCheck(TreePatternNode *Pat, TreePatternNode *Other, + const std::string &Prefix, bool isRoot = false) { + // Did we find one? + if (Pat->getExtTypes() != Other->getExtTypes()) { + // Move a type over from 'other' to 'pat'. + Pat->setTypes(Other->getExtTypes()); + // The top level node type is checked outside of the select function. + if (!isRoot) + emitCheck(Prefix + ".getValueType() == " + + getName(Pat->getTypeNum(0))); + return true; } - // If this node has a name associated with it, capture it in VariableMap. If - // we already saw this in the pattern, emit code to verify dagness. - if (!N->getName().empty()) { - std::string &VarMapEntry = VariableMap[N->getName()]; - if (VarMapEntry.empty()) { - VarMapEntry = RootName; - } else { - // If we get here, this is a second reference to a specific name. Since - // we already have checked that the first reference is valid, we don't - // have to recursively match it, just check that it's the same as the - // previously named thing. - emitCheck(VarMapEntry + " == " + RootName); - return; - } - - if (!N->isLeaf()) - OperatorMap[N->getName()] = N->getOperator(); - } - + unsigned OpNo = (unsigned)Pat->NodeHasProperty(SDNPHasChain, CGP); + for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i, ++OpNo) + if (InsertOneTypeCheck(Pat->getChild(i), Other->getChild(i), + Prefix + utostr(OpNo))) + return true; + return false; + } - // Emit code to load the child nodes and match their contents recursively. - unsigned OpNo = 0; - bool NodeHasChain = NodeHasProperty (N, SDNPHasChain, CGP); - bool HasChain = PatternHasProperty(N, SDNPHasChain, CGP); - bool EmittedUseCheck = false; - if (HasChain) { - if (NodeHasChain) - OpNo = 1; - if (!isRoot) { - // Multiple uses of actual result? - emitCheck(RootName + ".hasOneUse()"); - EmittedUseCheck = true; - if (NodeHasChain) { - // If the immediate use can somehow reach this node through another - // path, then can't fold it either or it will create a cycle. - // e.g. In the following diagram, XX can reach ld through YY. If - // ld is folded into XX, then YY is both a predecessor and a successor - // of XX. - // - // [ld] - // ^ ^ - // | | - // / \--- - // / [YY] - // | ^ - // [XX]-------| - bool NeedCheck = P != Pattern; - if (!NeedCheck) { - const SDNodeInfo &PInfo = CGP.getSDNodeInfo(P->getOperator()); - NeedCheck = - P->getOperator() == CGP.get_intrinsic_void_sdnode() || - P->getOperator() == CGP.get_intrinsic_w_chain_sdnode() || - P->getOperator() == CGP.get_intrinsic_wo_chain_sdnode() || - PInfo.getNumOperands() > 1 || - PInfo.hasProperty(SDNPHasChain) || - PInfo.hasProperty(SDNPInFlag) || - PInfo.hasProperty(SDNPOptInFlag); +private: + /// EmitInFlagSelectCode - Emit the flag operands for the DAG that is + /// being built. + void EmitInFlagSelectCode(TreePatternNode *N, const std::string &RootName, + bool &ChainEmitted, bool &InFlagDecled, + bool &ResNodeDecled, bool isRoot = false) { + const CodeGenTarget &T = CGP.getTargetInfo(); + unsigned OpNo = (unsigned)N->NodeHasProperty(SDNPHasChain, CGP); + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i, ++OpNo) { + TreePatternNode *Child = N->getChild(i); + if (!Child->isLeaf()) { + EmitInFlagSelectCode(Child, RootName + utostr(OpNo), ChainEmitted, + InFlagDecled, ResNodeDecled); + } else { + if (DefInit *DI = dynamic_cast(Child->getLeafValue())) { + if (!Child->getName().empty()) { + std::string Name = RootName + utostr(OpNo); + if (Duplicates.find(Name) != Duplicates.end()) + // A duplicate! Do not emit a copy for this node. + continue; } - if (NeedCheck) { - std::string ParentName(RootName.begin(), RootName.end()-1); - emitCheck("IsLegalAndProfitableToFold(" + RootName + - ".getNode(), " + ParentName + ".getNode(), N.getNode())"); + Record *RR = DI->getDef(); + if (RR->isSubClassOf("Register")) { + MVT::SimpleValueType RVT = getRegisterValueType(RR, T); + if (RVT == MVT::Flag) { + if (!InFlagDecled) { + emitCode("SDValue InFlag = " + + getValueName(RootName + utostr(OpNo)) + ";"); + InFlagDecled = true; + } else + emitCode("InFlag = " + + getValueName(RootName + utostr(OpNo)) + ";"); + } else { + if (!ChainEmitted) { + emitCode("SDValue Chain = CurDAG->getEntryNode();"); + ChainName = "Chain"; + ChainEmitted = true; + } + if (!InFlagDecled) { + emitCode("SDValue InFlag(0, 0);"); + InFlagDecled = true; + } + std::string Decl = (!ResNodeDecled) ? "SDNode *" : ""; + emitCode(Decl + "ResNode = CurDAG->getCopyToReg(" + ChainName + + ", " + getNodeName(RootName) + "->getDebugLoc()" + + ", " + getQualifiedName(RR) + + ", " + getValueName(RootName + utostr(OpNo)) + + ", InFlag).getNode();"); + ResNodeDecled = true; + emitCode(ChainName + " = SDValue(ResNode, 0);"); + emitCode("InFlag = SDValue(ResNode, 1);"); + } } } } - - if (NodeHasChain) { - if (FoundChain) { - emitCheck("(" + ChainName + ".getNode() == " + RootName + ".getNode() || " - "IsChainCompatible(" + ChainName + ".getNode(), " + - RootName + ".getNode()))"); - OrigChains.push_back(std::make_pair(ChainName, RootName)); - } else - FoundChain = true; - ChainName = "Chain" + ChainSuffix; - emitInit("SDValue " + ChainName + " = " + RootName + - ".getOperand(0);"); - } - } - - // Don't fold any node which reads or writes a flag and has multiple uses. - // FIXME: We really need to separate the concepts of flag and "glue". Those - // real flag results, e.g. X86CMP output, can have multiple uses. - // FIXME: If the optional incoming flag does not exist. Then it is ok to - // fold it. - if (!isRoot && - (PatternHasProperty(N, SDNPInFlag, CGP) || - PatternHasProperty(N, SDNPOptInFlag, CGP) || - PatternHasProperty(N, SDNPOutFlag, CGP))) { - if (!EmittedUseCheck) { - // Multiple uses of actual result? - emitCheck(RootName + ".hasOneUse()"); - } - } - - // If there are node predicates for this, emit the calls. - for (unsigned i = 0, e = N->getPredicateFns().size(); i != e; ++i) - emitCheck(N->getPredicateFns()[i] + "(" + RootName + ".getNode())"); - - // If this is an 'and R, 1234' where the operation is AND/OR and the RHS is - // a constant without a predicate fn that has more that one bit set, handle - // this as a special case. This is usually for targets that have special - // handling of certain large constants (e.g. alpha with it's 8/16/32-bit - // handling stuff). Using these instructions is often far more efficient - // than materializing the constant. Unfortunately, both the instcombiner - // and the dag combiner can often infer that bits are dead, and thus drop - // them from the mask in the dag. For example, it might turn 'AND X, 255' - // into 'AND X, 254' if it knows the low bit is set. Emit code that checks - // to handle this. - if (!N->isLeaf() && - (N->getOperator()->getName() == "and" || - N->getOperator()->getName() == "or") && - N->getChild(1)->isLeaf() && - N->getChild(1)->getPredicateFns().empty()) { - if (IntInit *II = dynamic_cast(N->getChild(1)->getLeafValue())) { - if (!isPowerOf2_32(II->getValue())) { // Don't bother with single bits. - emitInit("SDValue " + RootName + "0" + " = " + - RootName + ".getOperand(" + utostr(0) + ");"); - emitInit("SDValue " + RootName + "1" + " = " + - RootName + ".getOperand(" + utostr(1) + ");"); - - unsigned NTmp = TmpNo++; - emitCode("ConstantSDNode *Tmp" + utostr(NTmp) + - " = dyn_cast(" + RootName + "1);"); - emitCheck("Tmp" + utostr(NTmp)); - const char *MaskPredicate = N->getOperator()->getName() == "or" - ? "CheckOrMask(" : "CheckAndMask("; - emitCheck(MaskPredicate + RootName + "0, Tmp" + utostr(NTmp) + - ", INT64_C(" + itostr(II->getValue()) + "))"); - - EmitChildMatchCode(N->getChild(0), N, RootName + utostr(0), RootName, - ChainSuffix + utostr(0), FoundChain); - return; - } - } } - - for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i, ++OpNo) { - emitInit("SDValue " + RootName + utostr(OpNo) + " = " + - RootName + ".getOperand(" +utostr(OpNo) + ");"); - EmitChildMatchCode(N->getChild(i), N, RootName + utostr(OpNo), RootName, - ChainSuffix + utostr(OpNo), FoundChain); + if (N->NodeHasProperty(SDNPInFlag, CGP)) { + if (!InFlagDecled) { + emitCode("SDValue InFlag = " + getNodeName(RootName) + + "->getOperand(" + utostr(OpNo) + ");"); + InFlagDecled = true; + } else + abort(); + emitCode("InFlag = " + getNodeName(RootName) + + "->getOperand(" + utostr(OpNo) + ");"); } + } +}; - // Handle cases when root is a complex pattern. - const ComplexPattern *CP; - if (isRoot && N->isLeaf() && (CP = NodeGetComplexPattern(N, CGP))) { - std::string Fn = CP->getSelectFunc(); - unsigned NumOps = CP->getNumOperands(); - for (unsigned i = 0; i < NumOps; ++i) { - emitDecl("CPTmp" + RootName + "_" + utostr(i)); - emitCode("SDValue CPTmp" + RootName + "_" + utostr(i) + ";"); - } - if (CP->hasProperty(SDNPHasChain)) { - emitDecl("CPInChain"); - emitDecl("Chain" + ChainSuffix); - emitCode("SDValue CPInChain;"); - emitCode("SDValue Chain" + ChainSuffix + ";"); - } - std::string Code = Fn + "(" + RootName + ", " + RootName; - for (unsigned i = 0; i < NumOps; i++) - Code += ", CPTmp" + RootName + "_" + utostr(i); - if (CP->hasProperty(SDNPHasChain)) { - ChainName = "Chain" + ChainSuffix; - Code += ", CPInChain, Chain" + ChainSuffix; - } - emitCheck(Code + ")"); +/// EmitMatchCode - Emit a matcher for N, going to the label for PatternNo +/// if the match fails. At this point, we already know that the opcode for N +/// matches, and the SDNode for the result has the RootName specified name. +void PatternCodeEmitter::EmitMatchCode(TreePatternNode *N, TreePatternNode *P, + const std::string &RootName, + const std::string &ChainSuffix, + bool &FoundChain) { + // Save loads/stores matched by a pattern. + if (!N->isLeaf() && N->getName().empty()) { + if (N->NodeHasProperty(SDNPMemOperand, CGP)) + LSI.push_back(getNodeName(RootName)); + } + + bool isRoot = (P == NULL); + // Emit instruction predicates. Each predicate is just a string for now. + if (isRoot) { + // Record input varargs info. + NumInputRootOps = N->getNumChildren(); + emitCheck(PredicateCheck); + } + + if (N->isLeaf()) { + if (IntInit *II = dynamic_cast(N->getLeafValue())) { + emitCheck("cast(" + getNodeName(RootName) + + ")->getSExtValue() == INT64_C(" + + itostr(II->getValue()) + ")"); + return; } + assert(N->getComplexPatternInfo(CGP) != 0 && + "Cannot match this as a leaf value!"); } - - void EmitChildMatchCode(TreePatternNode *Child, TreePatternNode *Parent, - const std::string &RootName, - const std::string &ParentRootName, - const std::string &ChainSuffix, bool &FoundChain) { - if (!Child->isLeaf()) { - // If it's not a leaf, recursively match. - const SDNodeInfo &CInfo = CGP.getSDNodeInfo(Child->getOperator()); - emitCheck(RootName + ".getOpcode() == " + - CInfo.getEnumName()); - EmitMatchCode(Child, Parent, RootName, ChainSuffix, FoundChain); - bool HasChain = false; - if (NodeHasProperty(Child, SDNPHasChain, CGP)) { - HasChain = true; - FoldedChains.push_back(std::make_pair(RootName, CInfo.getNumResults())); - } - if (NodeHasProperty(Child, SDNPOutFlag, CGP)) { - assert(FoldedFlag.first == "" && FoldedFlag.second == 0 && - "Pattern folded multiple nodes which produce flags?"); - FoldedFlag = std::make_pair(RootName, - CInfo.getNumResults() + (unsigned)HasChain); - } + + // If this node has a name associated with it, capture it in VariableMap. If + // we already saw this in the pattern, emit code to verify dagness. + if (!N->getName().empty()) { + std::string &VarMapEntry = VariableMap[N->getName()]; + if (VarMapEntry.empty()) { + VarMapEntry = RootName; } else { - // If this child has a name associated with it, capture it in VarMap. If - // we already saw this in the pattern, emit code to verify dagness. - if (!Child->getName().empty()) { - std::string &VarMapEntry = VariableMap[Child->getName()]; - if (VarMapEntry.empty()) { - VarMapEntry = RootName; + // If we get here, this is a second reference to a specific name. Since + // we already have checked that the first reference is valid, we don't + // have to recursively match it, just check that it's the same as the + // previously named thing. + emitCheck(VarMapEntry + " == " + RootName); + return; + } + } + + + // Emit code to load the child nodes and match their contents recursively. + unsigned OpNo = 0; + bool NodeHasChain = N->NodeHasProperty(SDNPHasChain, CGP); + bool HasChain = N->TreeHasProperty(SDNPHasChain, CGP); + if (HasChain) { + if (NodeHasChain) + OpNo = 1; + if (!isRoot) { + // Check if it's profitable to fold the node. e.g. Check for multiple uses + // of actual result? + std::string ParentName(RootName.begin(), RootName.end()-1); + if (!NodeHasChain) { + // If this is just an interior node, check to see if it has a single + // use. If the node has multiple uses and the pattern has a load as + // an operand, then we can't fold the load. + emitCheck(getValueName(RootName) + ".hasOneUse()"); + } else if (!N->isLeaf()) { // ComplexPatterns do their own legality check. + // If the immediate use can somehow reach this node through another + // path, then can't fold it either or it will create a cycle. + // e.g. In the following diagram, XX can reach ld through YY. If + // ld is folded into XX, then YY is both a predecessor and a successor + // of XX. + // + // [ld] + // ^ ^ + // | | + // / \--- + // / [YY] + // | ^ + // [XX]-------| + + // We know we need the check if N's parent is not the root. + bool NeedCheck = P != Pattern; + if (!NeedCheck) { + // If the parent is the root and the node has more than one operand, + // we need to check. + const SDNodeInfo &PInfo = CGP.getSDNodeInfo(P->getOperator()); + NeedCheck = + P->getOperator() == CGP.get_intrinsic_void_sdnode() || + P->getOperator() == CGP.get_intrinsic_w_chain_sdnode() || + P->getOperator() == CGP.get_intrinsic_wo_chain_sdnode() || + PInfo.getNumOperands() > 1 || + PInfo.hasProperty(SDNPHasChain) || + PInfo.hasProperty(SDNPInFlag) || + PInfo.hasProperty(SDNPOptInFlag); + } + + if (NeedCheck) { + emitCheck("IsProfitableToFold(" + getValueName(RootName) + + ", " + getNodeName(ParentName) + ", N)"); + emitCheck("IsLegalToFold(" + getValueName(RootName) + + ", " + getNodeName(ParentName) + ", N)"); } else { - // If we get here, this is a second reference to a specific name. - // Since we already have checked that the first reference is valid, - // we don't have to recursively match it, just check that it's the - // same as the previously named thing. - emitCheck(VarMapEntry + " == " + RootName); - Duplicates.insert(RootName); - return; + // Otherwise, just verify that the node only has a single use. + emitCheck(getValueName(RootName) + ".hasOneUse()"); } } + } + + if (NodeHasChain) { + if (FoundChain) { + emitCheck("IsChainCompatible(" + ChainName + ".getNode(), " + + getNodeName(RootName) + ")"); + OrigChains.push_back(std::make_pair(ChainName, + getValueName(RootName))); + } else + FoundChain = true; + ChainName = "Chain" + ChainSuffix; - // Handle leaves of various types. - if (DefInit *DI = dynamic_cast(Child->getLeafValue())) { - Record *LeafRec = DI->getDef(); - if (LeafRec->isSubClassOf("RegisterClass") || - LeafRec->isSubClassOf("PointerLikeRegClass")) { - // Handle register references. Nothing to do here. - } else if (LeafRec->isSubClassOf("Register")) { - // Handle register references. - } else if (LeafRec->isSubClassOf("ComplexPattern")) { - // Handle complex pattern. - const ComplexPattern *CP = NodeGetComplexPattern(Child, CGP); - std::string Fn = CP->getSelectFunc(); - unsigned NumOps = CP->getNumOperands(); - for (unsigned i = 0; i < NumOps; ++i) { - emitDecl("CPTmp" + RootName + "_" + utostr(i)); - emitCode("SDValue CPTmp" + RootName + "_" + utostr(i) + ";"); - } - if (CP->hasProperty(SDNPHasChain)) { - const SDNodeInfo &PInfo = CGP.getSDNodeInfo(Parent->getOperator()); - FoldedChains.push_back(std::make_pair("CPInChain", - PInfo.getNumResults())); - ChainName = "Chain" + ChainSuffix; - emitDecl("CPInChain"); - emitDecl(ChainName); - emitCode("SDValue CPInChain;"); - emitCode("SDValue " + ChainName + ";"); - } - - std::string Code = Fn + "("; - if (CP->hasAttribute(CPAttrParentAsRoot)) { - Code += ParentRootName + ", "; - } else { - Code += "N, "; - } - if (CP->hasProperty(SDNPHasChain)) { - std::string ParentName(RootName.begin(), RootName.end()-1); - Code += ParentName + ", "; - } - Code += RootName; - for (unsigned i = 0; i < NumOps; i++) - Code += ", CPTmp" + RootName + "_" + utostr(i); - if (CP->hasProperty(SDNPHasChain)) - Code += ", CPInChain, Chain" + ChainSuffix; - emitCheck(Code + ")"); - } else if (LeafRec->getName() == "srcvalue") { - // Place holder for SRCVALUE nodes. Nothing to do here. - } else if (LeafRec->isSubClassOf("ValueType")) { - // Make sure this is the specified value type. - emitCheck("cast(" + RootName + - ")->getVT() == MVT::" + LeafRec->getName()); - } else if (LeafRec->isSubClassOf("CondCode")) { - // Make sure this is the specified cond code. - emitCheck("cast(" + RootName + - ")->get() == ISD::" + LeafRec->getName()); - } else { -#ifndef NDEBUG - Child->dump(); - errs() << " "; -#endif - assert(0 && "Unknown leaf type!"); - } + if (!N->getComplexPatternInfo(CGP) || + isRoot) + emitInit("SDValue " + ChainName + " = " + getNodeName(RootName) + + "->getOperand(0);"); + } + } + + // If there are node predicates for this, emit the calls. + for (unsigned i = 0, e = N->getPredicateFns().size(); i != e; ++i) + emitCheck(N->getPredicateFns()[i] + "(" + getNodeName(RootName) + ")"); + + // If this is an 'and R, 1234' where the operation is AND/OR and the RHS is + // a constant without a predicate fn that has more that one bit set, handle + // this as a special case. This is usually for targets that have special + // handling of certain large constants (e.g. alpha with it's 8/16/32-bit + // handling stuff). Using these instructions is often far more efficient + // than materializing the constant. Unfortunately, both the instcombiner + // and the dag combiner can often infer that bits are dead, and thus drop + // them from the mask in the dag. For example, it might turn 'AND X, 255' + // into 'AND X, 254' if it knows the low bit is set. Emit code that checks + // to handle this. + if (!N->isLeaf() && + (N->getOperator()->getName() == "and" || + N->getOperator()->getName() == "or") && + N->getChild(1)->isLeaf() && + N->getChild(1)->getPredicateFns().empty()) { + if (IntInit *II = dynamic_cast(N->getChild(1)->getLeafValue())) { + if (!isPowerOf2_32(II->getValue())) { // Don't bother with single bits. + emitInit("SDValue " + RootName + "0" + " = " + + getNodeName(RootName) + "->getOperand(" + utostr(0) + ");"); + emitInit("SDValue " + RootName + "1" + " = " + + getNodeName(RootName) + "->getOperand(" + utostr(1) + ");"); - // If there are node predicates for this, emit the calls. - for (unsigned i = 0, e = Child->getPredicateFns().size(); i != e; ++i) - emitCheck(Child->getPredicateFns()[i] + "(" + RootName + - ".getNode())"); - } else if (IntInit *II = - dynamic_cast(Child->getLeafValue())) { unsigned NTmp = TmpNo++; - emitCode("ConstantSDNode *Tmp"+ utostr(NTmp) + - " = dyn_cast("+ - RootName + ");"); + emitCode("ConstantSDNode *Tmp" + utostr(NTmp) + + " = dyn_cast(" + + getNodeName(RootName + "1") + ");"); emitCheck("Tmp" + utostr(NTmp)); - unsigned CTmp = TmpNo++; - emitCode("int64_t CN"+ utostr(CTmp) + - " = Tmp" + utostr(NTmp) + "->getSExtValue();"); - emitCheck("CN" + utostr(CTmp) + " == " - "INT64_C(" +itostr(II->getValue()) + ")"); - } else { -#ifndef NDEBUG - Child->dump(); -#endif - assert(0 && "Unknown leaf type!"); + const char *MaskPredicate = N->getOperator()->getName() == "or" + ? "CheckOrMask(" : "CheckAndMask("; + emitCheck(MaskPredicate + getValueName(RootName + "0") + + ", Tmp" + utostr(NTmp) + + ", INT64_C(" + itostr(II->getValue()) + "))"); + + EmitChildMatchCode(N->getChild(0), N, RootName + utostr(0), + ChainSuffix + utostr(0), FoundChain); + return; } } } + + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i, ++OpNo) { + emitInit("SDValue " + getValueName(RootName + utostr(OpNo)) + " = " + + getNodeName(RootName) + "->getOperand(" + utostr(OpNo) + ");"); + + EmitChildMatchCode(N->getChild(i), N, RootName + utostr(OpNo), + ChainSuffix + utostr(OpNo), FoundChain); + } + + // Handle complex patterns. + if (const ComplexPattern *CP = N->getComplexPatternInfo(CGP)) { + std::string Fn = CP->getSelectFunc(); + unsigned NumOps = CP->getNumOperands(); + for (unsigned i = 0; i < NumOps; ++i) { + emitDecl("CPTmp" + RootName + "_" + utostr(i)); + emitCode("SDValue CPTmp" + RootName + "_" + utostr(i) + ";"); + } + if (CP->hasProperty(SDNPHasChain)) { + emitDecl("CPInChain"); + emitDecl("Chain" + ChainSuffix); + emitCode("SDValue CPInChain;"); + emitCode("SDValue Chain" + ChainSuffix + ";"); + } + + std::string Code = Fn + "(N, "; // always pass in the root. + Code += getValueName(RootName); + for (unsigned i = 0; i < NumOps; i++) + Code += ", CPTmp" + RootName + "_" + utostr(i); + if (CP->hasProperty(SDNPHasChain)) { + ChainName = "Chain" + ChainSuffix; + Code += ", CPInChain, " + ChainName; + } + emitCheck(Code + ")"); + } +} - /// EmitResultCode - Emit the action for a pattern. Now that it has matched - /// we actually have to build a DAG! - std::vector - EmitResultCode(TreePatternNode *N, std::vector DstRegs, - bool InFlagDecled, bool ResNodeDecled, - bool LikeLeaf = false, bool isRoot = false) { - // List of arguments of getMachineNode() or SelectNodeTo(). - std::vector NodeOps; - // This is something selected from the pattern we matched. - if (!N->getName().empty()) { - const std::string &VarName = N->getName(); - std::string Val = VariableMap[VarName]; - bool ModifiedVal = false; - if (Val.empty()) { - errs() << "Variable '" << VarName << " referenced but not defined " - << "and not caught earlier!\n"; - abort(); - } - if (Val[0] == 'T' && Val[1] == 'm' && Val[2] == 'p') { - // Already selected this operand, just return the tmpval. - NodeOps.push_back(Val); - return NodeOps; - } +void PatternCodeEmitter::EmitChildMatchCode(TreePatternNode *Child, + TreePatternNode *Parent, + const std::string &RootName, + const std::string &ChainSuffix, + bool &FoundChain) { + if (!Child->isLeaf()) { + // If it's not a leaf, recursively match. + const SDNodeInfo &CInfo = CGP.getSDNodeInfo(Child->getOperator()); + emitCheck(getNodeName(RootName) + "->getOpcode() == " + + CInfo.getEnumName()); + EmitMatchCode(Child, Parent, RootName, ChainSuffix, FoundChain); + bool HasChain = false; + if (Child->NodeHasProperty(SDNPHasChain, CGP)) { + HasChain = true; + FoldedChains.push_back(std::make_pair(getValueName(RootName), + CInfo.getNumResults())); + } + if (Child->NodeHasProperty(SDNPOutFlag, CGP)) { + assert(FoldedFlag.first == "" && FoldedFlag.second == 0 && + "Pattern folded multiple nodes which produce flags?"); + FoldedFlag = std::make_pair(getValueName(RootName), + CInfo.getNumResults() + (unsigned)HasChain); + } + return; + } + + if (const ComplexPattern *CP = Child->getComplexPatternInfo(CGP)) { + EmitMatchCode(Child, Parent, RootName, ChainSuffix, FoundChain); + bool HasChain = false; - const ComplexPattern *CP; - unsigned ResNo = TmpNo++; - if (!N->isLeaf() && N->getOperator()->getName() == "imm") { - assert(N->getExtTypes().size() == 1 && "Multiple types not handled!"); - std::string CastType; - std::string TmpVar = "Tmp" + utostr(ResNo); - switch (N->getTypeNum(0)) { + if (Child->NodeHasProperty(SDNPHasChain, CGP)) { + HasChain = true; + const SDNodeInfo &PInfo = CGP.getSDNodeInfo(Parent->getOperator()); + FoldedChains.push_back(std::make_pair("CPInChain", + PInfo.getNumResults())); + } + if (Child->NodeHasProperty(SDNPOutFlag, CGP)) { + assert(FoldedFlag.first == "" && FoldedFlag.second == 0 && + "Pattern folded multiple nodes which produce flags?"); + FoldedFlag = std::make_pair(getValueName(RootName), + CP->getNumOperands() + (unsigned)HasChain); + } + return; + } + + // If this child has a name associated with it, capture it in VarMap. If + // we already saw this in the pattern, emit code to verify dagness. + if (!Child->getName().empty()) { + std::string &VarMapEntry = VariableMap[Child->getName()]; + if (VarMapEntry.empty()) { + VarMapEntry = getValueName(RootName); + } else { + // If we get here, this is a second reference to a specific name. + // Since we already have checked that the first reference is valid, + // we don't have to recursively match it, just check that it's the + // same as the previously named thing. + emitCheck(VarMapEntry + " == " + getValueName(RootName)); + Duplicates.insert(getValueName(RootName)); + return; + } + } + + // Handle leaves of various types. + if (DefInit *DI = dynamic_cast(Child->getLeafValue())) { + Record *LeafRec = DI->getDef(); + if (LeafRec->isSubClassOf("RegisterClass") || + LeafRec->isSubClassOf("PointerLikeRegClass")) { + // Handle register references. Nothing to do here. + } else if (LeafRec->isSubClassOf("Register")) { + // Handle register references. + } else if (LeafRec->getName() == "srcvalue") { + // Place holder for SRCVALUE nodes. Nothing to do here. + } else if (LeafRec->isSubClassOf("ValueType")) { + // Make sure this is the specified value type. + emitCheck("cast(" + getNodeName(RootName) + + ")->getVT() == MVT::" + LeafRec->getName()); + } else if (LeafRec->isSubClassOf("CondCode")) { + // Make sure this is the specified cond code. + emitCheck("cast(" + getNodeName(RootName) + + ")->get() == ISD::" + LeafRec->getName()); + } else { +#ifndef NDEBUG + Child->dump(); + errs() << " "; +#endif + assert(0 && "Unknown leaf type!"); + } + + // If there are node predicates for this, emit the calls. + for (unsigned i = 0, e = Child->getPredicateFns().size(); i != e; ++i) + emitCheck(Child->getPredicateFns()[i] + "(" + getNodeName(RootName) + + ")"); + return; + } + + if (IntInit *II = dynamic_cast(Child->getLeafValue())) { + unsigned NTmp = TmpNo++; + emitCode("ConstantSDNode *Tmp"+ utostr(NTmp) + + " = dyn_cast("+ + getNodeName(RootName) + ");"); + emitCheck("Tmp" + utostr(NTmp)); + unsigned CTmp = TmpNo++; + emitCode("int64_t CN"+ utostr(CTmp) + + " = Tmp" + utostr(NTmp) + "->getSExtValue();"); + emitCheck("CN" + utostr(CTmp) + " == " + "INT64_C(" +itostr(II->getValue()) + ")"); + return; + } +#ifndef NDEBUG + Child->dump(); +#endif + assert(0 && "Unknown leaf type!"); +} + +/// EmitResultCode - Emit the action for a pattern. Now that it has matched +/// we actually have to build a DAG! +std::vector +PatternCodeEmitter::EmitResultCode(TreePatternNode *N, + std::vector DstRegs, + bool InFlagDecled, bool ResNodeDecled, + bool LikeLeaf, bool isRoot) { + // List of arguments of getMachineNode() or SelectNodeTo(). + std::vector NodeOps; + // This is something selected from the pattern we matched. + if (!N->getName().empty()) { + const std::string &VarName = N->getName(); + std::string Val = VariableMap[VarName]; + if (Val.empty()) { + errs() << "Variable '" << VarName << " referenced but not defined " + << "and not caught earlier!\n"; + abort(); + } + + unsigned ResNo = TmpNo++; + if (!N->isLeaf() && N->getOperator()->getName() == "imm") { + assert(N->getExtTypes().size() == 1 && "Multiple types not handled!"); + std::string CastType; + std::string TmpVar = "Tmp" + utostr(ResNo); + switch (N->getTypeNum(0)) { default: errs() << "Cannot handle " << getEnumName(N->getTypeNum(0)) - << " type as an immediate constant. Aborting\n"; + << " type as an immediate constant. Aborting\n"; abort(); case MVT::i1: CastType = "bool"; break; case MVT::i8: CastType = "unsigned char"; break; case MVT::i16: CastType = "unsigned short"; break; case MVT::i32: CastType = "unsigned"; break; case MVT::i64: CastType = "uint64_t"; break; - } - emitCode("SDValue " + TmpVar + - " = CurDAG->getTargetConstant(((" + CastType + - ") cast(" + Val + ")->getZExtValue()), " + - getEnumName(N->getTypeNum(0)) + ");"); - // Add Tmp to VariableMap, so that we don't multiply select this - // value if used multiple times by this pattern result. - Val = TmpVar; - ModifiedVal = true; - NodeOps.push_back(Val); - } else if (!N->isLeaf() && N->getOperator()->getName() == "fpimm") { - assert(N->getExtTypes().size() == 1 && "Multiple types not handled!"); - std::string TmpVar = "Tmp" + utostr(ResNo); - emitCode("SDValue " + TmpVar + - " = CurDAG->getTargetConstantFP(*cast(" + - Val + ")->getConstantFPValue(), cast(" + - Val + ")->getValueType(0));"); - // Add Tmp to VariableMap, so that we don't multiply select this - // value if used multiple times by this pattern result. - Val = TmpVar; - ModifiedVal = true; - NodeOps.push_back(Val); - } else if (!N->isLeaf() && N->getOperator()->getName() == "texternalsym"){ - Record *Op = OperatorMap[N->getName()]; - // Transform ExternalSymbol to TargetExternalSymbol - if (Op && Op->getName() == "externalsym") { - std::string TmpVar = "Tmp"+utostr(ResNo); - emitCode("SDValue " + TmpVar + " = CurDAG->getTarget" - "ExternalSymbol(cast(" + - Val + ")->getSymbol(), " + - getEnumName(N->getTypeNum(0)) + ");"); - // Add Tmp to VariableMap, so that we don't multiply select - // this value if used multiple times by this pattern result. - Val = TmpVar; - ModifiedVal = true; - } - NodeOps.push_back(Val); - } else if (!N->isLeaf() && (N->getOperator()->getName() == "tglobaladdr" - || N->getOperator()->getName() == "tglobaltlsaddr")) { - Record *Op = OperatorMap[N->getName()]; - // Transform GlobalAddress to TargetGlobalAddress - if (Op && (Op->getName() == "globaladdr" || - Op->getName() == "globaltlsaddr")) { - std::string TmpVar = "Tmp" + utostr(ResNo); - emitCode("SDValue " + TmpVar + " = CurDAG->getTarget" - "GlobalAddress(cast(" + Val + - ")->getGlobal(), " + getEnumName(N->getTypeNum(0)) + - ");"); - // Add Tmp to VariableMap, so that we don't multiply select - // this value if used multiple times by this pattern result. - Val = TmpVar; - ModifiedVal = true; - } - NodeOps.push_back(Val); - } else if (!N->isLeaf() - && (N->getOperator()->getName() == "texternalsym" - || N->getOperator()->getName() == "tconstpool")) { - // Do not rewrite the variable name, since we don't generate a new - // temporary. - NodeOps.push_back(Val); - } else if (N->isLeaf() && (CP = NodeGetComplexPattern(N, CGP))) { - for (unsigned i = 0; i < CP->getNumOperands(); ++i) { - NodeOps.push_back("CPTmp" + Val + "_" + utostr(i)); - } - } else { - // This node, probably wrapped in a SDNodeXForm, behaves like a leaf - // node even if it isn't one. Don't select it. - if (!LikeLeaf) { - if (isRoot && N->isLeaf()) { - emitCode("ReplaceUses(N, " + Val + ");"); - emitCode("return NULL;"); - } - } - NodeOps.push_back(Val); } - - if (ModifiedVal) { - VariableMap[VarName] = Val; + emitCode("SDValue " + TmpVar + + " = CurDAG->getTargetConstant(((" + CastType + + ") cast(" + Val + ")->getZExtValue()), " + + getEnumName(N->getTypeNum(0)) + ");"); + NodeOps.push_back(getValueName(TmpVar)); + } else if (!N->isLeaf() && N->getOperator()->getName() == "fpimm") { + assert(N->getExtTypes().size() == 1 && "Multiple types not handled!"); + std::string TmpVar = "Tmp" + utostr(ResNo); + emitCode("SDValue " + TmpVar + + " = CurDAG->getTargetConstantFP(*cast(" + + Val + ")->getConstantFPValue(), cast(" + + Val + ")->getValueType(0));"); + NodeOps.push_back(getValueName(TmpVar)); + } else if (const ComplexPattern *CP = N->getComplexPatternInfo(CGP)) { + for (unsigned i = 0; i < CP->getNumOperands(); ++i) + NodeOps.push_back(getValueName("CPTmp" + Val + "_" + utostr(i))); + } else { + // This node, probably wrapped in a SDNodeXForm, behaves like a leaf + // node even if it isn't one. Don't select it. + if (!LikeLeaf) { + if (isRoot && N->isLeaf()) { + emitCode("ReplaceUses(SDValue(N, 0), " + Val + ");"); + emitCode("return NULL;"); + } } - return NodeOps; + NodeOps.push_back(getValueName(Val)); } - if (N->isLeaf()) { - // If this is an explicit register reference, handle it. - if (DefInit *DI = dynamic_cast(N->getLeafValue())) { - unsigned ResNo = TmpNo++; - if (DI->getDef()->isSubClassOf("Register")) { - emitCode("SDValue Tmp" + utostr(ResNo) + " = CurDAG->getRegister(" + - getQualifiedName(DI->getDef()) + ", " + - getEnumName(N->getTypeNum(0)) + ");"); - NodeOps.push_back("Tmp" + utostr(ResNo)); - return NodeOps; - } else if (DI->getDef()->getName() == "zero_reg") { - emitCode("SDValue Tmp" + utostr(ResNo) + - " = CurDAG->getRegister(0, " + - getEnumName(N->getTypeNum(0)) + ");"); - NodeOps.push_back("Tmp" + utostr(ResNo)); - return NodeOps; - } else if (DI->getDef()->isSubClassOf("RegisterClass")) { - // Handle a reference to a register class. This is used - // in COPY_TO_SUBREG instructions. - emitCode("SDValue Tmp" + utostr(ResNo) + - " = CurDAG->getTargetConstant(" + - getQualifiedName(DI->getDef()) + "RegClassID, " + - "MVT::i32);"); - NodeOps.push_back("Tmp" + utostr(ResNo)); - return NodeOps; - } - } else if (IntInit *II = dynamic_cast(N->getLeafValue())) { - unsigned ResNo = TmpNo++; - assert(N->getExtTypes().size() == 1 && "Multiple types not handled!"); - emitCode("SDValue Tmp" + utostr(ResNo) + - " = CurDAG->getTargetConstant(0x" + - utohexstr((uint64_t) II->getValue()) + - "ULL, " + getEnumName(N->getTypeNum(0)) + ");"); - NodeOps.push_back("Tmp" + utostr(ResNo)); + return NodeOps; + } + if (N->isLeaf()) { + // If this is an explicit register reference, handle it. + if (DefInit *DI = dynamic_cast(N->getLeafValue())) { + unsigned ResNo = TmpNo++; + if (DI->getDef()->isSubClassOf("Register")) { + emitCode("SDValue Tmp" + utostr(ResNo) + " = CurDAG->getRegister(" + + getQualifiedName(DI->getDef()) + ", " + + getEnumName(N->getTypeNum(0)) + ");"); + NodeOps.push_back(getValueName("Tmp" + utostr(ResNo))); + return NodeOps; + } else if (DI->getDef()->getName() == "zero_reg") { + emitCode("SDValue Tmp" + utostr(ResNo) + + " = CurDAG->getRegister(0, " + + getEnumName(N->getTypeNum(0)) + ");"); + NodeOps.push_back(getValueName("Tmp" + utostr(ResNo))); + return NodeOps; + } else if (DI->getDef()->isSubClassOf("RegisterClass")) { + // Handle a reference to a register class. This is used + // in COPY_TO_SUBREG instructions. + emitCode("SDValue Tmp" + utostr(ResNo) + + " = CurDAG->getTargetConstant(" + + getQualifiedName(DI->getDef()) + "RegClassID, " + + "MVT::i32);"); + NodeOps.push_back(getValueName("Tmp" + utostr(ResNo))); return NodeOps; } + } else if (IntInit *II = dynamic_cast(N->getLeafValue())) { + unsigned ResNo = TmpNo++; + assert(N->getExtTypes().size() == 1 && "Multiple types not handled!"); + emitCode("SDValue Tmp" + utostr(ResNo) + + " = CurDAG->getTargetConstant(0x" + + utohexstr((uint64_t) II->getValue()) + + "ULL, " + getEnumName(N->getTypeNum(0)) + ");"); + NodeOps.push_back(getValueName("Tmp" + utostr(ResNo))); + return NodeOps; + } #ifndef NDEBUG - N->dump(); + N->dump(); #endif - assert(0 && "Unknown leaf type!"); - return NodeOps; + assert(0 && "Unknown leaf type!"); + return NodeOps; + } + + Record *Op = N->getOperator(); + if (Op->isSubClassOf("Instruction")) { + const CodeGenTarget &CGT = CGP.getTargetInfo(); + CodeGenInstruction &II = CGT.getInstruction(Op->getName()); + const DAGInstruction &Inst = CGP.getInstruction(Op); + const TreePattern *InstPat = Inst.getPattern(); + // FIXME: Assume actual pattern comes before "implicit". + TreePatternNode *InstPatNode = + isRoot ? (InstPat ? InstPat->getTree(0) : Pattern) + : (InstPat ? InstPat->getTree(0) : NULL); + if (InstPatNode && !InstPatNode->isLeaf() && + InstPatNode->getOperator()->getName() == "set") { + InstPatNode = InstPatNode->getChild(InstPatNode->getNumChildren()-1); } - - Record *Op = N->getOperator(); - if (Op->isSubClassOf("Instruction")) { - const CodeGenTarget &CGT = CGP.getTargetInfo(); - CodeGenInstruction &II = CGT.getInstruction(Op->getName()); - const DAGInstruction &Inst = CGP.getInstruction(Op); - const TreePattern *InstPat = Inst.getPattern(); - // FIXME: Assume actual pattern comes before "implicit". - TreePatternNode *InstPatNode = - isRoot ? (InstPat ? InstPat->getTree(0) : Pattern) - : (InstPat ? InstPat->getTree(0) : NULL); - if (InstPatNode && !InstPatNode->isLeaf() && - InstPatNode->getOperator()->getName() == "set") { - InstPatNode = InstPatNode->getChild(InstPatNode->getNumChildren()-1); - } - bool IsVariadic = isRoot && II.isVariadic; - // FIXME: fix how we deal with physical register operands. - bool HasImpInputs = isRoot && Inst.getNumImpOperands() > 0; - bool HasImpResults = isRoot && DstRegs.size() > 0; - bool NodeHasOptInFlag = isRoot && - PatternHasProperty(Pattern, SDNPOptInFlag, CGP); - bool NodeHasInFlag = isRoot && - PatternHasProperty(Pattern, SDNPInFlag, CGP); - bool NodeHasOutFlag = isRoot && - PatternHasProperty(Pattern, SDNPOutFlag, CGP); - bool NodeHasChain = InstPatNode && - PatternHasProperty(InstPatNode, SDNPHasChain, CGP); - bool InputHasChain = isRoot && - NodeHasProperty(Pattern, SDNPHasChain, CGP); - unsigned NumResults = Inst.getNumResults(); - unsigned NumDstRegs = HasImpResults ? DstRegs.size() : 0; - - // Record output varargs info. - OutputIsVariadic = IsVariadic; - - if (NodeHasOptInFlag) { - emitCode("bool HasInFlag = " - "(N.getOperand(N.getNumOperands()-1).getValueType() == MVT::Flag);"); - } - if (IsVariadic) - emitCode("SmallVector Ops" + utostr(OpcNo) + ";"); - - // How many results is this pattern expected to produce? - unsigned NumPatResults = 0; - for (unsigned i = 0, e = Pattern->getExtTypes().size(); i != e; i++) { - MVT::SimpleValueType VT = Pattern->getTypeNum(i); - if (VT != MVT::isVoid && VT != MVT::Flag) - NumPatResults++; + bool IsVariadic = isRoot && II.isVariadic; + // FIXME: fix how we deal with physical register operands. + bool HasImpInputs = isRoot && Inst.getNumImpOperands() > 0; + bool HasImpResults = isRoot && DstRegs.size() > 0; + bool NodeHasOptInFlag = isRoot && + Pattern->TreeHasProperty(SDNPOptInFlag, CGP); + bool NodeHasInFlag = isRoot && + Pattern->TreeHasProperty(SDNPInFlag, CGP); + bool NodeHasOutFlag = isRoot && + Pattern->TreeHasProperty(SDNPOutFlag, CGP); + bool NodeHasChain = InstPatNode && + InstPatNode->TreeHasProperty(SDNPHasChain, CGP); + bool InputHasChain = isRoot && Pattern->NodeHasProperty(SDNPHasChain, CGP); + unsigned NumResults = Inst.getNumResults(); + unsigned NumDstRegs = HasImpResults ? DstRegs.size() : 0; + + // Record output varargs info. + OutputIsVariadic = IsVariadic; + + if (NodeHasOptInFlag) { + emitCode("bool HasInFlag = " + "(N->getOperand(N->getNumOperands()-1).getValueType() == " + "MVT::Flag);"); + } + if (IsVariadic) + emitCode("SmallVector Ops" + utostr(OpcNo) + ";"); + + // How many results is this pattern expected to produce? + unsigned NumPatResults = 0; + for (unsigned i = 0, e = Pattern->getExtTypes().size(); i != e; i++) { + MVT::SimpleValueType VT = Pattern->getTypeNum(i); + if (VT != MVT::isVoid && VT != MVT::Flag) + NumPatResults++; + } + + if (OrigChains.size() > 0) { + // The original input chain is being ignored. If it is not just + // pointing to the op that's being folded, we should create a + // TokenFactor with it and the chain of the folded op as the new chain. + // We could potentially be doing multiple levels of folding, in that + // case, the TokenFactor can have more operands. + emitCode("SmallVector InChains;"); + for (unsigned i = 0, e = OrigChains.size(); i < e; ++i) { + emitCode("if (" + OrigChains[i].first + ".getNode() != " + + OrigChains[i].second + ".getNode()) {"); + emitCode(" InChains.push_back(" + OrigChains[i].first + ");"); + emitCode("}"); } - - if (OrigChains.size() > 0) { - // The original input chain is being ignored. If it is not just - // pointing to the op that's being folded, we should create a - // TokenFactor with it and the chain of the folded op as the new chain. - // We could potentially be doing multiple levels of folding, in that - // case, the TokenFactor can have more operands. - emitCode("SmallVector InChains;"); - for (unsigned i = 0, e = OrigChains.size(); i < e; ++i) { - emitCode("if (" + OrigChains[i].first + ".getNode() != " + - OrigChains[i].second + ".getNode()) {"); - emitCode(" InChains.push_back(" + OrigChains[i].first + ");"); - emitCode("}"); - } - emitCode("InChains.push_back(" + ChainName + ");"); - emitCode(ChainName + " = CurDAG->getNode(ISD::TokenFactor, " - "N.getDebugLoc(), MVT::Other, " - "&InChains[0], InChains.size());"); - if (GenDebug) { - emitCode("CurDAG->setSubgraphColor(" + ChainName +".getNode(), \"yellow\");"); - emitCode("CurDAG->setSubgraphColor(" + ChainName +".getNode(), \"black\");"); - } + emitCode("InChains.push_back(" + ChainName + ");"); + emitCode(ChainName + " = CurDAG->getNode(ISD::TokenFactor, " + "N->getDebugLoc(), MVT::Other, " + "&InChains[0], InChains.size());"); + if (GenDebug) { + emitCode("CurDAG->setSubgraphColor(" + ChainName + + ".getNode(), \"yellow\");"); + emitCode("CurDAG->setSubgraphColor(" + ChainName + + ".getNode(), \"black\");"); } - - // Loop over all of the operands of the instruction pattern, emitting code - // to fill them all in. The node 'N' usually has number children equal to - // the number of input operands of the instruction. However, in cases - // where there are predicate operands for an instruction, we need to fill - // in the 'execute always' values. Match up the node operands to the - // instruction operands to do this. - std::vector AllOps; - for (unsigned ChildNo = 0, InstOpNo = NumResults; - InstOpNo != II.OperandList.size(); ++InstOpNo) { - std::vector Ops; - - // Determine what to emit for this operand. - Record *OperandNode = II.OperandList[InstOpNo].Rec; - if ((OperandNode->isSubClassOf("PredicateOperand") || - OperandNode->isSubClassOf("OptionalDefOperand")) && - !CGP.getDefaultOperand(OperandNode).DefaultOps.empty()) { - // This is a predicate or optional def operand; emit the - // 'default ops' operands. - const DAGDefaultOperand &DefaultOp = - CGP.getDefaultOperand(II.OperandList[InstOpNo].Rec); - for (unsigned i = 0, e = DefaultOp.DefaultOps.size(); i != e; ++i) { - Ops = EmitResultCode(DefaultOp.DefaultOps[i], DstRegs, - InFlagDecled, ResNodeDecled); - AllOps.insert(AllOps.end(), Ops.begin(), Ops.end()); - } - } else { - // Otherwise this is a normal operand or a predicate operand without - // 'execute always'; emit it. - Ops = EmitResultCode(N->getChild(ChildNo), DstRegs, + } + + // Loop over all of the operands of the instruction pattern, emitting code + // to fill them all in. The node 'N' usually has number children equal to + // the number of input operands of the instruction. However, in cases + // where there are predicate operands for an instruction, we need to fill + // in the 'execute always' values. Match up the node operands to the + // instruction operands to do this. + std::vector AllOps; + for (unsigned ChildNo = 0, InstOpNo = NumResults; + InstOpNo != II.OperandList.size(); ++InstOpNo) { + std::vector Ops; + + // Determine what to emit for this operand. + Record *OperandNode = II.OperandList[InstOpNo].Rec; + if ((OperandNode->isSubClassOf("PredicateOperand") || + OperandNode->isSubClassOf("OptionalDefOperand")) && + !CGP.getDefaultOperand(OperandNode).DefaultOps.empty()) { + // This is a predicate or optional def operand; emit the + // 'default ops' operands. + const DAGDefaultOperand &DefaultOp = + CGP.getDefaultOperand(II.OperandList[InstOpNo].Rec); + for (unsigned i = 0, e = DefaultOp.DefaultOps.size(); i != e; ++i) { + Ops = EmitResultCode(DefaultOp.DefaultOps[i], DstRegs, InFlagDecled, ResNodeDecled); AllOps.insert(AllOps.end(), Ops.begin(), Ops.end()); - ++ChildNo; - } - } - - // Emit all the chain and CopyToReg stuff. - bool ChainEmitted = NodeHasChain; - if (NodeHasInFlag || HasImpInputs) - EmitInFlagSelectCode(Pattern, "N", ChainEmitted, - InFlagDecled, ResNodeDecled, true); - if (NodeHasOptInFlag || NodeHasInFlag || HasImpInputs) { - if (!InFlagDecled) { - emitCode("SDValue InFlag(0, 0);"); - InFlagDecled = true; - } - if (NodeHasOptInFlag) { - emitCode("if (HasInFlag) {"); - emitCode(" InFlag = N.getOperand(N.getNumOperands()-1);"); - emitCode("}"); } - } - - unsigned ResNo = TmpNo++; - - unsigned OpsNo = OpcNo; - std::string CodePrefix; - bool ChainAssignmentNeeded = NodeHasChain && !isRoot; - std::deque After; - std::string NodeName; - if (!isRoot) { - NodeName = "Tmp" + utostr(ResNo); - CodePrefix = "SDValue " + NodeName + "("; } else { - NodeName = "ResNode"; - if (!ResNodeDecled) { - CodePrefix = "SDNode *" + NodeName + " = "; - ResNodeDecled = true; - } else - CodePrefix = NodeName + " = "; - } - - std::string Code = "Opc" + utostr(OpcNo); - - if (!isRoot || (InputHasChain && !NodeHasChain)) - // For call to "getMachineNode()". - Code += ", N.getDebugLoc()"; - - emitOpcode(II.Namespace + "::" + II.TheDef->getName()); - - // Output order: results, chain, flags - // Result types. - if (NumResults > 0 && N->getTypeNum(0) != MVT::isVoid) { - Code += ", VT" + utostr(VTNo); - emitVT(getEnumName(N->getTypeNum(0))); + // Otherwise this is a normal operand or a predicate operand without + // 'execute always'; emit it. + Ops = EmitResultCode(N->getChild(ChildNo), DstRegs, + InFlagDecled, ResNodeDecled); + AllOps.insert(AllOps.end(), Ops.begin(), Ops.end()); + ++ChildNo; } - // Add types for implicit results in physical registers, scheduler will - // care of adding copyfromreg nodes. - for (unsigned i = 0; i < NumDstRegs; i++) { - Record *RR = DstRegs[i]; - if (RR->isSubClassOf("Register")) { - MVT::SimpleValueType RVT = getRegisterValueType(RR, CGT); - Code += ", " + getEnumName(RVT); - } + } + + // Emit all the chain and CopyToReg stuff. + bool ChainEmitted = NodeHasChain; + if (NodeHasInFlag || HasImpInputs) + EmitInFlagSelectCode(Pattern, "N", ChainEmitted, + InFlagDecled, ResNodeDecled, true); + if (NodeHasOptInFlag || NodeHasInFlag || HasImpInputs) { + if (!InFlagDecled) { + emitCode("SDValue InFlag(0, 0);"); + InFlagDecled = true; } - if (NodeHasChain) - Code += ", MVT::Other"; - if (NodeHasOutFlag) - Code += ", MVT::Flag"; - - // Inputs. - if (IsVariadic) { - for (unsigned i = 0, e = AllOps.size(); i != e; ++i) - emitCode("Ops" + utostr(OpsNo) + ".push_back(" + AllOps[i] + ");"); - AllOps.clear(); - - // Figure out whether any operands at the end of the op list are not - // part of the variable section. - std::string EndAdjust; - if (NodeHasInFlag || HasImpInputs) - EndAdjust = "-1"; // Always has one flag. - else if (NodeHasOptInFlag) - EndAdjust = "-(HasInFlag?1:0)"; // May have a flag. - - emitCode("for (unsigned i = NumInputRootOps + " + utostr(NodeHasChain) + - ", e = N.getNumOperands()" + EndAdjust + "; i != e; ++i) {"); - - emitCode(" Ops" + utostr(OpsNo) + ".push_back(N.getOperand(i));"); + if (NodeHasOptInFlag) { + emitCode("if (HasInFlag) {"); + emitCode(" InFlag = N->getOperand(N->getNumOperands()-1);"); emitCode("}"); } - - // Populate MemRefs with entries for each memory accesses covered by - // this pattern. - if (isRoot && !LSI.empty()) { - std::string MemRefs = "MemRefs" + utostr(OpsNo); - emitCode("MachineSDNode::mmo_iterator " + MemRefs + " = " - "MF->allocateMemRefsArray(" + utostr(LSI.size()) + ");"); - for (unsigned i = 0, e = LSI.size(); i != e; ++i) - emitCode(MemRefs + "[" + utostr(i) + "] = " - "cast(" + LSI[i] + ")->getMemOperand();"); - After.push_back("cast(ResNode)->setMemRefs(" + - MemRefs + ", " + MemRefs + " + " + utostr(LSI.size()) + - ");"); + } + + unsigned ResNo = TmpNo++; + + unsigned OpsNo = OpcNo; + std::string CodePrefix; + bool ChainAssignmentNeeded = NodeHasChain && !isRoot; + std::deque After; + std::string NodeName; + if (!isRoot) { + NodeName = "Tmp" + utostr(ResNo); + CodePrefix = "SDValue " + NodeName + "("; + } else { + NodeName = "ResNode"; + if (!ResNodeDecled) { + CodePrefix = "SDNode *" + NodeName + " = "; + ResNodeDecled = true; + } else + CodePrefix = NodeName + " = "; + } + + std::string Code = "Opc" + utostr(OpcNo); + + if (!isRoot || (InputHasChain && !NodeHasChain)) + // For call to "getMachineNode()". + Code += ", N->getDebugLoc()"; + + emitOpcode(II.Namespace + "::" + II.TheDef->getName()); + + // Output order: results, chain, flags + // Result types. + if (NumResults > 0 && N->getTypeNum(0) != MVT::isVoid) { + Code += ", VT" + utostr(VTNo); + emitVT(getEnumName(N->getTypeNum(0))); + } + // Add types for implicit results in physical registers, scheduler will + // care of adding copyfromreg nodes. + for (unsigned i = 0; i < NumDstRegs; i++) { + Record *RR = DstRegs[i]; + if (RR->isSubClassOf("Register")) { + MVT::SimpleValueType RVT = getRegisterValueType(RR, CGT); + Code += ", " + getEnumName(RVT); } - - if (NodeHasChain) { - if (IsVariadic) - emitCode("Ops" + utostr(OpsNo) + ".push_back(" + ChainName + ");"); - else - AllOps.push_back(ChainName); + } + if (NodeHasChain) + Code += ", MVT::Other"; + if (NodeHasOutFlag) + Code += ", MVT::Flag"; + + // Inputs. + if (IsVariadic) { + for (unsigned i = 0, e = AllOps.size(); i != e; ++i) + emitCode("Ops" + utostr(OpsNo) + ".push_back(" + AllOps[i] + ");"); + AllOps.clear(); + + // Figure out whether any operands at the end of the op list are not + // part of the variable section. + std::string EndAdjust; + if (NodeHasInFlag || HasImpInputs) + EndAdjust = "-1"; // Always has one flag. + else if (NodeHasOptInFlag) + EndAdjust = "-(HasInFlag?1:0)"; // May have a flag. + + emitCode("for (unsigned i = NumInputRootOps + " + utostr(NodeHasChain) + + ", e = N->getNumOperands()" + EndAdjust + "; i != e; ++i) {"); + + emitCode(" Ops" + utostr(OpsNo) + ".push_back(N->getOperand(i));"); + emitCode("}"); + } + + // Populate MemRefs with entries for each memory accesses covered by + // this pattern. + if (isRoot && !LSI.empty()) { + std::string MemRefs = "MemRefs" + utostr(OpsNo); + emitCode("MachineSDNode::mmo_iterator " + MemRefs + " = " + "MF->allocateMemRefsArray(" + utostr(LSI.size()) + ");"); + for (unsigned i = 0, e = LSI.size(); i != e; ++i) + emitCode(MemRefs + "[" + utostr(i) + "] = " + "cast(" + LSI[i] + ")->getMemOperand();"); + After.push_back("cast(ResNode)->setMemRefs(" + + MemRefs + ", " + MemRefs + " + " + utostr(LSI.size()) + + ");"); + } + + if (NodeHasChain) { + if (IsVariadic) + emitCode("Ops" + utostr(OpsNo) + ".push_back(" + ChainName + ");"); + else + AllOps.push_back(ChainName); + } + + if (IsVariadic) { + if (NodeHasInFlag || HasImpInputs) + emitCode("Ops" + utostr(OpsNo) + ".push_back(InFlag);"); + else if (NodeHasOptInFlag) { + emitCode("if (HasInFlag)"); + emitCode(" Ops" + utostr(OpsNo) + ".push_back(InFlag);"); } - - if (IsVariadic) { - if (NodeHasInFlag || HasImpInputs) - emitCode("Ops" + utostr(OpsNo) + ".push_back(InFlag);"); - else if (NodeHasOptInFlag) { - emitCode("if (HasInFlag)"); - emitCode(" Ops" + utostr(OpsNo) + ".push_back(InFlag);"); - } - Code += ", &Ops" + utostr(OpsNo) + "[0], Ops" + utostr(OpsNo) + - ".size()"; - } else if (NodeHasInFlag || NodeHasOptInFlag || HasImpInputs) - AllOps.push_back("InFlag"); - - unsigned NumOps = AllOps.size(); - if (NumOps) { - if (!NodeHasOptInFlag && NumOps < 4) { - for (unsigned i = 0; i != NumOps; ++i) - Code += ", " + AllOps[i]; - } else { - std::string OpsCode = "SDValue Ops" + utostr(OpsNo) + "[] = { "; - for (unsigned i = 0; i != NumOps; ++i) { - OpsCode += AllOps[i]; - if (i != NumOps-1) - OpsCode += ", "; - } - emitCode(OpsCode + " };"); - Code += ", Ops" + utostr(OpsNo) + ", "; - if (NodeHasOptInFlag) { - Code += "HasInFlag ? "; - Code += utostr(NumOps) + " : " + utostr(NumOps-1); - } else - Code += utostr(NumOps); + Code += ", &Ops" + utostr(OpsNo) + "[0], Ops" + utostr(OpsNo) + + ".size()"; + } else if (NodeHasInFlag || NodeHasOptInFlag || HasImpInputs) + AllOps.push_back("InFlag"); + + unsigned NumOps = AllOps.size(); + if (NumOps) { + if (!NodeHasOptInFlag && NumOps < 4) { + for (unsigned i = 0; i != NumOps; ++i) + Code += ", " + AllOps[i]; + } else { + std::string OpsCode = "SDValue Ops" + utostr(OpsNo) + "[] = { "; + for (unsigned i = 0; i != NumOps; ++i) { + OpsCode += AllOps[i]; + if (i != NumOps-1) + OpsCode += ", "; } + emitCode(OpsCode + " };"); + Code += ", Ops" + utostr(OpsNo) + ", "; + if (NodeHasOptInFlag) { + Code += "HasInFlag ? "; + Code += utostr(NumOps) + " : " + utostr(NumOps-1); + } else + Code += utostr(NumOps); } - - if (!isRoot) - Code += "), 0"; - - std::vector ReplaceFroms; - std::vector ReplaceTos; - if (!isRoot) { - NodeOps.push_back("Tmp" + utostr(ResNo)); - } else { - + } + + if (!isRoot) + Code += "), 0"; + + std::vector ReplaceFroms; + std::vector ReplaceTos; + if (!isRoot) { + NodeOps.push_back("Tmp" + utostr(ResNo)); + } else { + if (NodeHasOutFlag) { if (!InFlagDecled) { After.push_back("SDValue InFlag(ResNode, " + @@ -1210,7 +1189,7 @@ public: utostr(NumResults+NumDstRegs+(unsigned)NodeHasChain) + ");"); } - + for (unsigned j = 0, e = FoldedChains.size(); j < e; j++) { ReplaceFroms.push_back("SDValue(" + FoldedChains[j].first + ".getNode(), " + @@ -1219,228 +1198,130 @@ public: ReplaceTos.push_back("SDValue(ResNode, " + utostr(NumResults+NumDstRegs) + ")"); } - + if (NodeHasOutFlag) { if (FoldedFlag.first != "") { ReplaceFroms.push_back("SDValue(" + FoldedFlag.first + ".getNode(), " + utostr(FoldedFlag.second) + ")"); ReplaceTos.push_back("InFlag"); } else { - assert(NodeHasProperty(Pattern, SDNPOutFlag, CGP)); - ReplaceFroms.push_back("SDValue(N.getNode(), " + + assert(Pattern->NodeHasProperty(SDNPOutFlag, CGP)); + ReplaceFroms.push_back("SDValue(N, " + utostr(NumPatResults + (unsigned)InputHasChain) + ")"); ReplaceTos.push_back("InFlag"); } } - + if (!ReplaceFroms.empty() && InputHasChain) { - ReplaceFroms.push_back("SDValue(N.getNode(), " + + ReplaceFroms.push_back("SDValue(N, " + utostr(NumPatResults) + ")"); ReplaceTos.push_back("SDValue(" + ChainName + ".getNode(), " + ChainName + ".getResNo()" + ")"); ChainAssignmentNeeded |= NodeHasChain; } - + // User does not expect the instruction would produce a chain! if ((!InputHasChain && NodeHasChain) && NodeHasOutFlag) { ; } else if (InputHasChain && !NodeHasChain) { // One of the inner node produces a chain. - if (NodeHasOutFlag) { - ReplaceFroms.push_back("SDValue(N.getNode(), " + - utostr(NumPatResults+1) + - ")"); - ReplaceTos.push_back("SDValue(ResNode, N.getResNo()-1)"); - } - ReplaceFroms.push_back("SDValue(N.getNode(), " + + assert(!NodeHasOutFlag && "Node has flag but not chain!"); + ReplaceFroms.push_back("SDValue(N, " + utostr(NumPatResults) + ")"); ReplaceTos.push_back(ChainName); } - } - - if (ChainAssignmentNeeded) { - // Remember which op produces the chain. - std::string ChainAssign; - if (!isRoot) - ChainAssign = ChainName + " = SDValue(" + NodeName + - ".getNode(), " + utostr(NumResults+NumDstRegs) + ");"; - else - ChainAssign = ChainName + " = SDValue(" + NodeName + - ", " + utostr(NumResults+NumDstRegs) + ");"; - - After.push_front(ChainAssign); - } - - if (ReplaceFroms.size() == 1) { - After.push_back("ReplaceUses(" + ReplaceFroms[0] + ", " + - ReplaceTos[0] + ");"); - } else if (!ReplaceFroms.empty()) { - After.push_back("const SDValue Froms[] = {"); - for (unsigned i = 0, e = ReplaceFroms.size(); i != e; ++i) - After.push_back(" " + ReplaceFroms[i] + (i + 1 != e ? "," : "")); - After.push_back("};"); - After.push_back("const SDValue Tos[] = {"); - for (unsigned i = 0, e = ReplaceFroms.size(); i != e; ++i) - After.push_back(" " + ReplaceTos[i] + (i + 1 != e ? "," : "")); - After.push_back("};"); - After.push_back("ReplaceUses(Froms, Tos, " + - itostr(ReplaceFroms.size()) + ");"); - } - - // We prefer to use SelectNodeTo since it avoids allocation when - // possible and it avoids CSE map recalculation for the node's - // users, however it's tricky to use in a non-root context. - // - // We also don't use if the pattern replacement is being used to - // jettison a chain result, since morphing the node in place - // would leave users of the chain dangling. - // - if (!isRoot || (InputHasChain && !NodeHasChain)) { - Code = "CurDAG->getMachineNode(" + Code; - } else { - Code = "CurDAG->SelectNodeTo(N.getNode(), " + Code; - } - if (isRoot) { - if (After.empty()) - CodePrefix = "return "; - else - After.push_back("return ResNode;"); - } - - emitCode(CodePrefix + Code + ");"); - - if (GenDebug) { - if (!isRoot) { - emitCode("CurDAG->setSubgraphColor(" + NodeName +".getNode(), \"yellow\");"); - emitCode("CurDAG->setSubgraphColor(" + NodeName +".getNode(), \"black\");"); - } - else { - emitCode("CurDAG->setSubgraphColor(" + NodeName +", \"yellow\");"); - emitCode("CurDAG->setSubgraphColor(" + NodeName +", \"black\");"); - } - } - - for (unsigned i = 0, e = After.size(); i != e; ++i) - emitCode(After[i]); - - return NodeOps; - } - if (Op->isSubClassOf("SDNodeXForm")) { - assert(N->getNumChildren() == 1 && "node xform should have one child!"); - // PatLeaf node - the operand may or may not be a leaf node. But it should - // behave like one. - std::vector Ops = - EmitResultCode(N->getChild(0), DstRegs, InFlagDecled, - ResNodeDecled, true); - unsigned ResNo = TmpNo++; - emitCode("SDValue Tmp" + utostr(ResNo) + " = Transform_" + Op->getName() - + "(" + Ops.back() + ".getNode());"); - NodeOps.push_back("Tmp" + utostr(ResNo)); - if (isRoot) - emitCode("return Tmp" + utostr(ResNo) + ".getNode();"); - return NodeOps; } - - N->dump(); - errs() << "\n"; - throw std::string("Unknown node in result pattern!"); - } - - /// InsertOneTypeCheck - Insert a type-check for an unresolved type in 'Pat' - /// and add it to the tree. 'Pat' and 'Other' are isomorphic trees except that - /// 'Pat' may be missing types. If we find an unresolved type to add a check - /// for, this returns true otherwise false if Pat has all types. - bool InsertOneTypeCheck(TreePatternNode *Pat, TreePatternNode *Other, - const std::string &Prefix, bool isRoot = false) { - // Did we find one? - if (Pat->getExtTypes() != Other->getExtTypes()) { - // Move a type over from 'other' to 'pat'. - Pat->setTypes(Other->getExtTypes()); - // The top level node type is checked outside of the select function. + + if (ChainAssignmentNeeded) { + // Remember which op produces the chain. + std::string ChainAssign; if (!isRoot) - emitCheck(Prefix + ".getNode()->getValueType(0) == " + - getName(Pat->getTypeNum(0))); - return true; + ChainAssign = ChainName + " = SDValue(" + NodeName + + ".getNode(), " + utostr(NumResults+NumDstRegs) + ");"; + else + ChainAssign = ChainName + " = SDValue(" + NodeName + + ", " + utostr(NumResults+NumDstRegs) + ");"; + + After.push_front(ChainAssign); } - - unsigned OpNo = - (unsigned) NodeHasProperty(Pat, SDNPHasChain, CGP); - for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i, ++OpNo) - if (InsertOneTypeCheck(Pat->getChild(i), Other->getChild(i), - Prefix + utostr(OpNo))) - return true; - return false; - } - -private: - /// EmitInFlagSelectCode - Emit the flag operands for the DAG that is - /// being built. - void EmitInFlagSelectCode(TreePatternNode *N, const std::string &RootName, - bool &ChainEmitted, bool &InFlagDecled, - bool &ResNodeDecled, bool isRoot = false) { - const CodeGenTarget &T = CGP.getTargetInfo(); - unsigned OpNo = - (unsigned) NodeHasProperty(N, SDNPHasChain, CGP); - bool HasInFlag = NodeHasProperty(N, SDNPInFlag, CGP); - for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i, ++OpNo) { - TreePatternNode *Child = N->getChild(i); - if (!Child->isLeaf()) { - EmitInFlagSelectCode(Child, RootName + utostr(OpNo), ChainEmitted, - InFlagDecled, ResNodeDecled); + + if (ReplaceFroms.size() == 1) { + After.push_back("ReplaceUses(" + ReplaceFroms[0] + ", " + + ReplaceTos[0] + ");"); + } else if (!ReplaceFroms.empty()) { + After.push_back("const SDValue Froms[] = {"); + for (unsigned i = 0, e = ReplaceFroms.size(); i != e; ++i) + After.push_back(" " + ReplaceFroms[i] + (i + 1 != e ? "," : "")); + After.push_back("};"); + After.push_back("const SDValue Tos[] = {"); + for (unsigned i = 0, e = ReplaceFroms.size(); i != e; ++i) + After.push_back(" " + ReplaceTos[i] + (i + 1 != e ? "," : "")); + After.push_back("};"); + After.push_back("ReplaceUses(Froms, Tos, " + + itostr(ReplaceFroms.size()) + ");"); + } + + // We prefer to use SelectNodeTo since it avoids allocation when + // possible and it avoids CSE map recalculation for the node's + // users, however it's tricky to use in a non-root context. + // + // We also don't use SelectNodeTo if the pattern replacement is being + // used to jettison a chain result, since morphing the node in place + // would leave users of the chain dangling. + // + if (!isRoot || (InputHasChain && !NodeHasChain)) { + Code = "CurDAG->getMachineNode(" + Code; + } else { + Code = "CurDAG->SelectNodeTo(N, " + Code; + } + if (isRoot) { + if (After.empty()) + CodePrefix = "return "; + else + After.push_back("return ResNode;"); + } + + emitCode(CodePrefix + Code + ");"); + + if (GenDebug) { + if (!isRoot) { + emitCode("CurDAG->setSubgraphColor(" + + NodeName +".getNode(), \"yellow\");"); + emitCode("CurDAG->setSubgraphColor(" + + NodeName +".getNode(), \"black\");"); } else { - if (DefInit *DI = dynamic_cast(Child->getLeafValue())) { - if (!Child->getName().empty()) { - std::string Name = RootName + utostr(OpNo); - if (Duplicates.find(Name) != Duplicates.end()) - // A duplicate! Do not emit a copy for this node. - continue; - } - - Record *RR = DI->getDef(); - if (RR->isSubClassOf("Register")) { - MVT::SimpleValueType RVT = getRegisterValueType(RR, T); - if (RVT == MVT::Flag) { - if (!InFlagDecled) { - emitCode("SDValue InFlag = " + RootName + utostr(OpNo) + ";"); - InFlagDecled = true; - } else - emitCode("InFlag = " + RootName + utostr(OpNo) + ";"); - } else { - if (!ChainEmitted) { - emitCode("SDValue Chain = CurDAG->getEntryNode();"); - ChainName = "Chain"; - ChainEmitted = true; - } - if (!InFlagDecled) { - emitCode("SDValue InFlag(0, 0);"); - InFlagDecled = true; - } - std::string Decl = (!ResNodeDecled) ? "SDNode *" : ""; - emitCode(Decl + "ResNode = CurDAG->getCopyToReg(" + ChainName + - ", " + RootName + ".getDebugLoc()" + - ", " + getQualifiedName(RR) + - ", " + RootName + utostr(OpNo) + ", InFlag).getNode();"); - ResNodeDecled = true; - emitCode(ChainName + " = SDValue(ResNode, 0);"); - emitCode("InFlag = SDValue(ResNode, 1);"); - } - } - } + emitCode("CurDAG->setSubgraphColor(" + NodeName +", \"yellow\");"); + emitCode("CurDAG->setSubgraphColor(" + NodeName +", \"black\");"); } } - - if (HasInFlag) { - if (!InFlagDecled) { - emitCode("SDValue InFlag = " + RootName + - ".getOperand(" + utostr(OpNo) + ");"); - InFlagDecled = true; - } else - emitCode("InFlag = " + RootName + - ".getOperand(" + utostr(OpNo) + ");"); - } + + for (unsigned i = 0, e = After.size(); i != e; ++i) + emitCode(After[i]); + + return NodeOps; } -}; + if (Op->isSubClassOf("SDNodeXForm")) { + assert(N->getNumChildren() == 1 && "node xform should have one child!"); + // PatLeaf node - the operand may or may not be a leaf node. But it should + // behave like one. + std::vector Ops = + EmitResultCode(N->getChild(0), DstRegs, InFlagDecled, + ResNodeDecled, true); + unsigned ResNo = TmpNo++; + emitCode("SDValue Tmp" + utostr(ResNo) + " = Transform_" + Op->getName() + + "(" + Ops.back() + ".getNode());"); + NodeOps.push_back("Tmp" + utostr(ResNo)); + if (isRoot) + emitCode("return Tmp" + utostr(ResNo) + ".getNode();"); + return NodeOps; + } + + N->dump(); + errs() << "\n"; + throw std::string("Unknown node in result pattern!"); +} + /// EmitCodeForPattern - Given a pattern to match, emit code to the specified /// stream to match the pattern, and generate the code for the match if it @@ -1465,7 +1346,8 @@ void DAGISelEmitter::GenerateCodeForPattern(const PatternToMatch &Pattern, bool FoundChain = false; Emitter.EmitMatchCode(Pattern.getSrcPattern(), NULL, "N", "", FoundChain); - // TP - Get *SOME* tree pattern, we don't care which. + // TP - Get *SOME* tree pattern, we don't care which. It is only used for + // diagnostics, which we know are impossible at this point. TreePattern &TP = *CGP.pf_begin()->second; // At this point, we know that we structurally match the pattern, but the @@ -1481,7 +1363,7 @@ void DAGISelEmitter::GenerateCodeForPattern(const PatternToMatch &Pattern, // types are resolved. // TreePatternNode *Pat = Pattern.getSrcPattern()->clone(); - RemoveAllTypes(Pat); + Pat->RemoveAllTypes(); do { // Resolve/propagate as many types as possible. @@ -1646,7 +1528,7 @@ static std::string getLegalCName(std::string OpName) { void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { const CodeGenTarget &Target = CGP.getTargetInfo(); - + // Get the namespace to insert instructions into. std::string InstNS = Target.getInstNamespace(); if (!InstNS.empty()) InstNS += "::"; @@ -1658,7 +1540,6 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), E = CGP.ptm_end(); I != E; ++I) { const PatternToMatch &Pattern = *I; - TreePatternNode *Node = Pattern.getSrcPattern(); if (!Node->isLeaf()) { PatternsByOpcode[getOpcodeName(Node->getOperator(), CGP)]. @@ -1668,7 +1549,7 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { if (dynamic_cast(Node->getLeafValue())) { PatternsByOpcode[getOpcodeName(CGP.getSDNodeNamed("imm"), CGP)]. push_back(&Pattern); - } else if ((CP = NodeGetComplexPattern(Node, CGP))) { + } else if ((CP = Node->getComplexPatternInfo(CGP))) { std::vector OpNodes = CP->getRootNodes(); for (unsigned j = 0, e = OpNodes.size(); j != e; j++) { PatternsByOpcode[getOpcodeName(OpNodes[j], CGP)] @@ -1763,7 +1644,7 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { AddedInits.push_back(GeneratedCode[j].second); } - std::string CalleeCode = "(const SDValue &N"; + std::string CalleeCode = "(SDNode *N"; std::string CallerCode = "(N"; for (unsigned j = 0, e = TargetOpcodes.size(); j != e; ++j) { CalleeCode += ", unsigned Opc" + utostr(j); @@ -1786,11 +1667,7 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { } CallerCode += ");"; - CalleeCode += ") "; - // Prevent emission routines from being inlined to reduce selection - // routines stack frame sizes. - CalleeCode += "DISABLE_INLINE "; - CalleeCode += "{\n"; + CalleeCode += ") {\n"; for (std::vector::const_reverse_iterator I = AddedInits.rbegin(), E = AddedInits.rend(); I != E; ++I) @@ -1811,22 +1688,26 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { } else { EmitFuncNum = EmitFunctions.size(); EmitFunctions.insert(std::make_pair(CalleeCode, EmitFuncNum)); + // Prevent emission routines from being inlined to reduce selection + // routines stack frame sizes. + OS << "DISABLE_INLINE "; OS << "SDNode *Emit_" << utostr(EmitFuncNum) << CalleeCode; } // Replace the emission code within selection routines with calls to the // emission functions. - if (GenDebug) { - GeneratedCode.push_back(std::make_pair(0, "CurDAG->setSubgraphColor(N.getNode(), \"red\");")); - } - CallerCode = "SDNode *Result = Emit_" + utostr(EmitFuncNum) + CallerCode; + if (GenDebug) + GeneratedCode.push_back(std::make_pair(0, + "CurDAG->setSubgraphColor(N, \"red\");")); + CallerCode = "SDNode *Result = Emit_" + utostr(EmitFuncNum) +CallerCode; GeneratedCode.push_back(std::make_pair(3, CallerCode)); if (GenDebug) { GeneratedCode.push_back(std::make_pair(0, "if(Result) {")); - GeneratedCode.push_back(std::make_pair(0, " CurDAG->setSubgraphColor(Result, \"yellow\");")); - GeneratedCode.push_back(std::make_pair(0, " CurDAG->setSubgraphColor(Result, \"black\");")); + GeneratedCode.push_back(std::make_pair(0, + " CurDAG->setSubgraphColor(Result, \"yellow\");")); + GeneratedCode.push_back(std::make_pair(0, + " CurDAG->setSubgraphColor(Result, \"black\");")); GeneratedCode.push_back(std::make_pair(0, "}")); - //GeneratedCode.push_back(std::make_pair(0, "CurDAG->setSubgraphColor(N.getNode(), \"black\");")); } GeneratedCode.push_back(std::make_pair(0, "return Result;")); } @@ -1895,7 +1776,7 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { std::reverse(CodeForPatterns.begin(), CodeForPatterns.end()); OS << "SDNode *Select_" << getLegalCName(OpName) - << OpVTStr << "(const SDValue &N) {\n"; + << OpVTStr << "(SDNode *N) {\n"; // Emit all of the patterns now, grouped together to share code. EmitPatterns(CodeForPatterns, 2, OS); @@ -1904,59 +1785,19 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { // catch the case where nothing handles a pattern. if (mightNotMatch) { OS << "\n"; - if (OpName != "ISD::INTRINSIC_W_CHAIN" && - OpName != "ISD::INTRINSIC_WO_CHAIN" && - OpName != "ISD::INTRINSIC_VOID") - OS << " CannotYetSelect(N);\n"; - else - OS << " CannotYetSelectIntrinsic(N);\n"; - + OS << " CannotYetSelect(N);\n"; OS << " return NULL;\n"; } OS << "}\n\n"; } } - // Emit boilerplate. - OS << "SDNode *Select_INLINEASM(SDValue N) {\n" - << " std::vector Ops(N.getNode()->op_begin(), N.getNode()->op_end());\n" - << " SelectInlineAsmMemoryOperands(Ops);\n\n" - - << " std::vector VTs;\n" - << " VTs.push_back(MVT::Other);\n" - << " VTs.push_back(MVT::Flag);\n" - << " SDValue New = CurDAG->getNode(ISD::INLINEASM, N.getDebugLoc(), " - "VTs, &Ops[0], Ops.size());\n" - << " return New.getNode();\n" - << "}\n\n"; - - OS << "SDNode *Select_UNDEF(const SDValue &N) {\n" - << " return CurDAG->SelectNodeTo(N.getNode(), TargetInstrInfo::IMPLICIT_DEF,\n" - << " N.getValueType());\n" - << "}\n\n"; - - OS << "SDNode *Select_DBG_LABEL(const SDValue &N) {\n" - << " SDValue Chain = N.getOperand(0);\n" - << " unsigned C = cast(N)->getLabelID();\n" - << " SDValue Tmp = CurDAG->getTargetConstant(C, MVT::i32);\n" - << " return CurDAG->SelectNodeTo(N.getNode(), TargetInstrInfo::DBG_LABEL,\n" - << " MVT::Other, Tmp, Chain);\n" - << "}\n\n"; - - OS << "SDNode *Select_EH_LABEL(const SDValue &N) {\n" - << " SDValue Chain = N.getOperand(0);\n" - << " unsigned C = cast(N)->getLabelID();\n" - << " SDValue Tmp = CurDAG->getTargetConstant(C, MVT::i32);\n" - << " return CurDAG->SelectNodeTo(N.getNode(), TargetInstrInfo::EH_LABEL,\n" - << " MVT::Other, Tmp, Chain);\n" - << "}\n\n"; - OS << "// The main instruction selector code.\n" - << "SDNode *SelectCode(SDValue N) {\n" - << " MVT::SimpleValueType NVT = N.getNode()->getValueType(0).getSimpleVT().SimpleTy;\n" - << " switch (N.getOpcode()) {\n" + << "SDNode *SelectCode(SDNode *N) {\n" + << " MVT::SimpleValueType NVT = N->getValueType(0).getSimpleVT().SimpleTy;\n" + << " switch (N->getOpcode()) {\n" << " default:\n" - << " assert(!N.isMachineOpcode() && \"Node already selected!\");\n" + << " assert(!N->isMachineOpcode() && \"Node already selected!\");\n" << " break;\n" << " case ISD::EntryToken: // These nodes remain the same.\n" << " case ISD::BasicBlock:\n" @@ -1967,6 +1808,7 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { << " case ISD::TargetConstantPool:\n" << " case ISD::TargetFrameIndex:\n" << " case ISD::TargetExternalSymbol:\n" + << " case ISD::TargetBlockAddress:\n" << " case ISD::TargetJumpTable:\n" << " case ISD::TargetGlobalTLSAddress:\n" << " case ISD::TargetGlobalAddress:\n" @@ -1977,11 +1819,10 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { << " }\n" << " case ISD::AssertSext:\n" << " case ISD::AssertZext: {\n" - << " ReplaceUses(N, N.getOperand(0));\n" + << " ReplaceUses(SDValue(N, 0), N->getOperand(0));\n" << " return NULL;\n" << " }\n" << " case ISD::INLINEASM: return Select_INLINEASM(N);\n" - << " case ISD::DBG_LABEL: return Select_DBG_LABEL(N);\n" << " case ISD::EH_LABEL: return Select_EH_LABEL(N);\n" << " case ISD::UNDEF: return Select_UNDEF(N);\n"; @@ -2045,41 +1886,41 @@ void DAGISelEmitter::EmitInstructionSelector(raw_ostream &OS) { } OS << " } // end of big switch.\n\n" - << " if (N.getOpcode() != ISD::INTRINSIC_W_CHAIN &&\n" - << " N.getOpcode() != ISD::INTRINSIC_WO_CHAIN &&\n" - << " N.getOpcode() != ISD::INTRINSIC_VOID) {\n" - << " CannotYetSelect(N);\n" - << " } else {\n" - << " CannotYetSelectIntrinsic(N);\n" - << " }\n" + << " CannotYetSelect(N);\n" << " return NULL;\n" << "}\n\n"; +} - OS << "void CannotYetSelect(SDValue N) DISABLE_INLINE {\n" - << " std::string msg;\n" - << " raw_string_ostream Msg(msg);\n" - << " Msg << \"Cannot yet select: \";\n" - << " N.getNode()->print(Msg, CurDAG);\n" - << " llvm_report_error(Msg.str());\n" - << "}\n\n"; - - OS << "void CannotYetSelectIntrinsic(SDValue N) DISABLE_INLINE {\n" - << " errs() << \"Cannot yet select: \";\n" - << " unsigned iid = cast(N.getOperand(" - << "N.getOperand(0).getValueType() == MVT::Other))->getZExtValue();\n" - << " if (iid < Intrinsic::num_intrinsics)\n" - << " llvm_report_error(\"Cannot yet select: intrinsic %\" + " - << "Intrinsic::getName((Intrinsic::ID)iid));\n"; - if (CGP.hasTargetIntrinsics()) { - OS << " else if (const TargetIntrinsicInfo *tii = TM.getIntrinsicInfo())\n" - << " llvm_report_error(Twine(\"Cannot yet select: target intrinsic " - << "%\") + tii->getName(iid));\n"; +namespace { +// PatternSortingPredicate - return true if we prefer to match LHS before RHS. +// In particular, we want to match maximal patterns first and lowest cost within +// a particular complexity first. +struct PatternSortingPredicate2 { + PatternSortingPredicate2(CodeGenDAGPatterns &cgp) : CGP(cgp) {} + CodeGenDAGPatterns &CGP; + + bool operator()(const PatternToMatch *LHS, + const PatternToMatch *RHS) { + unsigned LHSSize = getPatternSize(LHS->getSrcPattern(), CGP); + unsigned RHSSize = getPatternSize(RHS->getSrcPattern(), CGP); + LHSSize += LHS->getAddedComplexity(); + RHSSize += RHS->getAddedComplexity(); + if (LHSSize > RHSSize) return true; // LHS -> bigger -> less cost + if (LHSSize < RHSSize) return false; + + // If the patterns have equal complexity, compare generated instruction cost + unsigned LHSCost = getResultPatternCost(LHS->getDstPattern(), CGP); + unsigned RHSCost = getResultPatternCost(RHS->getDstPattern(), CGP); + if (LHSCost < RHSCost) return true; + if (LHSCost > RHSCost) return false; + + return getResultPatternSize(LHS->getDstPattern(), CGP) < + getResultPatternSize(RHS->getDstPattern(), CGP); } - OS << " else\n" - << " llvm_report_error(\"Cannot yet select: invalid intrinsic\");\n" - << "}\n\n"; +}; } + void DAGISelEmitter::run(raw_ostream &OS) { EmitSourceFileHeader("DAG Instruction Selector for the " + CGP.getTargetInfo().getName() + " target", OS); @@ -2103,9 +1944,37 @@ void DAGISelEmitter::run(raw_ostream &OS) { DEBUG(errs() << "\n"); } +#ifdef ENABLE_NEW_ISEL + // Add all the patterns to a temporary list so we can sort them. + std::vector Patterns; + for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), E = CGP.ptm_end(); + I != E; ++I) + Patterns.push_back(&*I); + + // We want to process the matches in order of minimal cost. Sort the patterns + // so the least cost one is at the start. + // FIXME: Eliminate "PatternSortingPredicate" and rename. + std::stable_sort(Patterns.begin(), Patterns.end(), + PatternSortingPredicate2(CGP)); + + + // Convert each pattern into Matcher's. + std::vector PatternMatchers; + for (unsigned i = 0, e = Patterns.size(); i != e; ++i) + PatternMatchers.push_back(ConvertPatternToMatcher(*Patterns[i], CGP)); + + Matcher *TheMatcher = new ScopeMatcher(&PatternMatchers[0], + PatternMatchers.size()); + + TheMatcher = OptimizeMatcher(TheMatcher); + //Matcher->dump(); + EmitMatcherTable(TheMatcher, OS); + delete TheMatcher; + +#else // At this point, we have full information about the 'Patterns' we need to // parse, both implicitly from instructions as well as from explicit pattern // definitions. Emit the resultant instruction selector. EmitInstructionSelector(OS); - +#endif }