Daniel's patchlinaro-local/jenkins/diana/precommit-patched

4 files changed, 239 insertions, 33 deletions

diff --git a/lib/Target/AArch64/AArch64InstructionSelector.cpp b/lib/Target/AArch64/AArch64InstructionSelector.cpp
index 9bfd570e9a8..ec3d93b75bd 100644
--- a/lib/Target/AArch64/AArch64InstructionSelector.cpp
+++ b/lib/Target/AArch64/AArch64InstructionSelector.cpp
@@ -56,6 +56,11 @@ public:
 private:
   /// tblgen-erated 'select' implementation, used as the initial selector for
   /// the patterns that don't require complex C++.
+  bool selectImpl5(MachineInstr &I) const;
+  bool selectImpl4(MachineInstr &I) const;
+  bool selectImpl3(MachineInstr &I) const;
+  bool selectImpl2(MachineInstr &I) const;
+  bool selectImpl1(MachineInstr &I) const;
   bool selectImpl(MachineInstr &I) const;
 
   bool selectVaStartAAPCS(MachineInstr &I, MachineFunction &MF,
diff --git a/lib/Target/X86/X86InstructionSelector.cpp b/lib/Target/X86/X86InstructionSelector.cpp
index 3457d35b7af..834eca74be9 100644
--- a/lib/Target/X86/X86InstructionSelector.cpp
+++ b/lib/Target/X86/X86InstructionSelector.cpp
@@ -53,6 +53,11 @@ public:
 private:
   /// tblgen-erated 'select' implementation, used as the initial selector for
   /// the patterns that don't require complex C++.
+  bool selectImpl5(MachineInstr &I) const;
+  bool selectImpl4(MachineInstr &I) const;
+  bool selectImpl3(MachineInstr &I) const;
+  bool selectImpl2(MachineInstr &I) const;
+  bool selectImpl1(MachineInstr &I) const;
   bool selectImpl(MachineInstr &I) const;
 
   // TODO: remove after selectImpl support pattern with a predicate.
diff --git a/test/TableGen/GlobalISelEmitter.td b/test/TableGen/GlobalISelEmitter.td
index 350331804b9..b91df9a1453 100644
--- a/test/TableGen/GlobalISelEmitter.td
+++ b/test/TableGen/GlobalISelEmitter.td
@@ -7,6 +7,10 @@ include "llvm/Target/Target.td"
 def MyTargetISA : InstrInfo;
 def MyTarget : Target { let InstructionSet = MyTargetISA; }
 
+let TargetPrefix = "mytarget" in {
+def int_mytarget_nop : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
+}
+
 def R0 : Register<"r0"> { let Namespace = "MyTarget"; }
 def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>;
 def GPR32Op : RegisterOperand<GPR32>;
@@ -134,6 +138,37 @@ def : Pat<(select GPR32:$src1, complex:$src2, complex:$src3),
 def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2),
             [(set GPR32:$dst, (add GPR32:$src1, GPR32:$src2))]>;
 
+//===- Test a simple pattern with an intrinsic. ---------------------------===//
+//
+
+// CHECK-LABEL: if ([&]() {
+// CHECK-NEXT:    MachineInstr &MI0 = I;
+// CHECK-NEXT:    if (MI0.getNumOperands() < 3)
+// CHECK-NEXT:      return false;
+// CHECK-NEXT:    if ((MI0.getOpcode() == TargetOpcode::G_INTRINSIC) &&
+// CHECK-NEXT:        ((/* dst */ (MRI.getType(MI0.getOperand(0).getReg()) == (LLT::scalar(32))) &&
+// CHECK-NEXT:         ((&RBI.getRegBankFromRegClass(MyTarget::GPR32RegClass) == RBI.getRegBank(MI0.getOperand(0).getReg(), MRI, TRI))))) &&
+// CHECK-NEXT:        ((/* Operand 1 */ (isOperandImmEqual(MI0.getOperand(1), [[ID:[0-9]+]], MRI)))) &&
+// CHECK-NEXT:        ((/* src1 */ (MRI.getType(MI0.getOperand(2).getReg()) == (LLT::scalar(32))) &&
+// CHECK-NEXT:         ((&RBI.getRegBankFromRegClass(MyTarget::GPR32RegClass) == RBI.getRegBank(MI0.getOperand(2).getReg(), MRI, TRI)))))) {
+// CHECK-NEXT:      // (intrinsic_wo_chain:i32 [[ID]]:iPTR, GPR32:i32:$src1) => (MOV:i32 GPR32:i32:$src1)
+// CHECK-NEXT:      MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(MyTarget::MOV));
+// CHECK-NEXT:      MIB.add(MI0.getOperand(0)/*dst*/);
+// CHECK-NEXT:      MIB.add(MI0.getOperand(2)/*src1*/);
+// CHECK-NEXT:      for (const auto *FromMI : {&MI0, })
+// CHECK-NEXT:        for (const auto &MMO : FromMI->memoperands())
+// CHECK-NEXT:          MIB.addMemOperand(MMO);
+// CHECK-NEXT:      I.eraseFromParent();
+// CHECK-NEXT:      MachineInstr &NewI = *MIB;
+// CHECK-NEXT:      constrainSelectedInstRegOperands(NewI, TII, TRI, RBI);
+// CHECK-NEXT:      return true;
+// CHECK-NEXT:    }
+// CHECK-NEXT:    return false;
+// CHECK-NEXT:  }()) { return true; }
+
+def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1),
+            [(set GPR32:$dst, (int_mytarget_nop GPR32:$src1))]>;
+
 //===- Test a nested instruction match. -----------------------------------===//
 
 // CHECK-LABEL: if ([&]() {
diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp
index f0b7c436fb5..1ec2077aef7 100644
--- a/utils/TableGen/GlobalISelEmitter.cpp
+++ b/utils/TableGen/GlobalISelEmitter.cpp
@@ -610,6 +610,8 @@ public:
 
     return false;
   };
+
+  const CodeGenInstruction *getInstruction() const { return I; }
 };
 
 /// Generates code to check that a set of predicates and operands match for a
@@ -627,7 +629,11 @@ protected:
   /// condition is always true.
   OperandVec Operands;
 
+  bool OmitNumOperandsCheck;
+
 public:
+  InstructionMatcher() : OmitNumOperandsCheck(false) {}
+
   /// Add an operand to the matcher.
   OperandMatcher &addOperand(unsigned OpIdx, const std::string &SymbolicName,
                              unsigned AllocatedTemporariesBaseID) {
@@ -676,11 +682,15 @@ public:
     return make_range(operands_begin(), operands_end());
   }
 
+  void omitNumOperandsCheck() { OmitNumOperandsCheck = true; }
+
   /// Emit C++ statements to check the shape of the match and capture
   /// instructions into local variables.
   void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, StringRef Expr) {
-    OS << "if (" << Expr << ".getNumOperands() < " << getNumOperands() << ")\n"
-       << "  return false;\n";
+    if (!OmitNumOperandsCheck)
+      OS << "if (" << Expr << ".getNumOperands() < " << getNumOperands()
+         << ")\n"
+         << "  goto label" << NumPatternEmitted << ";\n";
     for (const auto &Operand : Operands) {
       Operand->emitCxxCaptureStmts(OS, Rule, Operand->getOperandExpr(Expr));
     }
@@ -740,6 +750,16 @@ public:
                  return A + Operand->countRendererFns();
                });
   }
+
+#if 0
+  Optional<InstructionOpcodeMatcher *> getInstructionOpcodeMatcher() const {
+    for (const auto &Predicate : predicates())
+      if (const InstructionOpcodeMatcher *OpcodeMatcher =
+              dyn_cast<InstructionOpcodeMatcher>(Predicate.get()))
+        return OpcodeMatcher;
+    return None;
+  }
+#endif
 };
 
 /// Generates code to check that the operand is a register defined by an
@@ -774,8 +794,9 @@ public:
 
   void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule,
                            StringRef OperandExpr) const override {
-    OS << "if (!" << OperandExpr + ".isReg())\n"
-       << "  return false;\n";
+    OS << "if (!" << OperandExpr + ".isReg() || TRI.isPhysicalRegister("
+       << OperandExpr + ".getReg()))\n"
+       << "  goto label" << NumPatternEmitted << ";\n";
     std::string InsnVarName = Rule.defineInsnVar(
         OS, *InsnMatcher,
         ("*MRI.getVRegDef(" + OperandExpr + ".getReg())").str());
@@ -1088,8 +1109,7 @@ void RuleMatcher::emit(raw_ostream &OS,
   // on some targets but we don't need to make use of that yet.
   assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet");
 
-  OS << "if (";
-  OS << "[&]() {\n";
+  OS << "{\n";
   if (!RequiredFeatures.empty()) {
     OS << "  PredicateBitset ExpectedFeatures = {";
     StringRef Separator = "";
@@ -1101,7 +1121,7 @@ void RuleMatcher::emit(raw_ostream &OS,
     }
     OS << "};\n";
     OS << "if ((AvailableFeatures & ExpectedFeatures) != ExpectedFeatures)\n"
-       << "  return false;\n";
+       << "  goto label" << NumPatternEmitted << ";\n";
   }
 
   emitCxxCaptureStmts(OS, "I");
@@ -1121,7 +1141,7 @@ void RuleMatcher::emit(raw_ostream &OS,
 
       // Reject the difficult cases until we have a more accurate check.
       OS << "      if (!isObviouslySafeToFold(" << Pair.second
-         << ")) return false;\n";
+         << ")) goto label" << NumPatternEmitted << ";\n";
 
       // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
       //        account for unsafe cases.
@@ -1167,8 +1187,9 @@ void RuleMatcher::emit(raw_ostream &OS,
   OS << "      constrainSelectedInstRegOperands(NewI, TII, TRI, RBI);\n";
   OS << "      return true;\n";
   OS << "    }\n";
-  OS << "    return false;\n";
-  OS << "  }()) { return true; }\n\n";
+  OS << "    goto label" << NumPatternEmitted << ";\n";
+  OS << "  }\n"
+     << "label" << NumPatternEmitted++ << ":\n";
 }
 
 bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
@@ -1196,6 +1217,133 @@ unsigned RuleMatcher::countRendererFns() const {
       });
 }
 
+class PartitionByNumOperands {
+public:
+  typedef unsigned hash_type;
+
+  static hash_type hash(RuleMatcher &Rule) {
+    return Rule.insnmatcher_front().getNumOperands();
+  }
+
+  static void emitPartitionSelector(raw_ostream &OS, hash_type &Hash) {
+    OS << "I.getNumOperands() == " << Hash;
+  }
+
+  static void updateRule(RuleMatcher &Rule) {
+    Rule.insnmatcher_front().omitNumOperandsCheck();
+  }
+};
+
+#if 0
+class PartitionByOpcode {
+public:
+  typedef StringRef hash_type;
+
+  static hash_type hash(RuleMatcher &Rule) {
+    Optional<const CodeGenInstruction *> I = Rule.insnmatcher_front().getInstructionOpcodeMatcher();
+    if (!I.hasValue())
+      return "";
+    return I.getValue()->TheDef->getName();
+  }
+
+  static void emitPartitionSelector(raw_ostream &OS, hash_type &Hash) {
+    if (Hash.empty()) {
+      OS << "true";
+      return;
+    }
+    OS << "I.getOpcode() == AArch64::" << Hash;
+  }
+
+  static void updateRule(RuleMatcher &Rule) {
+    Rule.insnmatcher_front().removeOpcodeCheck();
+  }
+};
+#endif
+
+/// Emit code to match a set of rules, obeying any prioritization they may need.
+template <class Partitioner>
+class MultiPartitionedRuleSetMatcher {
+  typedef typename Partitioner::hash_type hash_type;
+  typedef std::pair<hash_type, RuleMatcher> value_type;
+  std::vector<value_type> Rules;
+  std::string TgtName;
+
+public:
+  MultiPartitionedRuleSetMatcher(const StringRef TgtName) : TgtName(TgtName) {}
+
+  void push_back(RuleMatcher &&Rule) {
+    Rules.emplace_back(Partitioner::hash(Rule), std::move(Rule));
+    Partitioner::updateRule(Rules.back().second);
+  }
+
+  unsigned countRendererFns() const {
+    unsigned MaxTemporaries = 0;
+    for (const auto &Pair : Rules)
+      MaxTemporaries = std::max(MaxTemporaries, Pair.second.countRendererFns());
+    return MaxTemporaries;
+  }
+
+  void emitRules(raw_ostream &OS, SubtargetFeatureInfoMap SubtargetFeatures,
+                 typename std::vector<value_type>::iterator Begin,
+                 typename std::vector<value_type>::iterator End) {
+    for (auto I = Begin; I != End; ++I) {
+      I->second.emit(OS, SubtargetFeatures);
+      ++NumPatternEmitted;
+    }
+  }
+
+  void emitFunctions(raw_ostream &OS,
+                     SubtargetFeatureInfoMap SubtargetFeatures) {
+    std::stable_sort(Rules.begin(), Rules.end(), [&](const value_type &A,
+                                                     const value_type &B) {
+      if (A.first < B.first)
+        return true;
+      if (A.first > B.first)
+        return false;
+
+      if (A.second.isHigherPriorityThan(B.second)) {
+        assert(!B.second.isHigherPriorityThan(A.second) &&
+               "Cannot be more important and less important at the same time");
+        return true;
+      }
+      return false;
+    });
+
+    auto EmitFunctionHeader = [&](raw_ostream &OS,
+                                 typename Partitioner::hash_type I) {
+      OS << "bool " << TgtName << "InstructionSelector::selectImpl" << I
+         << "(MachineInstr &I) const {\n"
+         << "  MachineFunction &MF = *I.getParent()->getParent();\n"
+         << "  const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
+         << "  // FIXME: This should be computed on a per-function basis "
+            "rather than per-insn.\n"
+         << "  AvailableFunctionFeatures = "
+            "computeAvailableFunctionFeatures(&STI, &MF);\n"
+         << "  const PredicateBitset AvailableFeatures = "
+            "getAvailableFeatures();\n";
+    };
+
+    auto L = Rules.begin();
+    EmitFunctionHeader(OS, L->first);
+    for (auto I = Rules.begin(), E = Rules.end();
+         I != E; ++I) {
+      if (L->first != I->first) {
+        emitRules(OS, SubtargetFeatures, L, I);
+        OS << "  return false;\n"
+           << "}\n";
+
+        EmitFunctionHeader(OS, I->first);
+
+        L = I;
+      }
+    }
+
+    emitRules(OS, SubtargetFeatures, L, Rules.end());
+    OS << "  return false;\n"
+       << "}\n";
+  }
+};
+
 //===- GlobalISelEmitter class --------------------------------------------===//
 
 class GlobalISelEmitter {
@@ -1315,8 +1463,27 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
 
   // Match the used operands (i.e. the children of the operator).
   for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) {
-    if (auto Error = importChildMatcher(InsnMatcher, Src->getChild(i), OpIdx++,
-                                        TempOpIdx))
+    TreePatternNode *SrcChild = Src->getChild(i);
+
+    // For G_INTRINSIC, the operand immediately following the defs is an
+    // intrinsic ID.
+    if (SrcGI.TheDef->getName() == "G_INTRINSIC" && i == 0) {
+      if (!SrcChild->isLeaf())
+        return failedImport("Expected IntInit containing intrinsic ID");
+
+      if (IntInit *SrcChildIntInit =
+              dyn_cast<IntInit>(SrcChild->getLeafValue())) {
+        OperandMatcher &OM =
+            InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx);
+        OM.addPredicate<IntOperandMatcher>(SrcChildIntInit->getValue());
+        continue;
+      }
+
+      return failedImport("Expected IntInit containing instrinsic ID)");
+    }
+
+    if (auto Error =
+            importChildMatcher(InsnMatcher, SrcChild, OpIdx++, TempOpIdx))
       return std::move(Error);
   }
 
@@ -1351,7 +1518,7 @@ Error GlobalISelEmitter::importChildMatcher(InstructionMatcher &InsnMatcher,
 
   auto OpTyOrNone = MVTToLLT(ChildTypes.front().getConcrete());
   if (!OpTyOrNone)
-    return failedImport("Src operand has an unsupported type");
+    return failedImport("Src operand has an unsupported type (" + to_string(*SrcChild) + ")");
   OM.addPredicate<LLTOperandMatcher>(*OpTyOrNone);
 
   // Check for nested instructions.
@@ -1655,7 +1822,8 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
 
   emitSourceFileHeader(("Global Instruction Selector for the " +
                        Target.getName() + " target").str(), OS);
-  std::vector<RuleMatcher> Rules;
+  MultiPartitionedRuleSetMatcher<PartitionByNumOperands> RuleSet(
+      Target.getName());
   // Look through the SelectionDAG patterns we found, possibly emitting some.
   for (const PatternToMatch &Pat : CGP.ptms()) {
     ++NumPatternTotal;
@@ -1674,23 +1842,10 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
       continue;
     }
 
-    Rules.push_back(std::move(MatcherOrErr.get()));
+    RuleSet.push_back(std::move(MatcherOrErr.get()));
   }
 
-  std::stable_sort(Rules.begin(), Rules.end(),
-            [&](const RuleMatcher &A, const RuleMatcher &B) {
-              if (A.isHigherPriorityThan(B)) {
-                assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
-                                                     "and less important at "
-                                                     "the same time");
-                return true;
-              }
-              return false;
-            });
-
-  unsigned MaxTemporaries = 0;
-  for (const auto &Rule : Rules)
-    MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());
+  unsigned MaxTemporaries = RuleSet.countRendererFns();
 
   OS << "#ifdef GET_GLOBALISEL_PREDICATE_BITSET\n"
      << "const unsigned MAX_SUBTARGET_PREDICATES = " << SubtargetFeatures.size()
@@ -1736,6 +1891,8 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
       "computeAvailableFunctionFeatures", FunctionFeatures, OS,
       "const MachineFunction *MF");
 
+  RuleSet.emitFunctions(OS, SubtargetFeatures);
+
   OS << "bool " << Target.getName()
      << "InstructionSelector::selectImpl(MachineInstr &I) const {\n"
      << "  MachineFunction &MF = *I.getParent()->getParent();\n"
@@ -1744,10 +1901,14 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
      << "  AvailableFunctionFeatures = computeAvailableFunctionFeatures(&STI, &MF);\n"
      << "  const PredicateBitset AvailableFeatures = getAvailableFeatures();\n";
 
-  for (auto &Rule : Rules) {
-    Rule.emit(OS, SubtargetFeatures);
-    ++NumPatternEmitted;
-  }
+  OS << "  switch (I.getNumOperands()) {\n"
+     << "  case 1: return selectImpl1(I);\n"
+     << "  case 2: return selectImpl2(I);\n"
+     << "  case 3: return selectImpl3(I);\n"
+     << "  case 4: return selectImpl4(I);\n"
+     << "  case 5: return selectImpl5(I);\n"
+     << "  default: llvm_unreachable(\"\");\n"
+     << "  }\n";
 
   OS << "  return false;\n"
      << "}\n"