Switch lowering: Take branch weight into account when ordering for fall-through

Previously, the code would try to put a fall-through case last,
even if that meant moving a case with much higher branch weight
further down the chain.

Ordering by branch weight is most important, putting a fall-through
block last is secondary.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@235942 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d7edd2c19665550ff27ab09cf2caa3c575f7512e..985a4dfd49d76adc0412b6df6ebf07538e4ebfcf 100644 (file)
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7678,11 +7678,12 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
       return a.Weight > b.Weight;
     });
 
-    // Rearrange the case blocks so that the last one falls through if possible.
-    // Start at the bottom as that's the case with the lowest weight.
-    // FIXME: Take branch probability into account.
+    // Rearrange the case blocks so that the last one falls through if possible
+    // without without changing the order of weights.
     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
       --I;
+      if (I->Weight > W.LastCluster->Weight)
+        break;
       if (I->Kind == CC_Range && I->MBB == NextMBB) {
         std::swap(*I, *W.LastCluster);
         break;

diff --git a/test/CodeGen/ARM/arm-and-tst-peephole.ll b/test/CodeGen/ARM/arm-and-tst-peephole.ll
index 7d7fae95bfc9878c078873e13f4ff1b3e38d93cb..3096587ccb06db528772e553ccc5bd4c901cd4db 100644 (file)
--- a/test/CodeGen/ARM/arm-and-tst-peephole.ll
+++ b/test/CodeGen/ARM/arm-and-tst-peephole.ll
@@ -49,14 +49,14 @@ tailrecurse.switch:                               ; preds = %tailrecurse
 ; V8-NEXT: beq
 ; V8-NEXT: %tailrecurse.switch
 ; V8: cmp
-; V8-NEXT: beq
+; V8-NEXT: bne
 ; V8-NEXT: b   
 ; The trailing space in the last line checks that the branch is unconditional
   switch i32 %and, label %sw.epilog [
     i32 1, label %sw.bb
     i32 3, label %sw.bb6
     i32 2, label %sw.bb8
-  ]
+  ], !prof !1
 
 sw.bb:                                            ; preds = %tailrecurse.switch, %tailrecurse
   %shl = shl i32 %acc.tr, 1
@@ -134,3 +134,5 @@ bb4:                                              ; preds = %bb2
 return:                                           ; preds = %bb2, %bb, %entry
   ret i8 1
 }
+
+!1 = !{!"branch_weights", i32 1, i32 1, i32 3, i32 2 }

diff --git a/test/CodeGen/Generic/MachineBranchProb.ll b/test/CodeGen/Generic/MachineBranchProb.ll
index f10bd395abe226f5d799a8b404ab065569ba3561..83277c98989d0d12b366ec05d53e9605cb0da97f 100644 (file)
--- a/test/CodeGen/Generic/MachineBranchProb.ll
+++ b/test/CodeGen/Generic/MachineBranchProb.ll
@@ -17,9 +17,9 @@ entry:
 ; CHECK: BB#0: derived from LLVM BB %entry
 ; CHECK: Successors according to CFG: BB#2(64) BB#4(14)
 ; CHECK: BB#4: derived from LLVM BB %entry
-; CHECK: Successors according to CFG: BB#1(4) BB#5(10)
+; CHECK: Successors according to CFG: BB#1(10) BB#5(4)
 ; CHECK: BB#5: derived from LLVM BB %entry
-; CHECK: Successors according to CFG: BB#1(10) BB#3(7)
+; CHECK: Successors according to CFG: BB#1(4) BB#3(7)
 
 sw.bb:
   br label %return

diff --git a/test/CodeGen/X86/ragreedy-hoist-spill.ll b/test/CodeGen/X86/ragreedy-hoist-spill.ll
index a958c4b9eea836455dff11d9b07fe924911ee2f6..e7dda5349568f5d4fb589f86b1b6c520de08435c 100644 (file)
--- a/test/CodeGen/X86/ragreedy-hoist-spill.ll
+++ b/test/CodeGen/X86/ragreedy-hoist-spill.ll
@@ -30,7 +30,7 @@ if.then:
 if.end:
   switch i64 undef, label %if.end25 [
     i64 0, label %if.then4
-    i64 1, label %land.lhs.true14
+    i64 1, label %if.end25
   ]
 
 if.then4:

diff --git a/test/CodeGen/X86/switch.ll b/test/CodeGen/X86/switch.ll
index fd386e1f821bd2b51ae653a10f2b088fc02f19f6..2e5c0a6a1737a08c467bb3eda7324ebcc6c71fe8 100644 (file)
--- a/test/CodeGen/X86/switch.ll
+++ b/test/CodeGen/X86/switch.ll
@@ -413,3 +413,32 @@ return: ret void
        i32 4294967295, i32 2, i32 4294967295,
        ; Cases 2,5,8,9:
        i32 3, i32 3, i32 3, i32 3}
+
+define void @order_by_weight_and_fallthrough(i32 %x) {
+entry:
+  switch i32 %x, label %return [
+    i32 100, label %bb1
+    i32 200, label %bb0
+    i32 300, label %bb0
+  ], !prof !2
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+return: ret void
+
+; Case 200 has the highest weight and should come first. 100 and 300 have the
+; same weight, but 300 goes to the 'next' block, so should be last.
+; CHECK-LABEL: order_by_weight_and_fallthrough
+; CHECK: cmpl $200
+; CHECK: cmpl $100
+; CHECK: cmpl $300
+}
+
+!2 = !{!"branch_weights",
+       ; Default:
+       i32 1,
+       ; Case 100:
+       i32 10,
+       ; Case 200:
+       i32 1000,
+       ; Case 300:
+       i32 10}