Skip to content

[SimplfyCFG] Set MD_prof for select used for certain conditional simplifications #154426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: users/mtrofin/08-28-_simplifycfg_probabilities_associated_with_same_condition_are_constant
Choose a base branch
from
+108 −60
Open

[SimplfyCFG] Set MD_prof for select used for certain conditional simplifications #154426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion llvm/lib/Transforms/Utils/SimplifyCFG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ static cl::opt<unsigned> MaxJumpThreadingLiveBlocks(
cl::desc("Limit number of blocks a define in a threaded block is allowed "
"to be live in"));

extern cl::opt<bool> ProfcheckDisableMetadataFixes;

STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
STATISTIC(NumLinearMaps,
"Number of switch instructions turned into linear mapping");
Expand Down Expand Up @@ -330,6 +332,13 @@ class SimplifyCFGOpt {
}
};

bool isSelectInRoleOfConjunctionOrDisjunction(const SelectInst *SI) {
return ((isa<ConstantInt>(SI->getTrueValue()) &&
(dyn_cast<ConstantInt>(SI->getTrueValue())->isOne())) ||
(isa<ConstantInt>(SI->getFalseValue()) &&
(dyn_cast<ConstantInt>(SI->getFalseValue())->isNullValue())));
}

} // end anonymous namespace

/// Return true if all the PHI nodes in the basic block \p BB
Expand Down Expand Up @@ -4024,6 +4033,7 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,

// Try to update branch weights.
uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
SmallVector<uint32_t, 2> MDWeights;
if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
SuccTrueWeight, SuccFalseWeight)) {
SmallVector<uint64_t, 8> NewWeights;
Expand Down Expand Up @@ -4054,7 +4064,7 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
// Halve the weights if any of them cannot fit in an uint32_t
fitWeights(NewWeights);

SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end());
append_range(MDWeights, NewWeights);
setBranchWeights(PBI, MDWeights[0], MDWeights[1], /*IsExpected=*/false);

// TODO: If BB is reachable from all paths through PredBlock, then we
Expand Down Expand Up @@ -4091,6 +4101,13 @@ static bool performBranchToCommonDestFolding(BranchInst *BI, BranchInst *PBI,
Value *BICond = VMap[BI->getCondition()];
PBI->setCondition(
createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond"));
if (!ProfcheckDisableMetadataFixes)
if (auto *SI = dyn_cast<SelectInst>(PBI->getCondition()))
if (!MDWeights.empty()) {
assert(isSelectInRoleOfConjunctionOrDisjunction(SI));
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: I preferred assert-ing here (and below) because the expectation is the condition comes from createLogicalOp and the only reason we get a select is because of how or or and get lowered there.

setBranchWeights(SI, MDWeights[0], MDWeights[1],
/*IsExpected=*/false);
}

++NumFoldBranchToCommonDest;
return true;
Expand Down Expand Up @@ -4789,6 +4806,18 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
fitWeights(NewWeights);

setBranchWeights(PBI, NewWeights[0], NewWeights[1], /*IsExpected=*/false);
// Cond may be a select instruction with the first operand set to "true", or
// the second to "false" (see how createLogicalOp works for `and` and `or`)
if (!ProfcheckDisableMetadataFixes)
if (auto *SI = dyn_cast<SelectInst>(Cond)) {
assert(isSelectInRoleOfConjunctionOrDisjunction(SI));
// The select is predicated on PBICond
assert(dyn_cast<SelectInst>(SI)->getCondition() == PBICond);
// The corresponding probabilities are what was referred to above as
// PredCommon and PredOther.
setBranchWeights(SI, PredCommon, PredOther,
/*IsExpected=*/false);
}
}

// OtherDest may have phi nodes. If so, add an entry from PBI's
Expand Down
29 changes: 20 additions & 9 deletions llvm/test/Transforms/SimplifyCFG/branch-fold-threshold.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 5
; RUN: opt %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s --check-prefixes=NORMAL,BASELINE
; RUN: opt %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S -bonus-inst-threshold=2 | FileCheck %s --check-prefixes=NORMAL,AGGRESSIVE
; RUN: opt %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S -bonus-inst-threshold=4 | FileCheck %s --check-prefixes=WAYAGGRESSIVE
Expand All @@ -11,12 +11,12 @@ define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, ptr %input) {
; BASELINE-SAME: i32 [[A:%.*]], i32 [[B:%.*]], i32 [[C:%.*]], i32 [[D:%.*]], ptr [[INPUT:%.*]]) {
; BASELINE-NEXT: [[ENTRY:.*]]:
; BASELINE-NEXT: [[CMP:%.*]] = icmp sgt i32 [[D]], 3
; BASELINE-NEXT: br i1 [[CMP]], label %[[COND_END:.*]], label %[[LOR_LHS_FALSE:.*]]
; BASELINE-NEXT: br i1 [[CMP]], label %[[COND_END:.*]], label %[[LOR_LHS_FALSE:.*]], !prof [[PROF0:![0-9]+]]
; BASELINE: [[LOR_LHS_FALSE]]:
; BASELINE-NEXT: [[MUL:%.*]] = shl i32 [[C]], 1
; BASELINE-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[A]]
; BASELINE-NEXT: [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B]]
; BASELINE-NEXT: br i1 [[CMP1]], label %[[COND_FALSE:.*]], label %[[COND_END]]
; BASELINE-NEXT: br i1 [[CMP1]], label %[[COND_FALSE:.*]], label %[[COND_END]], !prof [[PROF1:![0-9]+]]
; BASELINE: [[COND_FALSE]]:
; BASELINE-NEXT: [[TMP0:%.*]] = load i32, ptr [[INPUT]], align 4
; BASELINE-NEXT: br label %[[COND_END]]
Expand All @@ -31,8 +31,8 @@ define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, ptr %input) {
; AGGRESSIVE-NEXT: [[MUL:%.*]] = shl i32 [[C]], 1
; AGGRESSIVE-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[A]]
; AGGRESSIVE-NEXT: [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B]]
; AGGRESSIVE-NEXT: [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false
; AGGRESSIVE-NEXT: br i1 [[OR_COND]], label %[[COND_FALSE:.*]], label %[[COND_END:.*]]
; AGGRESSIVE-NEXT: [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false, !prof [[PROF0:![0-9]+]]
; AGGRESSIVE-NEXT: br i1 [[OR_COND]], label %[[COND_FALSE:.*]], label %[[COND_END:.*]], !prof [[PROF0]]
; AGGRESSIVE: [[COND_FALSE]]:
; AGGRESSIVE-NEXT: [[TMP0:%.*]] = load i32, ptr [[INPUT]], align 4
; AGGRESSIVE-NEXT: br label %[[COND_END]]
Expand All @@ -47,8 +47,8 @@ define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, ptr %input) {
; WAYAGGRESSIVE-NEXT: [[MUL:%.*]] = shl i32 [[C]], 1
; WAYAGGRESSIVE-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[A]]
; WAYAGGRESSIVE-NEXT: [[CMP1:%.*]] = icmp slt i32 [[ADD]], [[B]]
; WAYAGGRESSIVE-NEXT: [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false
; WAYAGGRESSIVE-NEXT: br i1 [[OR_COND]], label %[[COND_FALSE:.*]], label %[[COND_END:.*]]
; WAYAGGRESSIVE-NEXT: [[OR_COND:%.*]] = select i1 [[CMP]], i1 [[CMP1]], i1 false, !prof [[PROF0:![0-9]+]]
; WAYAGGRESSIVE-NEXT: br i1 [[OR_COND]], label %[[COND_FALSE:.*]], label %[[COND_END:.*]], !prof [[PROF0]]
; WAYAGGRESSIVE: [[COND_FALSE]]:
; WAYAGGRESSIVE-NEXT: [[TMP0:%.*]] = load i32, ptr [[INPUT]], align 4
; WAYAGGRESSIVE-NEXT: br label %[[COND_END]]
Expand All @@ -58,13 +58,13 @@ define i32 @foo(i32 %a, i32 %b, i32 %c, i32 %d, ptr %input) {
;
entry:
%cmp = icmp sgt i32 %d, 3
br i1 %cmp, label %cond.end, label %lor.lhs.false
br i1 %cmp, label %cond.end, label %lor.lhs.false, !prof !0

lor.lhs.false:
%mul = shl i32 %c, 1
%add = add nsw i32 %mul, %a
%cmp1 = icmp slt i32 %add, %b
br i1 %cmp1, label %cond.false, label %cond.end
br i1 %cmp1, label %cond.false, label %cond.end, !prof !1

cond.false:
%0 = load i32, ptr %input, align 4
Expand Down Expand Up @@ -160,3 +160,14 @@ cond.end:
%cond = phi i32 [ %0, %cond.false ], [ 0, %lor.lhs.false ],[ 0, %pred_a ],[ 0, %pred_b ]
ret i32 %cond
}

!0 = !{!"branch_weights", i32 7, i32 11}
!1 = !{!"branch_weights", i32 13, i32 5}
;.
; BASELINE: [[PROF0]] = !{!"branch_weights", i32 7, i32 11}
; BASELINE: [[PROF1]] = !{!"branch_weights", i32 13, i32 5}
;.
; AGGRESSIVE: [[PROF0]] = !{!"branch_weights", i32 143, i32 181}
;.
; WAYAGGRESSIVE: [[PROF0]] = !{!"branch_weights", i32 143, i32 181}
;.
18 changes: 13 additions & 5 deletions llvm/test/Transforms/SimplifyCFG/branch-fold.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s

define void @test(ptr %P, ptr %Q, i1 %A, i1 %B) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[A_NOT:%.*]] = xor i1 [[A:%.*]], true
; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[A_NOT]], i1 true, i1 [[B:%.*]]
; CHECK-NEXT: br i1 [[BRMERGE]], label [[B:%.*]], label [[COMMON_RET:%.*]]
; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[A_NOT]], i1 true, i1 [[B:%.*]], !prof [[PROF0:![0-9]+]]
; CHECK-NEXT: br i1 [[BRMERGE]], label [[B:%.*]], label [[COMMON_RET:%.*]], !prof [[PROF0]]
; CHECK: common.ret:
; CHECK-NEXT: ret void
; CHECK: b:
Expand All @@ -15,9 +15,9 @@ define void @test(ptr %P, ptr %Q, i1 %A, i1 %B) {
;

entry:
br i1 %A, label %a, label %b
br i1 %A, label %a, label %b, !prof !0
a:
br i1 %B, label %b, label %c
br i1 %B, label %b, label %c, !prof !1
b:
store i32 123, ptr %P
ret void
Expand Down Expand Up @@ -146,3 +146,11 @@ Succ:
}

declare void @dummy()

!0 = !{!"branch_weights", i32 3, i32 7}
!1 = !{!"branch_weights", i32 11, i32 4}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind ssp memory(read) uwtable }
;.
; CHECK: [[PROF0]] = !{!"branch_weights", i32 138, i32 12}
;.
Loading
Loading