bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

improve cold_path()

Browse source
This commit is contained in:
Jiri Bobek 2024-12-04 12:30:42 +01:00
parent c705b7d6f7
commit 7bb5f4dd78
6 changed files with 230 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

33
compiler/rustc_codegen_ssa/src/mir/block.rs
View file

@ -429,11 +429,34 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval);
bx.cond_br(cmp, ll1, ll2);
} else {
bx.switch(
discr_value,
helper.llbb_with_cleanup(self, targets.otherwise()),
target_iter.map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))),
);
let otherwise = targets.otherwise();
let otherwise_cold = self.cold_blocks[otherwise];
let otherwise_unreachable = self.mir[otherwise].is_empty_unreachable();
let cold_count = targets.iter().filter(|(_, target)| self.cold_blocks[*target]).count();
let none_cold = cold_count == 0;
let all_cold = cold_count == targets.iter().len();
if (none_cold && (!otherwise_cold || otherwise_unreachable))
|| (all_cold && (otherwise_cold || otherwise_unreachable))
{
// All targets have the same weight,
// or `otherwise` is unreachable and it's the only target with a different weight.
bx.switch(
discr_value,
helper.llbb_with_cleanup(self, targets.otherwise()),
target_iter
.map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))),
);
} else {
// Targets have different weights
bx.switch_with_weights(
discr_value,
helper.llbb_with_cleanup(self, targets.otherwise()),
otherwise_cold,
target_iter.map(|(value, target)| {
(value, helper.llbb_with_cleanup(self, target), self.cold_blocks[target])
}),
);
}
}
}

27
compiler/rustc_codegen_ssa/src/mir/mod.rs
View file

@ -502,14 +502,25 @@ fn find_cold_blocks<'tcx>(
for (bb, bb_data) in traversal::postorder(mir) {
let terminator = bb_data.terminator();
// If a BB ends with a call to a cold function, mark it as cold.
if let mir::TerminatorKind::Call { ref func, .. } = terminator.kind
&& let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
&& let attrs = tcx.codegen_fn_attrs(def_id)
&& attrs.flags.contains(CodegenFnAttrFlags::COLD)
{
cold_blocks[bb] = true;
continue;
match terminator.kind {
// If a BB ends with a call to a cold function, mark it as cold.
mir::TerminatorKind::Call { ref func, .. }
| mir::TerminatorKind::TailCall { ref func, .. }
if let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
&& let attrs = tcx.codegen_fn_attrs(def_id)
&& attrs.flags.contains(CodegenFnAttrFlags::COLD) =>
{
cold_blocks[bb] = true;
continue;
}
// If a BB ends with an `unreachable`, also mark it as cold.
mir::TerminatorKind::Unreachable => {
cold_blocks[bb] = true;
continue;
}
_ => {}
}
// If all successors of a BB are cold and there's at least one of them, mark this BB as cold

14
compiler/rustc_codegen_ssa/src/traits/builder.rs
View file

@ -110,6 +110,20 @@ pub trait BuilderMethods<'a, 'tcx>:
else_llbb: Self::BasicBlock,
cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock)>,
);
// This is like `switch()`, but every case has a bool flag indicating whether it's cold.
//
// Default implementation throws away the cold flags and calls `switch()`.
fn switch_with_weights(
&mut self,
v: Self::Value,
else_llbb: Self::BasicBlock,
_else_is_cold: bool,
cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
) {
self.switch(v, else_llbb, cases.map(|(val, bb, _)| (val, bb)))
}
fn invoke(
&mut self,
llty: Self::Type,