bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Rollup merge of #140024 - cjgillot:continue-jumping, r=compiler-errors

Browse source 
Remove early exits from JumpThreading.

This removes early exits from https://github.com/rust-lang/rust/pull/131203 as I asked during review.

The correctness of the backtracking is `mutated_statement` clearing all relevant conditions. If `process_statement` fails to insert a new condition, for instance by const-eval failure, `mutated_statement`  still removes the obsolete conditions from the state.

r? `@compiler-errors`
This commit is contained in:
Chris Denton 2025-04-19 19:30:49 +00:00 • committed by GitHub
commit 98515864d2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 64 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

135
compiler/rustc_mir_transform/src/jump_threading.rs
View file

@ -90,11 +90,7 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading {
}; };
for bb in body.basic_blocks.indices() { for bb in body.basic_blocks.indices() {
let old_len = finder.opportunities.len(); finder.start_from_switch(bb);
// If we have any const-eval errors discard any opportunities found
if finder.start_from_switch(bb).is_none() {
finder.opportunities.truncate(old_len);
}
} }
let opportunities = finder.opportunities; let opportunities = finder.opportunities;
@ -201,28 +197,26 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
/// Recursion entry point to find threading opportunities. /// Recursion entry point to find threading opportunities.
#[instrument(level = "trace", skip(self))] #[instrument(level = "trace", skip(self))]
fn start_from_switch(&mut self, bb: BasicBlock) -> Option<()> { fn start_from_switch(&mut self, bb: BasicBlock) {
let bbdata = &self.body[bb]; let bbdata = &self.body[bb];
if bbdata.is_cleanup || self.loop_headers.contains(bb) { if bbdata.is_cleanup || self.loop_headers.contains(bb) {
return Some(()); return;
} }
let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return Some(()) }; let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return };
let Some(discr) = discr.place() else { return Some(()) }; let Some(discr) = discr.place() else { return };
debug!(?discr, ?bb); debug!(?discr, ?bb);
let discr_ty = discr.ty(self.body, self.tcx).ty; let discr_ty = discr.ty(self.body, self.tcx).ty;
let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return };
return Some(());
};
let Some(discr) = self.map.find(discr.as_ref()) else { return Some(()) }; let Some(discr) = self.map.find(discr.as_ref()) else { return };
debug!(?discr); debug!(?discr);
let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body);
let mut state = State::new_reachable(); let mut state = State::new_reachable();
let conds = if let Some((value, then, else_)) = targets.as_static_if() { let conds = if let Some((value, then, else_)) = targets.as_static_if() {
let value = ScalarInt::try_from_uint(value, discr_layout.size)?; let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return };
self.arena.alloc_from_iter([ self.arena.alloc_from_iter([
Condition { value, polarity: Polarity::Eq, target: then }, Condition { value, polarity: Polarity::Eq, target: then },
Condition { value, polarity: Polarity::Ne, target: else_ }, Condition { value, polarity: Polarity::Ne, target: else_ },
@ -248,10 +242,10 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
mut state: State<ConditionSet<'a>>, mut state: State<ConditionSet<'a>>,
mut cost: CostChecker<'_, 'tcx>, mut cost: CostChecker<'_, 'tcx>,
depth: usize, depth: usize,
) -> Option<()> { ) {
// Do not thread through loop headers. // Do not thread through loop headers.
if self.loop_headers.contains(bb) { if self.loop_headers.contains(bb) {
return Some(()); return;
} }
debug!(cost = ?cost.cost()); debug!(cost = ?cost.cost());
@ -259,16 +253,16 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
self.body.basic_blocks[bb].statements.iter().enumerate().rev() self.body.basic_blocks[bb].statements.iter().enumerate().rev()
{ {
if self.is_empty(&state) { if self.is_empty(&state) {
return Some(()); return;
} }
cost.visit_statement(stmt, Location { block: bb, statement_index }); cost.visit_statement(stmt, Location { block: bb, statement_index });
if cost.cost() > MAX_COST { if cost.cost() > MAX_COST {
return Some(()); return;
} }
// Attempt to turn the `current_condition` on `lhs` into a condition on another place. // Attempt to turn the `current_condition` on `lhs` into a condition on another place.
self.process_statement(bb, stmt, &mut state)?; self.process_statement(bb, stmt, &mut state);
// When a statement mutates a place, assignments to that place that happen // When a statement mutates a place, assignments to that place that happen
// above the mutation cannot fulfill a condition. // above the mutation cannot fulfill a condition.
@ -280,7 +274,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
} }
if self.is_empty(&state) || depth >= MAX_BACKTRACK { if self.is_empty(&state) || depth >= MAX_BACKTRACK {
return Some(()); return;
} }
let last_non_rec = self.opportunities.len(); let last_non_rec = self.opportunities.len();
@ -293,9 +287,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
match term.kind { match term.kind {
TerminatorKind::SwitchInt { ref discr, ref targets } => { TerminatorKind::SwitchInt { ref discr, ref targets } => {
self.process_switch_int(discr, targets, bb, &mut state); self.process_switch_int(discr, targets, bb, &mut state);
self.find_opportunity(pred, state, cost, depth + 1)?; self.find_opportunity(pred, state, cost, depth + 1);
} }
_ => self.recurse_through_terminator(pred, || state, &cost, depth)?, _ => self.recurse_through_terminator(pred, || state, &cost, depth),
} }
} else if let &[ref predecessors @ .., last_pred] = &predecessors[..] { } else if let &[ref predecessors @ .., last_pred] = &predecessors[..] {
for &pred in predecessors { for &pred in predecessors {
@ -320,13 +314,12 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
let first = &mut new_tos[0]; let first = &mut new_tos[0];
*first = ThreadingOpportunity { chain: vec![bb], target: first.target }; *first = ThreadingOpportunity { chain: vec![bb], target: first.target };
self.opportunities.truncate(last_non_rec + 1); self.opportunities.truncate(last_non_rec + 1);
return Some(()); return;
} }
for op in self.opportunities[last_non_rec..].iter_mut() { for op in self.opportunities[last_non_rec..].iter_mut() {
op.chain.push(bb); op.chain.push(bb);
} }
Some(())
} }
/// Extract the mutated place from a statement. /// Extract the mutated place from a statement.
@ -440,23 +433,23 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
lhs: PlaceIndex, lhs: PlaceIndex,
rhs: &Operand<'tcx>, rhs: &Operand<'tcx>,
state: &mut State<ConditionSet<'a>>, state: &mut State<ConditionSet<'a>>,
) -> Option<()> { ) {
match rhs { match rhs {
// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`.
Operand::Constant(constant) => { Operand::Constant(constant) => {
let constant = self let Some(constant) =
.ecx self.ecx.eval_mir_constant(&constant.const_, constant.span, None).discard_err()
.eval_mir_constant(&constant.const_, constant.span, None) else {
.discard_err()?; return;
};
self.process_constant(bb, lhs, constant, state); self.process_constant(bb, lhs, constant, state);
} }
// Transfer the conditions on the copied rhs. // Transfer the conditions on the copied rhs.
Operand::Move(rhs) | Operand::Copy(rhs) => { Operand::Move(rhs) | Operand::Copy(rhs) => {
let Some(rhs) = self.map.find(rhs.as_ref()) else { return Some(()) }; let Some(rhs) = self.map.find(rhs.as_ref()) else { return };
state.insert_place_idx(rhs, lhs, &self.map); state.insert_place_idx(rhs, lhs, &self.map);
} }
} }
Some(())
} }
#[instrument(level = "trace", skip(self))] #[instrument(level = "trace", skip(self))]
@ -466,18 +459,14 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
lhs_place: &Place<'tcx>, lhs_place: &Place<'tcx>,
rhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>,
state: &mut State<ConditionSet<'a>>, state: &mut State<ConditionSet<'a>>,
) -> Option<()> { ) {
let Some(lhs) = self.map.find(lhs_place.as_ref()) else { let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return };
return Some(());
};
match rhs { match rhs {
Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state)?, Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state),
// Transfer the conditions on the copy rhs. // Transfer the conditions on the copy rhs.
Rvalue::CopyForDeref(rhs) => { Rvalue::CopyForDeref(rhs) => self.process_operand(bb, lhs, &Operand::Copy(*rhs), state),
self.process_operand(bb, lhs, &Operand::Copy(*rhs), state)?
}
Rvalue::Discriminant(rhs) => { Rvalue::Discriminant(rhs) => {
let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return Some(()) }; let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return };
state.insert_place_idx(rhs, lhs, &self.map); state.insert_place_idx(rhs, lhs, &self.map);
} }
// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`.
@ -485,7 +474,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
let agg_ty = lhs_place.ty(self.body, self.tcx).ty; let agg_ty = lhs_place.ty(self.body, self.tcx).ty;
let lhs = match kind { let lhs = match kind {
// Do not support unions. // Do not support unions.
AggregateKind::Adt(.., Some(_)) => return Some(()), AggregateKind::Adt(.., Some(_)) => return,
AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => { AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => {
if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant) if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant)
&& let Some(discr_value) = self && let Some(discr_value) = self
@ -498,23 +487,23 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) { if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) {
idx idx
} else { } else {
return Some(()); return;
} }
} }
_ => lhs, _ => lhs,
}; };
for (field_index, operand) in operands.iter_enumerated() { for (field_index, operand) in operands.iter_enumerated() {
if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) { if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) {
self.process_operand(bb, field, operand, state)?; self.process_operand(bb, field, operand, state);
} }
} }
} }
// Transfer the conditions on the copy rhs, after inverting the value of the condition. // Transfer the conditions on the copy rhs, after inverting the value of the condition.
Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => { Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => {
let layout = self.ecx.layout_of(place.ty(self.body, self.tcx).ty).unwrap(); let layout = self.ecx.layout_of(place.ty(self.body, self.tcx).ty).unwrap();
let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return Some(()) }; let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return };
let Some(place) = self.map.find(place.as_ref()) else { return Some(()) }; let Some(place) = self.map.find(place.as_ref()) else { return };
let conds = conditions.map(self.arena, |mut cond| { let Some(conds) = conditions.map(self.arena, |mut cond| {
cond.value = self cond.value = self
.ecx .ecx
.unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout)) .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout))
@ -522,7 +511,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
.to_scalar_int() .to_scalar_int()
.discard_err()?; .discard_err()?;
Some(cond) Some(cond)
})?; }) else {
return;
};
state.insert_value_idx(place, conds, &self.map); state.insert_value_idx(place, conds, &self.map);
} }
// We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`. // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`.
@ -532,34 +523,38 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value)) box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value))
| box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)), | box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)),
) => { ) => {
let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return Some(()) }; let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return };
let Some(place) = self.map.find(place.as_ref()) else { return Some(()) }; let Some(place) = self.map.find(place.as_ref()) else { return };
let equals = match op { let equals = match op {
BinOp::Eq => ScalarInt::TRUE, BinOp::Eq => ScalarInt::TRUE,
BinOp::Ne => ScalarInt::FALSE, BinOp::Ne => ScalarInt::FALSE,
_ => return Some(()), _ => return,
}; };
if value.const_.ty().is_floating_point() { if value.const_.ty().is_floating_point() {
// Floating point equality does not follow bit-patterns. // Floating point equality does not follow bit-patterns.
// -0.0 and NaN both have special rules for equality, // -0.0 and NaN both have special rules for equality,
// and therefore we cannot use integer comparisons for them. // and therefore we cannot use integer comparisons for them.
// Avoid handling them, though this could be extended in the future. // Avoid handling them, though this could be extended in the future.
return Some(()); return;
} }
let value = value.const_.try_eval_scalar_int(self.tcx, self.typing_env)?; let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env)
let conds = conditions.map(self.arena, |c| { else {
return;
};
let Some(conds) = conditions.map(self.arena, |c| {
Some(Condition { Some(Condition {
value, value,
polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne }, polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne },
..c ..c
}) })
})?; }) else {
return;
};
state.insert_value_idx(place, conds, &self.map); state.insert_value_idx(place, conds, &self.map);
} }
_ => {} _ => {}
} }
Some(())
} }
#[instrument(level = "trace", skip(self))] #[instrument(level = "trace", skip(self))]
@ -568,7 +563,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
bb: BasicBlock, bb: BasicBlock,
stmt: &Statement<'tcx>, stmt: &Statement<'tcx>,
state: &mut State<ConditionSet<'a>>, state: &mut State<ConditionSet<'a>>,
) -> Option<()> { ) {
let register_opportunity = |c: Condition| { let register_opportunity = |c: Condition| {
debug!(?bb, ?c.target, "register"); debug!(?bb, ?c.target, "register");
self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target })
@ -581,32 +576,30 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
// If we expect `discriminant(place) ?= A`, // If we expect `discriminant(place) ?= A`,
// we have an opportunity if `variant_index ?= A`. // we have an opportunity if `variant_index ?= A`.
StatementKind::SetDiscriminant { box place, variant_index } => { StatementKind::SetDiscriminant { box place, variant_index } => {
let Some(discr_target) = self.map.find_discr(place.as_ref()) else { let Some(discr_target) = self.map.find_discr(place.as_ref()) else { return };
return Some(());
};
let enum_ty = place.ty(self.body, self.tcx).ty; let enum_ty = place.ty(self.body, self.tcx).ty;
// `SetDiscriminant` guarantees that the discriminant is now `variant_index`. // `SetDiscriminant` guarantees that the discriminant is now `variant_index`.
// Even if the discriminant write does nothing due to niches, it is UB to set the // Even if the discriminant write does nothing due to niches, it is UB to set the
// discriminant when the data does not encode the desired discriminant. // discriminant when the data does not encode the desired discriminant.
let discr = let Some(discr) =
self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err()?; self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err()
self.process_immediate(bb, discr_target, discr, state); else {
return;
};
self.process_immediate(bb, discr_target, discr, state)
} }
// If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`.
StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(
Operand::Copy(place) | Operand::Move(place), Operand::Copy(place) | Operand::Move(place),
)) => { )) => {
let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { return };
return Some(()); conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity)
};
conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity);
} }
StatementKind::Assign(box (lhs_place, rhs)) => { StatementKind::Assign(box (lhs_place, rhs)) => {
self.process_assign(bb, lhs_place, rhs, state)?; self.process_assign(bb, lhs_place, rhs, state)
} }
_ => {} _ => {}
} }
Some(())
} }
#[instrument(level = "trace", skip(self, state, cost))] #[instrument(level = "trace", skip(self, state, cost))]
@ -617,7 +610,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
state: impl FnOnce() -> State<ConditionSet<'a>>, state: impl FnOnce() -> State<ConditionSet<'a>>,
cost: &CostChecker<'_, 'tcx>, cost: &CostChecker<'_, 'tcx>,
depth: usize, depth: usize,
) -> Option<()> { ) {
let term = self.body.basic_blocks[bb].terminator(); let term = self.body.basic_blocks[bb].terminator();
let place_to_flood = match term.kind { let place_to_flood = match term.kind {
// We come from a target, so those are not possible. // We come from a target, so those are not possible.
@ -632,9 +625,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
| TerminatorKind::FalseUnwind { .. } | TerminatorKind::FalseUnwind { .. }
| TerminatorKind::Yield { .. } => bug!("{term:?} invalid"), | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"),
// Cannot reason about inline asm. // Cannot reason about inline asm.
TerminatorKind::InlineAsm { .. } => return Some(()), TerminatorKind::InlineAsm { .. } => return,
// `SwitchInt` is handled specially. // `SwitchInt` is handled specially.
TerminatorKind::SwitchInt { .. } => return Some(()), TerminatorKind::SwitchInt { .. } => return,
// We can recurse, no thing particular to do. // We can recurse, no thing particular to do.
TerminatorKind::Goto { .. } => None, TerminatorKind::Goto { .. } => None,
// Flood the overwritten place, and progress through. // Flood the overwritten place, and progress through.