Auto merge of #117278 - lcnr:try-normalize-ty, r=compiler-errors

new solver normalization improvements

cool beans

At the core of this PR is a `try_normalize_ty` which stops for rigid aliases by using `commit_if_ok`.

Reworks alias-relate to fully normalize both the lhs and rhs and then equate the resulting rigid (or inference) types. This fixes https://github.com/rust-lang/trait-system-refactor-initiative/issues/68 by avoiding the exponential blowup. Also supersedes #116369 by only defining opaque types if the hidden type is rigid.

I removed the stability check in `EvalCtxt::evaluate_goal` due to https://github.com/rust-lang/trait-system-refactor-initiative/issues/75. While I personally have opinions on how to fix it, that still requires further t-types/`@nikomatsakis` buy-in, so I removed that for now. Once we've decided on our approach there, we can revert this commit.

r? `@compiler-errors`

compiler/rustc_middle/src/traits/solve/inspect.rs

@@ -122,6 +122,8 @@ pub enum ProbeStep<'tcx> {
     /// used whenever there are multiple candidates to prove the
     /// current goalby .
     NestedProbe(Probe<'tcx>),
+    CommitIfOkStart,
+    CommitIfOkSuccess,
 }
 
 /// What kind of probe we're in. In case the probe represents a candidate, or
@@ -142,6 +144,9 @@ pub enum ProbeKind<'tcx> {
     /// Used in the probe that wraps normalizing the non-self type for the unsize
     /// trait, which is also structurally matched on.
     UnsizeAssembly,
+    /// A call to `EvalCtxt::commit_if_ok` which failed, causing the work
+    /// to be discarded.
+    CommitIfOk,
     /// During upcasting from some source object to target object type, used to
     /// do a probe to find out what projection type(s) may be used to prove that
     /// the source type upholds all of the target type's object bounds.

compiler/rustc_middle/src/traits/solve/inspect/format.rs

@@ -109,6 +109,9 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
             ProbeKind::UpcastProjectionCompatibility => {
                 writeln!(self.f, "PROBING FOR PROJECTION COMPATIBILITY FOR UPCASTING:")
             }
+            ProbeKind::CommitIfOk => {
+                writeln!(self.f, "COMMIT_IF_OK:")
+            }
             ProbeKind::MiscCandidate { name, result } => {
                 writeln!(self.f, "CANDIDATE {name}: {result:?}")
             }
@@ -123,6 +126,8 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
                     ProbeStep::AddGoal(goal) => writeln!(this.f, "ADDED GOAL: {goal:?}")?,
                     ProbeStep::EvaluateGoals(eval) => this.format_added_goals_evaluation(eval)?,
                     ProbeStep::NestedProbe(probe) => this.format_probe(probe)?,
+                    ProbeStep::CommitIfOkStart => writeln!(this.f, "COMMIT_IF_OK START")?,
+                    ProbeStep::CommitIfOkSuccess => writeln!(this.f, "COMMIT_IF_OK SUCCESS")?,
                 }
             }
             Ok(())

compiler/rustc_middle/src/ty/sty.rs

@@ -1245,6 +1245,28 @@ impl<'tcx> AliasTy<'tcx> {
         }
     }
 
+    /// Whether this alias type is an opaque.
+    pub fn is_opaque(self, tcx: TyCtxt<'tcx>) -> bool {
+        matches!(self.opt_kind(tcx), Some(ty::AliasKind::Opaque))
+    }
+
+    /// FIXME: rename `AliasTy` to `AliasTerm` and always handle
+    /// constants. This function can then be removed.
+    pub fn opt_kind(self, tcx: TyCtxt<'tcx>) -> Option<ty::AliasKind> {
+        match tcx.def_kind(self.def_id) {
+            DefKind::AssocTy
+                if let DefKind::Impl { of_trait: false } =
+                    tcx.def_kind(tcx.parent(self.def_id)) =>
+            {
+                Some(ty::Inherent)
+            }
+            DefKind::AssocTy => Some(ty::Projection),
+            DefKind::OpaqueTy => Some(ty::Opaque),
+            DefKind::TyAlias => Some(ty::Weak),
+            _ => None,
+        }
+    }
+
     pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         Ty::new_alias(tcx, self.kind(tcx), self)
     }