Auto merge of #117749 - aliemjay:perf-canon-cache, r=lcnr
cache param env canonicalization Canonicalize ParamEnv only once and store it. Then whenever we try to canonicalize `ParamEnvAnd<'tcx, T>` we only have to canonicalize `T` and then merge the results. Prelimiary results show ~3-4% savings in diesel and serde benchmarks. Best to review commits individually. Some commits have a short description. Initial implementation had a soundness bug (https://github.com/rust-lang/rust/pull/117749#issuecomment-1840453387) due to cache invalidation: - When canonicalizing `Ty<'?0>` we first try to resolve region variables in the current InferCtxt which may have a constraint `?0 == 'static`. This means that we register `Ty<'?0> => Canonical<Ty<'static>>` in the cache, which is obviously incorrect in another inference context. - This is fixed by not doing region resolution when canonicalizing the query *input* (vs. response), which is the only place where ParamEnv is used, and then in a later commit we *statically* guard against any form of inference variable resolution of the cached canonical ParamEnv's. r? `@ghost`
This commit is contained in:
commit
d23e1a6894
8 changed files with 199 additions and 125 deletions
|
@ -35,36 +35,13 @@ impl<'tcx> InferCtxt<'tcx> {
|
||||||
/// [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html#canonicalizing-the-query
|
/// [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html#canonicalizing-the-query
|
||||||
pub fn canonicalize_query<V>(
|
pub fn canonicalize_query<V>(
|
||||||
&self,
|
&self,
|
||||||
value: V,
|
value: ty::ParamEnvAnd<'tcx, V>,
|
||||||
query_state: &mut OriginalQueryValues<'tcx>,
|
query_state: &mut OriginalQueryValues<'tcx>,
|
||||||
) -> Canonical<'tcx, V>
|
) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>>
|
||||||
where
|
where
|
||||||
V: TypeFoldable<TyCtxt<'tcx>>,
|
V: TypeFoldable<TyCtxt<'tcx>>,
|
||||||
{
|
{
|
||||||
Canonicalizer::canonicalize(value, self, self.tcx, &CanonicalizeAllFreeRegions, query_state)
|
self.canonicalize_query_with_mode(value, query_state, &CanonicalizeAllFreeRegions)
|
||||||
}
|
|
||||||
|
|
||||||
/// Like [Self::canonicalize_query], but preserves distinct universes. For
|
|
||||||
/// example, canonicalizing `&'?0: Trait<'?1>`, where `'?0` is in `U1` and
|
|
||||||
/// `'?1` is in `U3` would be canonicalized to have `?0` in `U1` and `'?1`
|
|
||||||
/// in `U2`.
|
|
||||||
///
|
|
||||||
/// This is used for Chalk integration.
|
|
||||||
pub fn canonicalize_query_preserving_universes<V>(
|
|
||||||
&self,
|
|
||||||
value: V,
|
|
||||||
query_state: &mut OriginalQueryValues<'tcx>,
|
|
||||||
) -> Canonical<'tcx, V>
|
|
||||||
where
|
|
||||||
V: TypeFoldable<TyCtxt<'tcx>>,
|
|
||||||
{
|
|
||||||
Canonicalizer::canonicalize(
|
|
||||||
value,
|
|
||||||
self,
|
|
||||||
self.tcx,
|
|
||||||
&CanonicalizeAllFreeRegionsPreservingUniverses,
|
|
||||||
query_state,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Canonicalizes a query *response* `V`. When we canonicalize a
|
/// Canonicalizes a query *response* `V`. When we canonicalize a
|
||||||
|
@ -99,7 +76,7 @@ impl<'tcx> InferCtxt<'tcx> {
|
||||||
let mut query_state = OriginalQueryValues::default();
|
let mut query_state = OriginalQueryValues::default();
|
||||||
Canonicalizer::canonicalize(
|
Canonicalizer::canonicalize(
|
||||||
value,
|
value,
|
||||||
self,
|
Some(self),
|
||||||
self.tcx,
|
self.tcx,
|
||||||
&CanonicalizeQueryResponse,
|
&CanonicalizeQueryResponse,
|
||||||
&mut query_state,
|
&mut query_state,
|
||||||
|
@ -113,7 +90,7 @@ impl<'tcx> InferCtxt<'tcx> {
|
||||||
let mut query_state = OriginalQueryValues::default();
|
let mut query_state = OriginalQueryValues::default();
|
||||||
Canonicalizer::canonicalize(
|
Canonicalizer::canonicalize(
|
||||||
value,
|
value,
|
||||||
self,
|
Some(self),
|
||||||
self.tcx,
|
self.tcx,
|
||||||
&CanonicalizeUserTypeAnnotation,
|
&CanonicalizeUserTypeAnnotation,
|
||||||
&mut query_state,
|
&mut query_state,
|
||||||
|
@ -126,19 +103,53 @@ impl<'tcx> InferCtxt<'tcx> {
|
||||||
/// handling of `'static` regions (e.g. trait evaluation).
|
/// handling of `'static` regions (e.g. trait evaluation).
|
||||||
pub fn canonicalize_query_keep_static<V>(
|
pub fn canonicalize_query_keep_static<V>(
|
||||||
&self,
|
&self,
|
||||||
value: V,
|
value: ty::ParamEnvAnd<'tcx, V>,
|
||||||
query_state: &mut OriginalQueryValues<'tcx>,
|
query_state: &mut OriginalQueryValues<'tcx>,
|
||||||
) -> Canonical<'tcx, V>
|
) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>>
|
||||||
where
|
where
|
||||||
V: TypeFoldable<TyCtxt<'tcx>>,
|
V: TypeFoldable<TyCtxt<'tcx>>,
|
||||||
{
|
{
|
||||||
Canonicalizer::canonicalize(
|
self.canonicalize_query_with_mode(
|
||||||
value,
|
value,
|
||||||
self,
|
query_state,
|
||||||
self.tcx,
|
|
||||||
&CanonicalizeFreeRegionsOtherThanStatic,
|
&CanonicalizeFreeRegionsOtherThanStatic,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonicalize_query_with_mode<V>(
|
||||||
|
&self,
|
||||||
|
value: ty::ParamEnvAnd<'tcx, V>,
|
||||||
|
query_state: &mut OriginalQueryValues<'tcx>,
|
||||||
|
canonicalize_region_mode: &dyn CanonicalizeMode,
|
||||||
|
) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>>
|
||||||
|
where
|
||||||
|
V: TypeFoldable<TyCtxt<'tcx>>,
|
||||||
|
{
|
||||||
|
let (param_env, value) = value.into_parts();
|
||||||
|
let base = self.tcx.canonical_param_env_cache.get_or_insert(
|
||||||
|
self.tcx,
|
||||||
|
param_env,
|
||||||
|
query_state,
|
||||||
|
|tcx, param_env, query_state| {
|
||||||
|
Canonicalizer::canonicalize(
|
||||||
|
param_env,
|
||||||
|
None,
|
||||||
|
tcx,
|
||||||
|
&CanonicalizeFreeRegionsOtherThanStatic,
|
||||||
|
query_state,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
Canonicalizer::canonicalize_with_base(
|
||||||
|
base,
|
||||||
|
value,
|
||||||
|
Some(self),
|
||||||
|
self.tcx,
|
||||||
|
canonicalize_region_mode,
|
||||||
query_state,
|
query_state,
|
||||||
)
|
)
|
||||||
|
.unchecked_map(|(param_env, value)| param_env.and(value))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -168,8 +179,22 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
|
||||||
fn canonicalize_free_region<'tcx>(
|
fn canonicalize_free_region<'tcx>(
|
||||||
&self,
|
&self,
|
||||||
canonicalizer: &mut Canonicalizer<'_, 'tcx>,
|
canonicalizer: &mut Canonicalizer<'_, 'tcx>,
|
||||||
r: ty::Region<'tcx>,
|
mut r: ty::Region<'tcx>,
|
||||||
) -> ty::Region<'tcx> {
|
) -> ty::Region<'tcx> {
|
||||||
|
let infcx = canonicalizer.infcx.unwrap();
|
||||||
|
|
||||||
|
if let ty::ReVar(vid) = *r {
|
||||||
|
r = infcx
|
||||||
|
.inner
|
||||||
|
.borrow_mut()
|
||||||
|
.unwrap_region_constraints()
|
||||||
|
.opportunistic_resolve_var(canonicalizer.tcx, vid);
|
||||||
|
debug!(
|
||||||
|
"canonical: region var found with vid {vid:?}, \
|
||||||
|
opportunistically resolved to {r:?}",
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
match *r {
|
match *r {
|
||||||
ty::ReLateParam(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyParam(..) => r,
|
ty::ReLateParam(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyParam(..) => r,
|
||||||
|
|
||||||
|
@ -179,7 +204,8 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
|
||||||
),
|
),
|
||||||
|
|
||||||
ty::ReVar(vid) => {
|
ty::ReVar(vid) => {
|
||||||
let universe = canonicalizer.region_var_universe(vid);
|
let universe =
|
||||||
|
infcx.inner.borrow_mut().unwrap_region_constraints().var_universe(vid);
|
||||||
canonicalizer.canonical_var_for_region(
|
canonicalizer.canonical_var_for_region(
|
||||||
CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) },
|
CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) },
|
||||||
r,
|
r,
|
||||||
|
@ -264,30 +290,6 @@ impl CanonicalizeMode for CanonicalizeAllFreeRegions {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct CanonicalizeAllFreeRegionsPreservingUniverses;
|
|
||||||
|
|
||||||
impl CanonicalizeMode for CanonicalizeAllFreeRegionsPreservingUniverses {
|
|
||||||
fn canonicalize_free_region<'tcx>(
|
|
||||||
&self,
|
|
||||||
canonicalizer: &mut Canonicalizer<'_, 'tcx>,
|
|
||||||
r: ty::Region<'tcx>,
|
|
||||||
) -> ty::Region<'tcx> {
|
|
||||||
let universe = canonicalizer.infcx.universe_of_region(r);
|
|
||||||
canonicalizer.canonical_var_for_region(
|
|
||||||
CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) },
|
|
||||||
r,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn any(&self) -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
fn preserve_universes(&self) -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
struct CanonicalizeFreeRegionsOtherThanStatic;
|
struct CanonicalizeFreeRegionsOtherThanStatic;
|
||||||
|
|
||||||
impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic {
|
impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic {
|
||||||
|
@ -309,7 +311,8 @@ impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic {
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Canonicalizer<'cx, 'tcx> {
|
struct Canonicalizer<'cx, 'tcx> {
|
||||||
infcx: &'cx InferCtxt<'tcx>,
|
/// Set to `None` to disable the resolution of inference variables.
|
||||||
|
infcx: Option<&'cx InferCtxt<'tcx>>,
|
||||||
tcx: TyCtxt<'tcx>,
|
tcx: TyCtxt<'tcx>,
|
||||||
variables: SmallVec<[CanonicalVarInfo<'tcx>; 8]>,
|
variables: SmallVec<[CanonicalVarInfo<'tcx>; 8]>,
|
||||||
query_state: &'cx mut OriginalQueryValues<'tcx>,
|
query_state: &'cx mut OriginalQueryValues<'tcx>,
|
||||||
|
@ -347,25 +350,12 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ty::ReVar(vid) => {
|
|
||||||
let resolved = self
|
|
||||||
.infcx
|
|
||||||
.inner
|
|
||||||
.borrow_mut()
|
|
||||||
.unwrap_region_constraints()
|
|
||||||
.opportunistic_resolve_var(self.tcx, vid);
|
|
||||||
debug!(
|
|
||||||
"canonical: region var found with vid {vid:?}, \
|
|
||||||
opportunistically resolved to {resolved:?}",
|
|
||||||
);
|
|
||||||
self.canonicalize_mode.canonicalize_free_region(self, resolved)
|
|
||||||
}
|
|
||||||
|
|
||||||
ty::ReStatic
|
ty::ReStatic
|
||||||
| ty::ReEarlyParam(..)
|
| ty::ReEarlyParam(..)
|
||||||
| ty::ReError(_)
|
| ty::ReError(_)
|
||||||
| ty::ReLateParam(_)
|
| ty::ReLateParam(_)
|
||||||
| ty::RePlaceholder(..)
|
| ty::RePlaceholder(..)
|
||||||
|
| ty::ReVar(_)
|
||||||
| ty::ReErased => self.canonicalize_mode.canonicalize_free_region(self, r),
|
| ty::ReErased => self.canonicalize_mode.canonicalize_free_region(self, r),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -376,14 +366,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
|
||||||
// We need to canonicalize the *root* of our ty var.
|
// We need to canonicalize the *root* of our ty var.
|
||||||
// This is so that our canonical response correctly reflects
|
// This is so that our canonical response correctly reflects
|
||||||
// any equated inference vars correctly!
|
// any equated inference vars correctly!
|
||||||
let root_vid = self.infcx.root_var(vid);
|
let root_vid = self.infcx.unwrap().root_var(vid);
|
||||||
if root_vid != vid {
|
if root_vid != vid {
|
||||||
t = Ty::new_var(self.infcx.tcx, root_vid);
|
t = Ty::new_var(self.tcx, root_vid);
|
||||||
vid = root_vid;
|
vid = root_vid;
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("canonical: type var found with vid {:?}", vid);
|
debug!("canonical: type var found with vid {:?}", vid);
|
||||||
match self.infcx.probe_ty_var(vid) {
|
match self.infcx.unwrap().probe_ty_var(vid) {
|
||||||
// `t` could be a float / int variable; canonicalize that instead.
|
// `t` could be a float / int variable; canonicalize that instead.
|
||||||
Ok(t) => {
|
Ok(t) => {
|
||||||
debug!("(resolved to {:?})", t);
|
debug!("(resolved to {:?})", t);
|
||||||
|
@ -408,7 +398,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
|
||||||
}
|
}
|
||||||
|
|
||||||
ty::Infer(ty::IntVar(vid)) => {
|
ty::Infer(ty::IntVar(vid)) => {
|
||||||
let nt = self.infcx.opportunistic_resolve_int_var(vid);
|
let nt = self.infcx.unwrap().opportunistic_resolve_int_var(vid);
|
||||||
if nt != t {
|
if nt != t {
|
||||||
return self.fold_ty(nt);
|
return self.fold_ty(nt);
|
||||||
} else {
|
} else {
|
||||||
|
@ -419,7 +409,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ty::Infer(ty::FloatVar(vid)) => {
|
ty::Infer(ty::FloatVar(vid)) => {
|
||||||
let nt = self.infcx.opportunistic_resolve_float_var(vid);
|
let nt = self.infcx.unwrap().opportunistic_resolve_float_var(vid);
|
||||||
if nt != t {
|
if nt != t {
|
||||||
return self.fold_ty(nt);
|
return self.fold_ty(nt);
|
||||||
} else {
|
} else {
|
||||||
|
@ -490,14 +480,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
|
||||||
// We need to canonicalize the *root* of our const var.
|
// We need to canonicalize the *root* of our const var.
|
||||||
// This is so that our canonical response correctly reflects
|
// This is so that our canonical response correctly reflects
|
||||||
// any equated inference vars correctly!
|
// any equated inference vars correctly!
|
||||||
let root_vid = self.infcx.root_const_var(vid);
|
let root_vid = self.infcx.unwrap().root_const_var(vid);
|
||||||
if root_vid != vid {
|
if root_vid != vid {
|
||||||
ct = ty::Const::new_var(self.infcx.tcx, root_vid, ct.ty());
|
ct = ty::Const::new_var(self.tcx, root_vid, ct.ty());
|
||||||
vid = root_vid;
|
vid = root_vid;
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("canonical: const var found with vid {:?}", vid);
|
debug!("canonical: const var found with vid {:?}", vid);
|
||||||
match self.infcx.probe_const_var(vid) {
|
match self.infcx.unwrap().probe_const_var(vid) {
|
||||||
Ok(c) => {
|
Ok(c) => {
|
||||||
debug!("(resolved to {:?})", c);
|
debug!("(resolved to {:?})", c);
|
||||||
return self.fold_const(c);
|
return self.fold_const(c);
|
||||||
|
@ -518,8 +508,8 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ty::ConstKind::Infer(InferConst::EffectVar(vid)) => {
|
ty::ConstKind::Infer(InferConst::EffectVar(vid)) => {
|
||||||
match self.infcx.probe_effect_var(vid) {
|
match self.infcx.unwrap().probe_effect_var(vid) {
|
||||||
Some(value) => return self.fold_const(value.as_const(self.infcx.tcx)),
|
Some(value) => return self.fold_const(value.as_const(self.tcx)),
|
||||||
None => {
|
None => {
|
||||||
return self.canonicalize_const_var(
|
return self.canonicalize_const_var(
|
||||||
CanonicalVarInfo { kind: CanonicalVarKind::Effect },
|
CanonicalVarInfo { kind: CanonicalVarKind::Effect },
|
||||||
|
@ -562,11 +552,38 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
/// `canonicalize_query` and `canonicalize_response`.
|
/// `canonicalize_query` and `canonicalize_response`.
|
||||||
fn canonicalize<V>(
|
fn canonicalize<V>(
|
||||||
value: V,
|
value: V,
|
||||||
infcx: &InferCtxt<'tcx>,
|
infcx: Option<&InferCtxt<'tcx>>,
|
||||||
tcx: TyCtxt<'tcx>,
|
tcx: TyCtxt<'tcx>,
|
||||||
canonicalize_region_mode: &dyn CanonicalizeMode,
|
canonicalize_region_mode: &dyn CanonicalizeMode,
|
||||||
query_state: &mut OriginalQueryValues<'tcx>,
|
query_state: &mut OriginalQueryValues<'tcx>,
|
||||||
) -> Canonical<'tcx, V>
|
) -> Canonical<'tcx, V>
|
||||||
|
where
|
||||||
|
V: TypeFoldable<TyCtxt<'tcx>>,
|
||||||
|
{
|
||||||
|
let base = Canonical {
|
||||||
|
max_universe: ty::UniverseIndex::ROOT,
|
||||||
|
variables: List::empty(),
|
||||||
|
value: (),
|
||||||
|
};
|
||||||
|
Canonicalizer::canonicalize_with_base(
|
||||||
|
base,
|
||||||
|
value,
|
||||||
|
infcx,
|
||||||
|
tcx,
|
||||||
|
canonicalize_region_mode,
|
||||||
|
query_state,
|
||||||
|
)
|
||||||
|
.unchecked_map(|((), val)| val)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn canonicalize_with_base<U, V>(
|
||||||
|
base: Canonical<'tcx, U>,
|
||||||
|
value: V,
|
||||||
|
infcx: Option<&InferCtxt<'tcx>>,
|
||||||
|
tcx: TyCtxt<'tcx>,
|
||||||
|
canonicalize_region_mode: &dyn CanonicalizeMode,
|
||||||
|
query_state: &mut OriginalQueryValues<'tcx>,
|
||||||
|
) -> Canonical<'tcx, (U, V)>
|
||||||
where
|
where
|
||||||
V: TypeFoldable<TyCtxt<'tcx>>,
|
V: TypeFoldable<TyCtxt<'tcx>>,
|
||||||
{
|
{
|
||||||
|
@ -578,12 +595,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
|
|
||||||
// Fast path: nothing that needs to be canonicalized.
|
// Fast path: nothing that needs to be canonicalized.
|
||||||
if !value.has_type_flags(needs_canonical_flags) {
|
if !value.has_type_flags(needs_canonical_flags) {
|
||||||
let canon_value = Canonical {
|
return base.unchecked_map(|b| (b, value));
|
||||||
max_universe: ty::UniverseIndex::ROOT,
|
|
||||||
variables: List::empty(),
|
|
||||||
value,
|
|
||||||
};
|
|
||||||
return canon_value;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut canonicalizer = Canonicalizer {
|
let mut canonicalizer = Canonicalizer {
|
||||||
|
@ -591,11 +603,20 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
tcx,
|
tcx,
|
||||||
canonicalize_mode: canonicalize_region_mode,
|
canonicalize_mode: canonicalize_region_mode,
|
||||||
needs_canonical_flags,
|
needs_canonical_flags,
|
||||||
variables: SmallVec::new(),
|
variables: SmallVec::from_slice(base.variables),
|
||||||
query_state,
|
query_state,
|
||||||
indices: FxHashMap::default(),
|
indices: FxHashMap::default(),
|
||||||
binder_index: ty::INNERMOST,
|
binder_index: ty::INNERMOST,
|
||||||
};
|
};
|
||||||
|
if canonicalizer.query_state.var_values.spilled() {
|
||||||
|
canonicalizer.indices = canonicalizer
|
||||||
|
.query_state
|
||||||
|
.var_values
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, &kind)| (kind, BoundVar::new(i)))
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
let out_value = value.fold_with(&mut canonicalizer);
|
let out_value = value.fold_with(&mut canonicalizer);
|
||||||
|
|
||||||
// Once we have canonicalized `out_value`, it should not
|
// Once we have canonicalized `out_value`, it should not
|
||||||
|
@ -612,7 +633,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
.max()
|
.max()
|
||||||
.unwrap_or(ty::UniverseIndex::ROOT);
|
.unwrap_or(ty::UniverseIndex::ROOT);
|
||||||
|
|
||||||
Canonical { max_universe, variables: canonical_variables, value: out_value }
|
Canonical { max_universe, variables: canonical_variables, value: (base.value, out_value) }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a canonical variable replacing `kind` from the input,
|
/// Creates a canonical variable replacing `kind` from the input,
|
||||||
|
@ -761,11 +782,6 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns the universe in which `vid` is defined.
|
|
||||||
fn region_var_universe(&self, vid: ty::RegionVid) -> ty::UniverseIndex {
|
|
||||||
self.infcx.inner.borrow_mut().unwrap_region_constraints().var_universe(vid)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Creates a canonical variable (with the given `info`)
|
/// Creates a canonical variable (with the given `info`)
|
||||||
/// representing the region `r`; return a region referencing it.
|
/// representing the region `r`; return a region referencing it.
|
||||||
fn canonical_var_for_region(
|
fn canonical_var_for_region(
|
||||||
|
@ -783,14 +799,9 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
/// *that*. Otherwise, create a new canonical variable for
|
/// *that*. Otherwise, create a new canonical variable for
|
||||||
/// `ty_var`.
|
/// `ty_var`.
|
||||||
fn canonicalize_ty_var(&mut self, info: CanonicalVarInfo<'tcx>, ty_var: Ty<'tcx>) -> Ty<'tcx> {
|
fn canonicalize_ty_var(&mut self, info: CanonicalVarInfo<'tcx>, ty_var: Ty<'tcx>) -> Ty<'tcx> {
|
||||||
let infcx = self.infcx;
|
debug_assert!(!self.infcx.is_some_and(|infcx| ty_var != infcx.shallow_resolve(ty_var)));
|
||||||
let bound_to = infcx.shallow_resolve(ty_var);
|
let var = self.canonical_var(info, ty_var.into());
|
||||||
if bound_to != ty_var {
|
Ty::new_bound(self.tcx, self.binder_index, var.into())
|
||||||
self.fold_ty(bound_to)
|
|
||||||
} else {
|
|
||||||
let var = self.canonical_var(info, ty_var.into());
|
|
||||||
Ty::new_bound(self.tcx, self.binder_index, var.into())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Given a type variable `const_var` of the given kind, first check
|
/// Given a type variable `const_var` of the given kind, first check
|
||||||
|
@ -802,13 +813,10 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
|
||||||
info: CanonicalVarInfo<'tcx>,
|
info: CanonicalVarInfo<'tcx>,
|
||||||
const_var: ty::Const<'tcx>,
|
const_var: ty::Const<'tcx>,
|
||||||
) -> ty::Const<'tcx> {
|
) -> ty::Const<'tcx> {
|
||||||
let infcx = self.infcx;
|
debug_assert!(
|
||||||
let bound_to = infcx.shallow_resolve(const_var);
|
!self.infcx.is_some_and(|infcx| const_var != infcx.shallow_resolve(const_var))
|
||||||
if bound_to != const_var {
|
);
|
||||||
self.fold_const(bound_to)
|
let var = self.canonical_var(info, const_var.into());
|
||||||
} else {
|
ty::Const::new_bound(self.tcx, self.binder_index, var, self.fold_ty(const_var.ty()))
|
||||||
let var = self.canonical_var(info, const_var.into());
|
|
||||||
ty::Const::new_bound(self.tcx, self.binder_index, var, self.fold_ty(const_var.ty()))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -172,7 +172,7 @@ impl<'tcx> InferCtxt<'tcx> {
|
||||||
// two const param's types are able to be equal has to go through a canonical query with the actual logic
|
// two const param's types are able to be equal has to go through a canonical query with the actual logic
|
||||||
// in `rustc_trait_selection`.
|
// in `rustc_trait_selection`.
|
||||||
let canonical = self.canonicalize_query(
|
let canonical = self.canonicalize_query(
|
||||||
(relation.param_env(), a.ty(), b.ty()),
|
relation.param_env().and((a.ty(), b.ty())),
|
||||||
&mut OriginalQueryValues::default(),
|
&mut OriginalQueryValues::default(),
|
||||||
);
|
);
|
||||||
self.tcx.check_tys_might_be_eq(canonical).map_err(|_| {
|
self.tcx.check_tys_might_be_eq(canonical).map_err(|_| {
|
||||||
|
|
|
@ -21,17 +21,20 @@
|
||||||
//!
|
//!
|
||||||
//! [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html
|
//! [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html
|
||||||
|
|
||||||
|
use rustc_data_structures::fx::FxHashMap;
|
||||||
|
use rustc_data_structures::sync::Lock;
|
||||||
use rustc_macros::HashStable;
|
use rustc_macros::HashStable;
|
||||||
use rustc_type_ir::Canonical as IrCanonical;
|
use rustc_type_ir::Canonical as IrCanonical;
|
||||||
use rustc_type_ir::CanonicalVarInfo as IrCanonicalVarInfo;
|
use rustc_type_ir::CanonicalVarInfo as IrCanonicalVarInfo;
|
||||||
pub use rustc_type_ir::{CanonicalTyVarKind, CanonicalVarKind};
|
pub use rustc_type_ir::{CanonicalTyVarKind, CanonicalVarKind};
|
||||||
use smallvec::SmallVec;
|
use smallvec::SmallVec;
|
||||||
|
use std::collections::hash_map::Entry;
|
||||||
use std::ops::Index;
|
use std::ops::Index;
|
||||||
|
|
||||||
use crate::infer::MemberConstraint;
|
use crate::infer::MemberConstraint;
|
||||||
use crate::mir::ConstraintCategory;
|
use crate::mir::ConstraintCategory;
|
||||||
use crate::ty::GenericArg;
|
use crate::ty::GenericArg;
|
||||||
use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt};
|
use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
|
||||||
|
|
||||||
pub type Canonical<'tcx, V> = IrCanonical<TyCtxt<'tcx>, V>;
|
pub type Canonical<'tcx, V> = IrCanonical<TyCtxt<'tcx>, V>;
|
||||||
|
|
||||||
|
@ -291,3 +294,62 @@ impl<'tcx> Index<BoundVar> for CanonicalVarValues<'tcx> {
|
||||||
&self.var_values[value.as_usize()]
|
&self.var_values[value.as_usize()]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct CanonicalParamEnvCache<'tcx> {
|
||||||
|
map: Lock<
|
||||||
|
FxHashMap<
|
||||||
|
ty::ParamEnv<'tcx>,
|
||||||
|
(Canonical<'tcx, ty::ParamEnv<'tcx>>, &'tcx [GenericArg<'tcx>]),
|
||||||
|
>,
|
||||||
|
>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'tcx> CanonicalParamEnvCache<'tcx> {
|
||||||
|
/// Gets the cached canonical form of `key` or executes
|
||||||
|
/// `canonicalize_op` and caches the result if not present.
|
||||||
|
///
|
||||||
|
/// `canonicalize_op` is intentionally not allowed to be a closure to
|
||||||
|
/// statically prevent it from capturing `InferCtxt` and resolving
|
||||||
|
/// inference variables, which invalidates the cache.
|
||||||
|
pub fn get_or_insert(
|
||||||
|
&self,
|
||||||
|
tcx: TyCtxt<'tcx>,
|
||||||
|
key: ty::ParamEnv<'tcx>,
|
||||||
|
state: &mut OriginalQueryValues<'tcx>,
|
||||||
|
canonicalize_op: fn(
|
||||||
|
TyCtxt<'tcx>,
|
||||||
|
ty::ParamEnv<'tcx>,
|
||||||
|
&mut OriginalQueryValues<'tcx>,
|
||||||
|
) -> Canonical<'tcx, ty::ParamEnv<'tcx>>,
|
||||||
|
) -> Canonical<'tcx, ty::ParamEnv<'tcx>> {
|
||||||
|
if !key.has_type_flags(
|
||||||
|
TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_FREE_REGIONS,
|
||||||
|
) {
|
||||||
|
return Canonical {
|
||||||
|
max_universe: ty::UniverseIndex::ROOT,
|
||||||
|
variables: List::empty(),
|
||||||
|
value: key,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
assert_eq!(state.var_values.len(), 0);
|
||||||
|
assert_eq!(state.universe_map.len(), 1);
|
||||||
|
debug_assert_eq!(&*state.universe_map, &[ty::UniverseIndex::ROOT]);
|
||||||
|
|
||||||
|
match self.map.borrow().entry(key) {
|
||||||
|
Entry::Occupied(e) => {
|
||||||
|
let (canonical, var_values) = e.get();
|
||||||
|
state.var_values.extend_from_slice(var_values);
|
||||||
|
canonical.clone()
|
||||||
|
}
|
||||||
|
Entry::Vacant(e) => {
|
||||||
|
let canonical = canonicalize_op(tcx, key, state);
|
||||||
|
let OriginalQueryValues { var_values, universe_map } = state;
|
||||||
|
assert_eq!(universe_map.len(), 1);
|
||||||
|
e.insert((canonical.clone(), tcx.arena.alloc_slice(var_values)));
|
||||||
|
canonical
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -2177,7 +2177,9 @@ rustc_queries! {
|
||||||
/// Used in `super_combine_consts` to ICE if the type of the two consts are definitely not going to end up being
|
/// Used in `super_combine_consts` to ICE if the type of the two consts are definitely not going to end up being
|
||||||
/// equal to eachother. This might return `Ok` even if the types are not equal, but will never return `Err` if
|
/// equal to eachother. This might return `Ok` even if the types are not equal, but will never return `Err` if
|
||||||
/// the types might be equal.
|
/// the types might be equal.
|
||||||
query check_tys_might_be_eq(arg: Canonical<'tcx, (ty::ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>) -> Result<(), NoSolution> {
|
query check_tys_might_be_eq(
|
||||||
|
arg: Canonical<'tcx, ty::ParamEnvAnd<'tcx, (Ty<'tcx>, Ty<'tcx>)>>
|
||||||
|
) -> Result<(), NoSolution> {
|
||||||
desc { "check whether two const param are definitely not equal to eachother"}
|
desc { "check whether two const param are definitely not equal to eachother"}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ pub mod tls;
|
||||||
|
|
||||||
use crate::arena::Arena;
|
use crate::arena::Arena;
|
||||||
use crate::dep_graph::{DepGraph, DepKindStruct};
|
use crate::dep_graph::{DepGraph, DepKindStruct};
|
||||||
use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
|
use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarInfo, CanonicalVarInfos};
|
||||||
use crate::lint::struct_lint_level;
|
use crate::lint::struct_lint_level;
|
||||||
use crate::metadata::ModChild;
|
use crate::metadata::ModChild;
|
||||||
use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
|
use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
|
||||||
|
@ -653,6 +653,8 @@ pub struct GlobalCtxt<'tcx> {
|
||||||
pub new_solver_evaluation_cache: solve::EvaluationCache<'tcx>,
|
pub new_solver_evaluation_cache: solve::EvaluationCache<'tcx>,
|
||||||
pub new_solver_coherence_evaluation_cache: solve::EvaluationCache<'tcx>,
|
pub new_solver_coherence_evaluation_cache: solve::EvaluationCache<'tcx>,
|
||||||
|
|
||||||
|
pub canonical_param_env_cache: CanonicalParamEnvCache<'tcx>,
|
||||||
|
|
||||||
/// Data layout specification for the current target.
|
/// Data layout specification for the current target.
|
||||||
pub data_layout: TargetDataLayout,
|
pub data_layout: TargetDataLayout,
|
||||||
|
|
||||||
|
@ -817,6 +819,7 @@ impl<'tcx> TyCtxt<'tcx> {
|
||||||
evaluation_cache: Default::default(),
|
evaluation_cache: Default::default(),
|
||||||
new_solver_evaluation_cache: Default::default(),
|
new_solver_evaluation_cache: Default::default(),
|
||||||
new_solver_coherence_evaluation_cache: Default::default(),
|
new_solver_coherence_evaluation_cache: Default::default(),
|
||||||
|
canonical_param_env_cache: Default::default(),
|
||||||
data_layout,
|
data_layout,
|
||||||
alloc_map: Lock::new(interpret::AllocMap::new()),
|
alloc_map: Lock::new(interpret::AllocMap::new()),
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,7 @@ use rustc_infer::infer::canonical::Canonical;
|
||||||
use rustc_infer::infer::{RegionResolutionError, TyCtxtInferExt};
|
use rustc_infer::infer::{RegionResolutionError, TyCtxtInferExt};
|
||||||
use rustc_infer::traits::query::NoSolution;
|
use rustc_infer::traits::query::NoSolution;
|
||||||
use rustc_infer::{infer::outlives::env::OutlivesEnvironment, traits::FulfillmentError};
|
use rustc_infer::{infer::outlives::env::OutlivesEnvironment, traits::FulfillmentError};
|
||||||
use rustc_middle::ty::{self, AdtDef, GenericArg, List, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
|
use rustc_middle::ty::{self, AdtDef, GenericArg, List, Ty, TyCtxt, TypeVisitableExt};
|
||||||
use rustc_span::DUMMY_SP;
|
use rustc_span::DUMMY_SP;
|
||||||
|
|
||||||
use super::outlives_bounds::InferCtxtExt;
|
use super::outlives_bounds::InferCtxtExt;
|
||||||
|
@ -209,10 +209,10 @@ pub fn all_fields_implement_trait<'tcx>(
|
||||||
|
|
||||||
pub fn check_tys_might_be_eq<'tcx>(
|
pub fn check_tys_might_be_eq<'tcx>(
|
||||||
tcx: TyCtxt<'tcx>,
|
tcx: TyCtxt<'tcx>,
|
||||||
canonical: Canonical<'tcx, (ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>,
|
canonical: Canonical<'tcx, ty::ParamEnvAnd<'tcx, (Ty<'tcx>, Ty<'tcx>)>>,
|
||||||
) -> Result<(), NoSolution> {
|
) -> Result<(), NoSolution> {
|
||||||
let (infcx, (param_env, ty_a, ty_b), _) =
|
let (infcx, key, _) = tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical);
|
||||||
tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical);
|
let (param_env, (ty_a, ty_b)) = key.into_parts();
|
||||||
let ocx = ObligationCtxt::new(&infcx);
|
let ocx = ObligationCtxt::new(&infcx);
|
||||||
|
|
||||||
let result = ocx.eq(&ObligationCause::dummy(), param_env, ty_a, ty_b);
|
let result = ocx.eq(&ObligationCause::dummy(), param_env, ty_a, ty_b);
|
||||||
|
|
|
@ -14,7 +14,6 @@ pub(crate) struct BlanketImplFinder<'a, 'tcx> {
|
||||||
impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> {
|
impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> {
|
||||||
pub(crate) fn get_blanket_impls(&mut self, item_def_id: DefId) -> Vec<Item> {
|
pub(crate) fn get_blanket_impls(&mut self, item_def_id: DefId) -> Vec<Item> {
|
||||||
let cx = &mut self.cx;
|
let cx = &mut self.cx;
|
||||||
let param_env = cx.tcx.param_env(item_def_id);
|
|
||||||
let ty = cx.tcx.type_of(item_def_id);
|
let ty = cx.tcx.type_of(item_def_id);
|
||||||
|
|
||||||
trace!("get_blanket_impls({ty:?})");
|
trace!("get_blanket_impls({ty:?})");
|
||||||
|
@ -40,7 +39,7 @@ impl<'a, 'tcx> BlanketImplFinder<'a, 'tcx> {
|
||||||
let infcx = cx.tcx.infer_ctxt().build();
|
let infcx = cx.tcx.infer_ctxt().build();
|
||||||
let args = infcx.fresh_args_for_item(DUMMY_SP, item_def_id);
|
let args = infcx.fresh_args_for_item(DUMMY_SP, item_def_id);
|
||||||
let impl_ty = ty.instantiate(infcx.tcx, args);
|
let impl_ty = ty.instantiate(infcx.tcx, args);
|
||||||
let param_env = EarlyBinder::bind(param_env).instantiate(infcx.tcx, args);
|
let param_env = ty::ParamEnv::empty();
|
||||||
|
|
||||||
let impl_args = infcx.fresh_args_for_item(DUMMY_SP, impl_def_id);
|
let impl_args = infcx.fresh_args_for_item(DUMMY_SP, impl_def_id);
|
||||||
let impl_trait_ref = trait_ref.instantiate(infcx.tcx, impl_args);
|
let impl_trait_ref = trait_ref.instantiate(infcx.tcx, impl_args);
|
||||||
|
|
|
@ -26,7 +26,7 @@ use rustc_middle::middle::resolve_bound_vars as rbv;
|
||||||
use rustc_middle::ty::fold::TypeFolder;
|
use rustc_middle::ty::fold::TypeFolder;
|
||||||
use rustc_middle::ty::GenericArgsRef;
|
use rustc_middle::ty::GenericArgsRef;
|
||||||
use rustc_middle::ty::TypeVisitableExt;
|
use rustc_middle::ty::TypeVisitableExt;
|
||||||
use rustc_middle::ty::{self, AdtKind, EarlyBinder, Ty, TyCtxt};
|
use rustc_middle::ty::{self, AdtKind, Ty, TyCtxt};
|
||||||
use rustc_middle::{bug, span_bug};
|
use rustc_middle::{bug, span_bug};
|
||||||
use rustc_span::hygiene::{AstPass, MacroKind};
|
use rustc_span::hygiene::{AstPass, MacroKind};
|
||||||
use rustc_span::symbol::{kw, sym, Ident, Symbol};
|
use rustc_span::symbol::{kw, sym, Ident, Symbol};
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue