Auto merge of #2891 - RalfJung:rustup, r=RalfJung

Rustup

Cargo.lock

@@ -741,7 +741,7 @@ dependencies = [
  "tracing-subscriber",
  "unified-diff",
  "walkdir",
- "windows 0.46.0",
+ "windows",
 ]
 
 [[package]]
@@ -1647,7 +1647,7 @@ dependencies = [
  "iana-time-zone-haiku",
  "js-sys",
  "wasm-bindgen",
- "windows 0.48.0",
+ "windows",
 ]
 
 [[package]]
@@ -3259,7 +3259,7 @@ dependencies = [
  "tempfile",
  "thorin-dwp",
  "tracing",
- "windows 0.46.0",
+ "windows",
 ]
 
 [[package]]
@@ -3315,7 +3315,7 @@ dependencies = [
  "tempfile",
  "thin-vec",
  "tracing",
- "windows 0.46.0",
+ "windows",
 ]
 
 [[package]]
@@ -3376,7 +3376,7 @@ dependencies = [
  "rustc_ty_utils",
  "serde_json",
  "tracing",
- "windows 0.46.0",
+ "windows",
 ]
 
 [[package]]
@@ -3426,7 +3426,7 @@ dependencies = [
  "termize",
  "tracing",
  "unicode-width",
- "windows 0.46.0",
+ "windows",
 ]
 
 [[package]]
@@ -4096,7 +4096,7 @@ dependencies = [
  "smallvec",
  "termize",
  "tracing",
- "windows 0.46.0",
+ "windows",
 ]
 
 [[package]]
@@ -5498,15 +5498,6 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
-[[package]]
-name = "windows"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25"
-dependencies = [
- "windows-targets 0.42.2",
-]
-
 [[package]]
 name = "windows"
 version = "0.48.0"

compiler/rustc_borrowck/src/def_use.rs

@@ -55,7 +55,7 @@ pub fn categorize(context: PlaceContext) -> Option<DefUse> {
         // `PlaceMention` and `AscribeUserType` both evaluate the place, which must not
         // contain dangling references.
         PlaceContext::NonMutatingUse(NonMutatingUseContext::PlaceMention) |
-        PlaceContext::NonUse(NonUseContext::AscribeUserTy) |
+        PlaceContext::NonUse(NonUseContext::AscribeUserTy(_)) |
 
         PlaceContext::MutatingUse(MutatingUseContext::AddressOf) |
         PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf) |

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -777,7 +777,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
                 Inspect | Copy | Move | PlaceMention | SharedBorrow | ShallowBorrow | UniqueBorrow
                 | AddressOf | Projection,
             ) => ty::Covariant,
-            PlaceContext::NonUse(AscribeUserTy) => ty::Covariant,
+            PlaceContext::NonUse(AscribeUserTy(variance)) => variance,
         }
     }
 

compiler/rustc_builtin_macros/src/concat_idents.rs

@@ -19,7 +19,7 @@ pub fn expand_concat_idents<'cx>(
     }
 
     let mut res_str = String::new();
-    for (i, e) in tts.into_trees().enumerate() {
+    for (i, e) in tts.trees().enumerate() {
         if i & 1 == 1 {
             match e {
                 TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}

compiler/rustc_builtin_macros/src/trace_macros.rs

@@ -8,7 +8,7 @@ pub fn expand_trace_macros(
     sp: Span,
     tt: TokenStream,
 ) -> Box<dyn base::MacResult + 'static> {
-    let mut cursor = tt.into_trees();
+    let mut cursor = tt.trees();
     let mut err = false;
     let value = match &cursor.next() {
         Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true,

compiler/rustc_codegen_llvm/src/callee.rs

@@ -94,11 +94,11 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) ->
             // LLVM will prefix the name with `__imp_`. Ideally, we'd like the
             // existing logic below to set the Storage Class, but it has an
             // exemption for MinGW for backwards compatability.
-            let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi);
+            let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance));
             unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); }
             llfn
         } else {
-            cx.declare_fn(sym, fn_abi)
+            cx.declare_fn(sym, fn_abi, Some(instance))
         };
         debug!("get_fn: not casting pointer!");
 

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -207,6 +207,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<
             )),
             ty::List::empty(),
         ),
+        None,
     );
 
     llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage);

compiler/rustc_codegen_llvm/src/declare.rs

@@ -19,8 +19,11 @@ use crate::llvm::AttributePlace::Function;
 use crate::type_::Type;
 use crate::value::Value;
 use rustc_codegen_ssa::traits::TypeMembershipMethods;
-use rustc_middle::ty::Ty;
+use rustc_middle::ty::{Instance, Ty};
-use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions};
+use rustc_symbol_mangling::typeid::{
+    kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance,
+    TypeIdOptions,
+};
 use smallvec::SmallVec;
 
 /// Declare a function.
@@ -116,7 +119,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     ///
     /// If there’s a value with the same name already declared, the function will
     /// update the declaration and return existing Value instead.
-    pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value {
+    pub fn declare_fn(
+        &self,
+        name: &str,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+        instance: Option<Instance<'tcx>>,
+    ) -> &'ll Value {
         debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi);
 
         // Function addresses in Rust are never significant, allowing functions to
@@ -132,18 +140,35 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         fn_abi.apply_attrs_llfn(self, llfn);
 
         if self.tcx.sess.is_sanitizer_cfi_enabled() {
-            let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty());
+            if let Some(instance) = instance {
-            self.set_type_metadata(llfn, typeid);
+                let typeid = typeid_for_instance(self.tcx, &instance, TypeIdOptions::empty());
-            let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS);
+                self.set_type_metadata(llfn, typeid);
-            self.add_type_metadata(llfn, typeid);
+                let typeid =
-            let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS);
+                    typeid_for_instance(self.tcx, &instance, TypeIdOptions::GENERALIZE_POINTERS);
-            self.add_type_metadata(llfn, typeid);
+                self.add_type_metadata(llfn, typeid);
-            let typeid = typeid_for_fnabi(
+                let typeid =
-                self.tcx,
+                    typeid_for_instance(self.tcx, &instance, TypeIdOptions::NORMALIZE_INTEGERS);
-                fn_abi,
+                self.add_type_metadata(llfn, typeid);
-                TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS,
+                let typeid = typeid_for_instance(
-            );
+                    self.tcx,
-            self.add_type_metadata(llfn, typeid);
+                    &instance,
+                    TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS,
+                );
+                self.add_type_metadata(llfn, typeid);
+            } else {
+                let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty());
+                self.set_type_metadata(llfn, typeid);
+                let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS);
+                self.add_type_metadata(llfn, typeid);
+                let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS);
+                self.add_type_metadata(llfn, typeid);
+                let typeid = typeid_for_fnabi(
+                    self.tcx,
+                    fn_abi,
+                    TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS,
+                );
+                self.add_type_metadata(llfn, typeid);
+            }
         }
 
         if self.tcx.sess.is_sanitizer_kcfi_enabled() {
@@ -156,8 +181,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
                 options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
             }
 
-            let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options);
+            if let Some(instance) = instance {
-            self.set_kcfi_type_metadata(llfn, kcfi_typeid);
+                let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, &instance, options);
+                self.set_kcfi_type_metadata(llfn, kcfi_typeid);
+            } else {
+                let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options);
+                self.set_kcfi_type_metadata(llfn, kcfi_typeid);
+            }
         }
 
         llfn

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -772,7 +772,7 @@ fn gen_fn<'ll, 'tcx>(
 ) -> (&'ll Type, &'ll Value) {
     let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
     let llty = fn_abi.llvm_type(cx);
-    let llfn = cx.declare_fn(name, fn_abi);
+    let llfn = cx.declare_fn(name, fn_abi, None);
     cx.set_frame_pointer_type(llfn);
     cx.apply_target_cpu_attr(llfn);
     // FIXME(eddyb) find a nicer way to do this.

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -680,7 +680,9 @@ pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_v
 pub mod coverageinfo {
     use super::coverage_map;
 
-    /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230)
+    /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`.
+    ///
+    /// Must match the layout of `LLVMRustCounterMappingRegionKind`.
     #[derive(Copy, Clone, Debug)]
     #[repr(C)]
     pub enum RegionKind {
@@ -714,7 +716,9 @@ pub mod coverageinfo {
     /// array", encoded separately), and source location (start and end positions of the represented
     /// code region).
     ///
-    /// Matches LLVMRustCounterMappingRegion.
+    /// Corresponds to struct `llvm::coverage::CounterMappingRegion`.
+    ///
+    /// Must match the layout of `LLVMRustCounterMappingRegion`.
     #[derive(Copy, Clone, Debug)]
     #[repr(C)]
     pub struct CounterMappingRegion {

compiler/rustc_codegen_llvm/src/mono_item.rs

@@ -51,7 +51,7 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> {
         assert!(!instance.substs.has_infer());
 
         let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
-        let lldecl = self.declare_fn(symbol_name, fn_abi);
+        let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance));
         unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) };
         let attrs = self.tcx.codegen_fn_attrs(instance.def_id());
         base::set_link_section(lldecl, attrs);

compiler/rustc_codegen_ssa/Cargo.toml

@@ -51,5 +51,5 @@ default-features = false
 features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"]
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.46.0"
+version = "0.48.0"
 features = ["Win32_Globalization"]

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -546,10 +546,36 @@ fn link_staticlib<'a>(
 
     ab.build(out_filename);
 
-    if !all_native_libs.is_empty() {
+    let crates = codegen_results.crate_info.used_crates.iter();
-        if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) {
+
-            print_native_static_libs(sess, &all_native_libs);
+    let fmts = codegen_results
+        .crate_info
+        .dependency_formats
+        .iter()
+        .find_map(|&(ty, ref list)| if ty == CrateType::Staticlib { Some(list) } else { None })
+        .expect("no dependency formats for staticlib");
+
+    let mut all_rust_dylibs = vec![];
+    for &cnum in crates {
+        match fmts.get(cnum.as_usize() - 1) {
+            Some(&Linkage::Dynamic) => {}
+            _ => continue,
         }
+        let crate_name = codegen_results.crate_info.crate_name[&cnum];
+        let used_crate_source = &codegen_results.crate_info.used_crate_source[&cnum];
+        if let Some((path, _)) = &used_crate_source.dylib {
+            all_rust_dylibs.push(&**path);
+        } else {
+            if used_crate_source.rmeta.is_some() {
+                sess.emit_fatal(errors::LinkRlibError::OnlyRmetaFound { crate_name });
+            } else {
+                sess.emit_fatal(errors::LinkRlibError::NotFound { crate_name });
+            }
+        }
+    }
+
+    if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) {
+        print_native_static_libs(sess, &all_native_libs, &all_rust_dylibs);
     }
 
     Ok(())
@@ -1370,8 +1396,12 @@ enum RlibFlavor {
     StaticlibBase,
 }
 
-fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) {
+fn print_native_static_libs(
-    let lib_args: Vec<_> = all_native_libs
+    sess: &Session,
+    all_native_libs: &[NativeLib],
+    all_rust_dylibs: &[&Path],
+) {
+    let mut lib_args: Vec<_> = all_native_libs
         .iter()
         .filter(|l| relevant_lib(sess, l))
         .filter_map(|lib| {
@@ -1401,6 +1431,41 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) {
             }
         })
         .collect();
+    for path in all_rust_dylibs {
+        // FIXME deduplicate with add_dynamic_crate
+
+        // Just need to tell the linker about where the library lives and
+        // what its name is
+        let parent = path.parent();
+        if let Some(dir) = parent {
+            let dir = fix_windows_verbatim_for_gcc(dir);
+            if sess.target.is_like_msvc {
+                let mut arg = String::from("/LIBPATH:");
+                arg.push_str(&dir.display().to_string());
+                lib_args.push(arg);
+            } else {
+                lib_args.push("-L".to_owned());
+                lib_args.push(dir.display().to_string());
+            }
+        }
+        let stem = path.file_stem().unwrap().to_str().unwrap();
+        // Convert library file-stem into a cc -l argument.
+        let prefix = if stem.starts_with("lib") && !sess.target.is_like_windows { 3 } else { 0 };
+        let lib = &stem[prefix..];
+        let path = parent.unwrap_or_else(|| Path::new(""));
+        if sess.target.is_like_msvc {
+            // When producing a dll, the MSVC linker may not actually emit a
+            // `foo.lib` file if the dll doesn't actually export any symbols, so we
+            // check to see if the file is there and just omit linking to it if it's
+            // not present.
+            let name = format!("{}.dll.lib", lib);
+            if path.join(&name).exists() {
+                lib_args.push(name);
+            }
+        } else {
+            lib_args.push(format!("-l{}", lib));
+        }
+    }
     if !lib_args.is_empty() {
         sess.emit_note(errors::StaticLibraryNativeArtifacts);
         // Prefix for greppability

compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs

@@ -1,6 +1,6 @@
 use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex};
 
-/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L95)
+/// Must match the layout of `LLVMRustCounterKind`.
 #[derive(Copy, Clone, Debug)]
 #[repr(C)]
 pub enum CounterKind {
@@ -17,8 +17,10 @@ pub enum CounterKind {
 ///     `instrprof.increment()`)
 ///   * For `CounterKind::Expression`, `id` is the index into the coverage map's array of
 ///     counter expressions.
-/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L102-L103)
+///
-/// Important: The Rust struct layout (order and types of fields) must match its C++ counterpart.
+/// Corresponds to struct `llvm::coverage::Counter`.
+///
+/// Must match the layout of `LLVMRustCounter`.
 #[derive(Copy, Clone, Debug)]
 #[repr(C)]
 pub struct Counter {
@@ -59,7 +61,9 @@ impl Counter {
     }
 }
 
-/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L150)
+/// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`.
+///
+/// Must match the layout of `LLVMRustCounterExprKind`.
 #[derive(Copy, Clone, Debug)]
 #[repr(C)]
 pub enum ExprKind {
@@ -67,9 +71,9 @@ pub enum ExprKind {
     Add = 1,
 }
 
-/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L152)
+/// Corresponds to struct `llvm::coverage::CounterExpression`.
-/// Important: The Rust struct layout (order and types of fields) must match its C++
+///
-/// counterpart.
+/// Must match the layout of `LLVMRustCounterExpression`.
 #[derive(Copy, Clone, Debug)]
 #[repr(C)]
 pub struct CounterExpression {

compiler/rustc_data_structures/Cargo.toml

@@ -37,7 +37,7 @@ itertools = "0.10.1"
 version = "0.11"
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.46.0"
+version = "0.48.0"
 features = [
     "Win32_Foundation",
     "Win32_Storage_FileSystem",

compiler/rustc_data_structures/src/profiling.rs

@@ -865,14 +865,16 @@ cfg_if! {
             use std::mem;
 
             use windows::{
-                Win32::System::ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS},
+                // FIXME: change back to K32GetProcessMemoryInfo when windows crate
+                // updated to 0.49.0+ to drop dependency on psapi.dll
+                Win32::System::ProcessStatus::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS},
                 Win32::System::Threading::GetCurrentProcess,
             };
 
             let mut pmc = PROCESS_MEMORY_COUNTERS::default();
             let pmc_size = mem::size_of_val(&pmc);
             unsafe {
-                K32GetProcessMemoryInfo(
+                GetProcessMemoryInfo(
                     GetCurrentProcess(),
                     &mut pmc,
                     pmc_size as u32,

compiler/rustc_driver_impl/Cargo.toml

@@ -57,7 +57,7 @@ rustc_mir_transform = { path = "../rustc_mir_transform" }
 libc = "0.2"
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.46.0"
+version = "0.48.0"
 features = [
     "Win32_System_Diagnostics_Debug",
 ]

compiler/rustc_errors/Cargo.toml

@@ -27,12 +27,11 @@ serde = { version = "1.0.125", features = [ "derive" ] }
 serde_json = "1.0.59"
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.46.0"
+version = "0.48.0"
 features = [
     "Win32_Foundation",
     "Win32_Security",
     "Win32_System_Threading",
-    "Win32_System_WindowsProgramming",
 ]
 
 [features]

compiler/rustc_errors/src/lock.rs

@@ -19,8 +19,7 @@ pub fn acquire_global_lock(name: &str) -> Box<dyn Any> {
     use windows::{
         core::PCSTR,
         Win32::Foundation::{CloseHandle, HANDLE, WAIT_ABANDONED, WAIT_OBJECT_0},
-        Win32::System::Threading::{CreateMutexA, ReleaseMutex, WaitForSingleObject},
+        Win32::System::Threading::{CreateMutexA, ReleaseMutex, WaitForSingleObject, INFINITE},
-        Win32::System::WindowsProgramming::INFINITE,
     };
 
     struct Handle(HANDLE);

compiler/rustc_hir_analysis/src/check/check.rs

@@ -31,6 +31,7 @@ use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
+use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
 use rustc_trait_selection::traits::{self, ObligationCtxt, TraitEngine, TraitEngineExt as _};
 
 use std::ops::ControlFlow;
@@ -222,7 +223,7 @@ fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) {
     if check_opaque_for_cycles(tcx, item.owner_id.def_id, substs, span, &origin).is_err() {
         return;
     }
-    check_opaque_meets_bounds(tcx, item.owner_id.def_id, substs, span, &origin);
+    check_opaque_meets_bounds(tcx, item.owner_id.def_id, span, &origin);
 }
 
 /// Checks that an opaque type does not use `Self` or `T::Foo` projections that would result
@@ -391,7 +392,6 @@ pub(super) fn check_opaque_for_cycles<'tcx>(
 fn check_opaque_meets_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
-    substs: SubstsRef<'tcx>,
     span: Span,
     origin: &hir::OpaqueTyOrigin,
 ) {
@@ -406,6 +406,8 @@ fn check_opaque_meets_bounds<'tcx>(
         .with_opaque_type_inference(DefiningAnchor::Bind(defining_use_anchor))
         .build();
     let ocx = ObligationCtxt::new(&infcx);
+
+    let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
     let opaque_ty = tcx.mk_opaque(def_id.to_def_id(), substs);
 
     // `ReErased` regions appear in the "parent_substs" of closures/generators.
@@ -448,9 +450,18 @@ fn check_opaque_meets_bounds<'tcx>(
     match origin {
         // Checked when type checking the function containing them.
         hir::OpaqueTyOrigin::FnReturn(..) | hir::OpaqueTyOrigin::AsyncFn(..) => {}
+        // Nested opaque types occur only in associated types:
+        // ` type Opaque<T> = impl Trait<&'static T, AssocTy = impl Nested>; `
+        // They can only be referenced as `<Opaque<T> as Trait<&'static T>>::AssocTy`.
+        // We don't have to check them here because their well-formedness follows from the WF of
+        // the projection input types in the defining- and use-sites.
+        hir::OpaqueTyOrigin::TyAlias
+            if tcx.def_kind(tcx.parent(def_id.to_def_id())) == DefKind::OpaqueTy => {}
         // Can have different predicates to their defining use
         hir::OpaqueTyOrigin::TyAlias => {
-            let outlives_env = OutlivesEnvironment::new(param_env);
+            let wf_tys = ocx.assumed_wf_types(param_env, span, def_id);
+            let implied_bounds = infcx.implied_bounds_tys(param_env, def_id, wf_tys);
+            let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
             let _ = ocx.resolve_regions_and_report_errors(defining_use_anchor, &outlives_env);
         }
     }

compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs

@@ -278,9 +278,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         span: Span,
     ) -> bool {
         if let traits::FulfillmentErrorCode::CodeSelectionError(
-            traits::SelectionError::OutputTypeParameterMismatch(_, expected, _),
+            traits::SelectionError::OutputTypeParameterMismatch(box traits::SelectionOutputTypeParameterMismatch{
+                expected_trait_ref, ..
+            }),
         ) = error.code
-            && let ty::Closure(def_id, _) | ty::Generator(def_id, ..) = expected.skip_binder().self_ty().kind()
+            && let ty::Closure(def_id, _) | ty::Generator(def_id, ..) = expected_trait_ref.skip_binder().self_ty().kind()
             && span.overlaps(self.tcx.def_span(*def_id))
         {
             true

compiler/rustc_hir_typeck/src/lib.rs

@@ -2,6 +2,7 @@
 #![feature(let_chains)]
 #![feature(try_blocks)]
 #![feature(never_type)]
+#![feature(box_patterns)]
 #![feature(min_specialization)]
 #![feature(control_flow_enum)]
 #![feature(drain_filter)]

compiler/rustc_infer/src/infer/mod.rs

@@ -1530,7 +1530,7 @@ impl<'tcx> InferCtxt<'tcx> {
         // variables
         let tcx = self.tcx;
         if substs.has_non_region_infer() {
-            if let Some(ct) = tcx.bound_abstract_const(unevaluated.def)? {
+            if let Some(ct) = tcx.thir_abstract_const(unevaluated.def)? {
                 let ct = tcx.expand_abstract_consts(ct.subst(tcx, substs));
                 if let Err(e) = ct.error_reported() {
                     return Err(ErrorHandled::Reported(e));

compiler/rustc_interface/src/interface.rs

@@ -173,12 +173,21 @@ pub fn parse_check_cfg(specs: Vec<String>) -> CheckCfg {
                                         let expected_values = check_cfg
                                             .expecteds
                                             .entry(ident.name.to_string())
+                                            .and_modify(|expected_values| match expected_values {
+                                                ExpectedValues::Some(_) => {}
+                                                ExpectedValues::Any => {
+                                                    // handle the case where names(...) was done
+                                                    // before values by changing to a list
+                                                    *expected_values =
+                                                        ExpectedValues::Some(FxHashSet::default());
+                                                }
+                                            })
                                             .or_insert_with(|| {
                                                 ExpectedValues::Some(FxHashSet::default())
                                             });
 
                                         let ExpectedValues::Some(expected_values) = expected_values else {
-                                            bug!("shoudn't be possible")
+                                            bug!("`expected_values` should be a list a values")
                                         };
 
                                         for val in values {

compiler/rustc_lint/src/builtin.rs

@@ -1882,8 +1882,8 @@ declare_lint_pass!(
 struct UnderMacro(bool);
 
 impl KeywordIdents {
-    fn check_tokens(&mut self, cx: &EarlyContext<'_>, tokens: TokenStream) {
+    fn check_tokens(&mut self, cx: &EarlyContext<'_>, tokens: &TokenStream) {
-        for tt in tokens.into_trees() {
+        for tt in tokens.trees() {
             match tt {
                 // Only report non-raw idents.
                 TokenTree::Token(token, _) => {
@@ -1944,10 +1944,10 @@ impl KeywordIdents {
 
 impl EarlyLintPass for KeywordIdents {
     fn check_mac_def(&mut self, cx: &EarlyContext<'_>, mac_def: &ast::MacroDef) {
-        self.check_tokens(cx, mac_def.body.tokens.clone());
+        self.check_tokens(cx, &mac_def.body.tokens);
     }
     fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::MacCall) {
-        self.check_tokens(cx, mac.args.tokens.clone());
+        self.check_tokens(cx, &mac.args.tokens);
     }
     fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: Ident) {
         self.check_ident_token(cx, UnderMacro(false), ident);

compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp

@@ -8,18 +8,100 @@
 
 using namespace llvm;
 
+// FFI equivalent of enum `llvm::coverage::Counter::CounterKind`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L97-L99
+enum class LLVMRustCounterKind {
+  Zero = 0,
+  CounterValueReference = 1,
+  Expression = 2,
+};
+
+// FFI equivalent of struct `llvm::coverage::Counter`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L94-L149
+struct LLVMRustCounter {
+  LLVMRustCounterKind CounterKind;
+  uint32_t ID;
+};
+
+static coverage::Counter fromRust(LLVMRustCounter Counter) {
+  switch (Counter.CounterKind) {
+  case LLVMRustCounterKind::Zero:
+    return coverage::Counter::getZero();
+  case LLVMRustCounterKind::CounterValueReference:
+    return coverage::Counter::getCounter(Counter.ID);
+  case LLVMRustCounterKind::Expression:
+    return coverage::Counter::getExpression(Counter.ID);
+  }
+  report_fatal_error("Bad LLVMRustCounterKind!");
+}
+
+// FFI equivalent of enum `llvm::coverage::CounterMappingRegion::RegionKind`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L213-L234
+enum class LLVMRustCounterMappingRegionKind {
+  CodeRegion = 0,
+  ExpansionRegion = 1,
+  SkippedRegion = 2,
+  GapRegion = 3,
+  BranchRegion = 4,
+};
+
+static coverage::CounterMappingRegion::RegionKind
+fromRust(LLVMRustCounterMappingRegionKind Kind) {
+  switch (Kind) {
+  case LLVMRustCounterMappingRegionKind::CodeRegion:
+    return coverage::CounterMappingRegion::CodeRegion;
+  case LLVMRustCounterMappingRegionKind::ExpansionRegion:
+    return coverage::CounterMappingRegion::ExpansionRegion;
+  case LLVMRustCounterMappingRegionKind::SkippedRegion:
+    return coverage::CounterMappingRegion::SkippedRegion;
+  case LLVMRustCounterMappingRegionKind::GapRegion:
+    return coverage::CounterMappingRegion::GapRegion;
+  case LLVMRustCounterMappingRegionKind::BranchRegion:
+    return coverage::CounterMappingRegion::BranchRegion;
+  }
+  report_fatal_error("Bad LLVMRustCounterMappingRegionKind!");
+}
+
+// FFI equivalent of struct `llvm::coverage::CounterMappingRegion`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304
 struct LLVMRustCounterMappingRegion {
-  coverage::Counter Count;
+  LLVMRustCounter Count;
-  coverage::Counter FalseCount;
+  LLVMRustCounter FalseCount;
   uint32_t FileID;
   uint32_t ExpandedFileID;
   uint32_t LineStart;
   uint32_t ColumnStart;
   uint32_t LineEnd;
   uint32_t ColumnEnd;
-  coverage::CounterMappingRegion::RegionKind Kind;
+  LLVMRustCounterMappingRegionKind Kind;
 };
 
+// FFI equivalent of enum `llvm::coverage::CounterExpression::ExprKind`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L154
+enum class LLVMRustCounterExprKind {
+  Subtract = 0,
+  Add = 1,
+};
+
+// FFI equivalent of struct `llvm::coverage::CounterExpression`
+// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L160
+struct LLVMRustCounterExpression {
+  LLVMRustCounterExprKind Kind;
+  LLVMRustCounter LHS;
+  LLVMRustCounter RHS;
+};
+
+static coverage::CounterExpression::ExprKind
+fromRust(LLVMRustCounterExprKind Kind) {
+  switch (Kind) {
+  case LLVMRustCounterExprKind::Subtract:
+    return coverage::CounterExpression::Subtract;
+  case LLVMRustCounterExprKind::Add:
+    return coverage::CounterExpression::Add;
+  }
+  report_fatal_error("Bad LLVMRustCounterExprKind!");
+}
+
 extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer(
     const char* const Filenames[],
     size_t FilenamesLen,
@@ -37,9 +119,9 @@ extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer(
 extern "C" void LLVMRustCoverageWriteMappingToBuffer(
     const unsigned *VirtualFileMappingIDs,
     unsigned NumVirtualFileMappingIDs,
-    const coverage::CounterExpression *Expressions,
+    const LLVMRustCounterExpression *RustExpressions,
     unsigned NumExpressions,
-    LLVMRustCounterMappingRegion *RustMappingRegions,
+    const LLVMRustCounterMappingRegion *RustMappingRegions,
     unsigned NumMappingRegions,
     RustStringRef BufferOut) {
   // Convert from FFI representation to LLVM representation.
@@ -48,13 +130,24 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer(
   for (const auto &Region : ArrayRef<LLVMRustCounterMappingRegion>(
            RustMappingRegions, NumMappingRegions)) {
     MappingRegions.emplace_back(
-        Region.Count, Region.FalseCount, Region.FileID, Region.ExpandedFileID,
+        fromRust(Region.Count), fromRust(Region.FalseCount),
+        Region.FileID, Region.ExpandedFileID,
         Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd,
-        Region.Kind);
+        fromRust(Region.Kind));
   }
+
+  std::vector<coverage::CounterExpression> Expressions;
+  Expressions.reserve(NumExpressions);
+  for (const auto &Expression :
+       ArrayRef<LLVMRustCounterExpression>(RustExpressions, NumExpressions)) {
+    Expressions.emplace_back(fromRust(Expression.Kind),
+                             fromRust(Expression.LHS),
+                             fromRust(Expression.RHS));
+  }
+
   auto CoverageMappingWriter = coverage::CoverageMappingWriter(
       ArrayRef<unsigned>(VirtualFileMappingIDs, NumVirtualFileMappingIDs),
-      ArrayRef<coverage::CounterExpression>(Expressions, NumExpressions),
+      Expressions,
       MappingRegions);
   RawRustStringOstream OS(BufferOut);
   CoverageMappingWriter.write(OS);

compiler/rustc_metadata/src/dependency_format.rs

@@ -89,11 +89,25 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         // to try to eagerly statically link all dependencies. This is normally
         // done for end-product dylibs, not intermediate products.
         //
-        // Treat cdylibs similarly. If `-C prefer-dynamic` is set, the caller may
+        // Treat cdylibs and staticlibs similarly. If `-C prefer-dynamic` is set,
-        // be code-size conscious, but without it, it makes sense to statically
+        // the caller may be code-size conscious, but without it, it makes sense
-        // link a cdylib.
+        // to statically link a cdylib or staticlib. For staticlibs we use
-        CrateType::Dylib | CrateType::Cdylib if !sess.opts.cg.prefer_dynamic => Linkage::Static,
+        // `-Z staticlib-prefer-dynamic` for now. This may be merged into
-        CrateType::Dylib | CrateType::Cdylib => Linkage::Dynamic,
+        // `-C prefer-dynamic` in the future.
+        CrateType::Dylib | CrateType::Cdylib => {
+            if sess.opts.cg.prefer_dynamic {
+                Linkage::Dynamic
+            } else {
+                Linkage::Static
+            }
+        }
+        CrateType::Staticlib => {
+            if sess.opts.unstable_opts.staticlib_prefer_dynamic {
+                Linkage::Dynamic
+            } else {
+                Linkage::Static
+            }
+        }
 
         // If the global prefer_dynamic switch is turned off, or the final
         // executable will be statically linked, prefer static crate linkage.
@@ -108,9 +122,6 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         // No linkage happens with rlibs, we just needed the metadata (which we
         // got long ago), so don't bother with anything.
         CrateType::Rlib => Linkage::NotLinked,
-
-        // staticlibs must have all static dependencies.
-        CrateType::Staticlib => Linkage::Static,
     };
 
     match preferred_linkage {
@@ -123,9 +134,9 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
                 return v;
             }
 
-            // Staticlibs and static executables must have all static dependencies.
+            // Static executables must have all static dependencies.
             // If any are not found, generate some nice pretty errors.
-            if ty == CrateType::Staticlib
+            if (ty == CrateType::Staticlib && !sess.opts.unstable_opts.staticlib_allow_rdylib_deps)
                 || (ty == CrateType::Executable
                     && sess.crt_static(Some(ty))
                     && !sess.target.crt_static_allows_dylibs)

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -862,6 +862,11 @@ fn should_encode_attrs(def_kind: DefKind) -> bool {
         | DefKind::Macro(_)
         | DefKind::Field
         | DefKind::Impl { .. } => true,
+        // Tools may want to be able to detect their tool lints on
+        // closures from upstream crates, too. This is used by
+        // https://github.com/model-checking/kani and is not a performance
+        // or maintenance issue for us.
+        DefKind::Closure => true,
         DefKind::TyParam
         | DefKind::ConstParam
         | DefKind::Ctor(..)
@@ -874,7 +879,6 @@ fn should_encode_attrs(def_kind: DefKind) -> bool {
         | DefKind::ImplTraitPlaceholder
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Closure
         | DefKind::Generator => false,
     }
 }

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -394,7 +394,7 @@ define_tables! {
     mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
     mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
     promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,
-    thir_abstract_const: Table<DefIndex, LazyValue<ty::Const<'static>>>,
+    thir_abstract_const: Table<DefIndex, LazyValue<ty::EarlyBinder<ty::Const<'static>>>>,
     impl_parent: Table<DefIndex, RawDefId>,
     impl_polarity: Table<DefIndex, ty::ImplPolarity>,
     constness: Table<DefIndex, hir::Constness>,

compiler/rustc_middle/src/mir/visit.rs

@@ -64,7 +64,7 @@
 
 use crate::mir::*;
 use crate::ty::subst::SubstsRef;
-use crate::ty::{CanonicalUserTypeAnnotation, Ty};
+use crate::ty::{self, CanonicalUserTypeAnnotation, Ty};
 use rustc_span::Span;
 
 macro_rules! make_mir_visitor {
@@ -782,12 +782,12 @@ macro_rules! make_mir_visitor {
 
             fn super_ascribe_user_ty(&mut self,
                                      place: & $($mutability)? Place<'tcx>,
-                                     _variance: $(& $mutability)? ty::Variance,
+                                     variance: $(& $mutability)? ty::Variance,
                                      user_ty: & $($mutability)? UserTypeProjection,
                                      location: Location) {
                 self.visit_place(
                     place,
-                    PlaceContext::NonUse(NonUseContext::AscribeUserTy),
+                    PlaceContext::NonUse(NonUseContext::AscribeUserTy($(* &$mutability *)? variance)),
                     location
                 );
                 self.visit_user_type_projection(user_ty);
@@ -1320,7 +1320,7 @@ pub enum NonUseContext {
     /// Ending a storage live range.
     StorageDead,
     /// User type annotation assertions for NLL.
-    AscribeUserTy,
+    AscribeUserTy(ty::Variance),
     /// The data of a user variable, for debug info.
     VarDebugInfo,
 }

compiler/rustc_middle/src/query/erase.rs

@@ -82,9 +82,10 @@ impl EraseType for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuarantee
         [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
 }
 
-impl EraseType for Result<Option<ty::Const<'_>>, rustc_errors::ErrorGuaranteed> {
+impl EraseType for Result<Option<ty::EarlyBinder<ty::Const<'_>>>, rustc_errors::ErrorGuaranteed> {
-    type Result =
+    type Result = [u8; size_of::<
-        [u8; size_of::<Result<Option<ty::Const<'static>>, rustc_errors::ErrorGuaranteed>>()];
+        Result<Option<ty::EarlyBinder<ty::Const<'static>>>, rustc_errors::ErrorGuaranteed>,
+    >()];
 }
 
 impl EraseType for Result<ty::GenericArg<'_>, traits::query::NoSolution> {

compiler/rustc_middle/src/query/mod.rs

@@ -402,7 +402,7 @@ rustc_queries! {
     /// Try to build an abstract representation of the given constant.
     query thir_abstract_const(
         key: DefId
-    ) -> Result<Option<ty::Const<'tcx>>, ErrorGuaranteed> {
+    ) -> Result<Option<ty::EarlyBinder<ty::Const<'tcx>>>, ErrorGuaranteed> {
         desc {
             |tcx| "building an abstract representation for `{}`", tcx.def_path_str(key),
         }
@@ -1016,7 +1016,7 @@ rustc_queries! {
         desc { "converting literal to mir constant" }
     }
 
-    query check_match(key: LocalDefId) {
+    query check_match(key: LocalDefId) -> Result<(), rustc_errors::ErrorGuaranteed> {
         desc { |tcx| "match-checking `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }

compiler/rustc_middle/src/traits/mod.rs

@@ -580,11 +580,7 @@ pub enum SelectionError<'tcx> {
     /// After a closure impl has selected, its "outputs" were evaluated
     /// (which for closures includes the "input" type params) and they
     /// didn't resolve. See `confirm_poly_trait_refs` for more.
-    OutputTypeParameterMismatch(
+    OutputTypeParameterMismatch(Box<SelectionOutputTypeParameterMismatch<'tcx>>),
-        ty::PolyTraitRef<'tcx>,
-        ty::PolyTraitRef<'tcx>,
-        ty::error::TypeError<'tcx>,
-    ),
     /// The trait pointed by `DefId` is not object safe.
     TraitNotObjectSafe(DefId),
     /// A given constant couldn't be evaluated.
@@ -596,6 +592,13 @@ pub enum SelectionError<'tcx> {
     ErrorReporting,
 }
 
+#[derive(Clone, Debug, TypeVisitable, Lift)]
+pub struct SelectionOutputTypeParameterMismatch<'tcx> {
+    pub found_trait_ref: ty::PolyTraitRef<'tcx>,
+    pub expected_trait_ref: ty::PolyTraitRef<'tcx>,
+    pub terr: ty::error::TypeError<'tcx>,
+}
+
 /// When performing resolution, it is typically the case that there
 /// can be one of three outcomes:
 ///

compiler/rustc_middle/src/ty/abstract_const.rs

@@ -4,7 +4,6 @@ use crate::ty::{
     TypeVisitableExt,
 };
 use rustc_errors::ErrorGuaranteed;
-use rustc_hir::def_id::DefId;
 
 #[derive(Hash, Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq)]
 #[derive(TyDecodable, TyEncodable, HashStable, TypeVisitable, TypeFoldable)]
@@ -35,12 +34,6 @@ TrivialTypeTraversalAndLiftImpls! {
 pub type BoundAbstractConst<'tcx> = Result<Option<EarlyBinder<ty::Const<'tcx>>>, ErrorGuaranteed>;
 
 impl<'tcx> TyCtxt<'tcx> {
-    /// Returns a const without substs applied
-    pub fn bound_abstract_const(self, uv: DefId) -> BoundAbstractConst<'tcx> {
-        let ac = self.thir_abstract_const(uv);
-        Ok(ac?.map(|ac| EarlyBinder(ac)))
-    }
-
     pub fn expand_abstract_consts<T: TypeFoldable<TyCtxt<'tcx>>>(self, ac: T) -> T {
         struct Expander<'tcx> {
             tcx: TyCtxt<'tcx>,
@@ -59,7 +52,7 @@ impl<'tcx> TyCtxt<'tcx> {
             }
             fn fold_const(&mut self, c: Const<'tcx>) -> Const<'tcx> {
                 let ct = match c.kind() {
-                    ty::ConstKind::Unevaluated(uv) => match self.tcx.bound_abstract_const(uv.def) {
+                    ty::ConstKind::Unevaluated(uv) => match self.tcx.thir_abstract_const(uv.def) {
                         Err(e) => self.tcx.const_error_with_guaranteed(c.ty(), e),
                         Ok(Some(bac)) => {
                             let substs = self.tcx.erase_regions(uv.substs);

compiler/rustc_middle/src/ty/sty.rs

@@ -2366,13 +2366,11 @@ impl<'tcx> Ty<'tcx> {
 
             ty::Adt(def, _substs) => def.sized_constraint(tcx).0.is_empty(),
 
-            ty::Alias(..) | ty::Param(_) => false,
+            ty::Alias(..) | ty::Param(_) | ty::Placeholder(..) => false,
 
             ty::Infer(ty::TyVar(_)) => false,
 
-            ty::Bound(..)
+            ty::Bound(..) | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
-            | ty::Placeholder(..)
-            | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
                 bug!("`is_trivially_sized` applied to unexpected type: {:?}", self)
             }
         }

compiler/rustc_middle/src/ty/util.rs

@@ -1253,7 +1253,7 @@ pub enum ExplicitSelf<'tcx> {
 
 impl<'tcx> ExplicitSelf<'tcx> {
     /// Categorizes an explicit self declaration like `self: SomeType`
-    /// into either `self`, `&self`, `&mut self`, `Box<self>`, or
+    /// into either `self`, `&self`, `&mut self`, `Box<Self>`, or
     /// `Other`.
     /// This is mainly used to require the arbitrary_self_types feature
     /// in the case of `Other`, to improve error messages in the common cases,

compiler/rustc_mir_build/src/build/mod.rs

@@ -42,7 +42,9 @@ fn mir_build(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
     // Ensure unsafeck and abstract const building is ran before we steal the THIR.
     tcx.ensure_with_value().thir_check_unsafety(def);
     tcx.ensure_with_value().thir_abstract_const(def);
-    tcx.ensure_with_value().check_match(def);
+    if let Err(e) = tcx.check_match(def) {
+        return construct_error(tcx, def, e);
+    }
 
     let body = match tcx.thir_body(def) {
         Err(error_reported) => construct_error(tcx, def, error_reported),

compiler/rustc_mir_build/src/thir/pattern/check_match.rs

@@ -26,8 +26,8 @@ use rustc_session::Session;
 use rustc_span::hygiene::DesugaringKind;
 use rustc_span::Span;
 
-pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) {
+pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(), ErrorGuaranteed> {
-    let Ok((thir, expr)) = tcx.thir_body(def_id) else { return };
+    let (thir, expr) = tcx.thir_body(def_id)?;
     let thir = thir.borrow();
     let pattern_arena = TypedArena::default();
     let mut visitor = MatchVisitor {
@@ -37,13 +37,16 @@ pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) {
         lint_level: tcx.hir().local_def_id_to_hir_id(def_id),
         let_source: LetSource::None,
         pattern_arena: &pattern_arena,
+        error: Ok(()),
     };
     visitor.visit_expr(&thir[expr]);
+
     for param in thir.params.iter() {
         if let Some(box ref pattern) = param.pat {
             visitor.check_irrefutable(pattern, "function argument", None);
         }
     }
+    visitor.error
 }
 
 fn create_e0004(
@@ -77,6 +80,7 @@ struct MatchVisitor<'a, 'p, 'tcx> {
     lint_level: HirId,
     let_source: LetSource,
     pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+    error: Result<(), ErrorGuaranteed>,
 }
 
 impl<'a, 'tcx> Visitor<'a, 'tcx> for MatchVisitor<'a, '_, 'tcx> {
@@ -276,9 +280,9 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
                 let [pat_field] = &subpatterns[..] else { bug!() };
                 self.check_irrefutable(&pat_field.pattern, "`for` loop binding", None);
             } else {
-                non_exhaustive_match(
+                self.error = Err(non_exhaustive_match(
                     &cx, self.thir, scrut_ty, scrut.span, witnesses, arms, expr_span,
-                );
+                ));
             }
         }
     }
@@ -406,7 +410,7 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn check_irrefutable(&self, pat: &Pat<'tcx>, origin: &str, sp: Option<Span>) {
+    fn check_irrefutable(&mut self, pat: &Pat<'tcx>, origin: &str, sp: Option<Span>) {
         let mut cx = self.new_cx(self.lint_level, false);
 
         let pattern = self.lower_pattern(&mut cx, pat);
@@ -475,7 +479,7 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             AdtDefinedHere { adt_def_span, ty, variants }
         };
 
-        self.tcx.sess.emit_err(PatternNotCovered {
+        self.error = Err(self.tcx.sess.emit_err(PatternNotCovered {
             span: pat.span,
             origin,
             uncovered: Uncovered::new(pat.span, &cx, witnesses),
@@ -486,7 +490,7 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             let_suggestion,
             misc_suggestion,
             adt_defined_here,
-        });
+        }));
     }
 }
 
@@ -628,7 +632,7 @@ fn non_exhaustive_match<'p, 'tcx>(
     witnesses: Vec<DeconstructedPat<'p, 'tcx>>,
     arms: &[ArmId],
     expr_span: Span,
-) {
+) -> ErrorGuaranteed {
     let is_empty_match = arms.is_empty();
     let non_empty_enum = match scrut_ty.kind() {
         ty::Adt(def, _) => def.is_enum() && !def.variants().is_empty(),
@@ -640,13 +644,12 @@ fn non_exhaustive_match<'p, 'tcx>(
     let pattern;
     let patterns_len;
     if is_empty_match && !non_empty_enum {
-        cx.tcx.sess.emit_err(NonExhaustivePatternsTypeNotEmpty {
+        return cx.tcx.sess.emit_err(NonExhaustivePatternsTypeNotEmpty {
             cx,
             expr_span,
             span: sp,
             ty: scrut_ty,
         });
-        return;
     } else {
         // FIXME: migration of this diagnostic will require list support
         let joined_patterns = joined_uncovered_patterns(cx, &witnesses);
@@ -797,7 +800,7 @@ fn non_exhaustive_match<'p, 'tcx>(
     } else {
         err.help(msg);
     }
-    err.emit();
+    err.emit()
 }
 
 pub(crate) fn joined_uncovered_patterns<'p, 'tcx>(

compiler/rustc_mir_dataflow/src/framework/lattice.rs

@@ -75,12 +75,12 @@ pub trait MeetSemiLattice: Eq {
 
 /// A set that has a "bottom" element, which is less than or equal to any other element.
 pub trait HasBottom {
-    fn bottom() -> Self;
+    const BOTTOM: Self;
 }
 
 /// A set that has a "top" element, which is greater than or equal to any other element.
 pub trait HasTop {
-    fn top() -> Self;
+    const TOP: Self;
 }
 
 /// A `bool` is a "two-point" lattice with `true` as the top element and `false` as the bottom:
@@ -113,15 +113,11 @@ impl MeetSemiLattice for bool {
 }
 
 impl HasBottom for bool {
-    fn bottom() -> Self {
+    const BOTTOM: Self = false;
-        false
-    }
 }
 
 impl HasTop for bool {
-    fn top() -> Self {
+    const TOP: Self = true;
-        true
-    }
 }
 
 /// A tuple (or list) of lattices is itself a lattice whose least upper bound is the concatenation
@@ -274,13 +270,9 @@ impl<T: Clone + Eq> MeetSemiLattice for FlatSet<T> {
 }
 
 impl<T> HasBottom for FlatSet<T> {
-    fn bottom() -> Self {
+    const BOTTOM: Self = Self::Bottom;
-        Self::Bottom
-    }
 }
 
 impl<T> HasTop for FlatSet<T> {
-    fn top() -> Self {
+    const TOP: Self = Self::Top;
-        Self::Top
-    }
 }

compiler/rustc_mir_dataflow/src/value_analysis.rs

@@ -32,9 +32,12 @@
 //! Because of that, we can assume that the only way to change the value behind a tracked place is
 //! by direct assignment.
 
+use std::collections::VecDeque;
 use std::fmt::{Debug, Formatter};
+use std::ops::Range;
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_index::bit_set::BitSet;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor};
@@ -65,8 +68,8 @@ pub trait ValueAnalysis<'tcx> {
             StatementKind::Assign(box (place, rvalue)) => {
                 self.handle_assign(*place, rvalue, state);
             }
-            StatementKind::SetDiscriminant { box ref place, .. } => {
+            StatementKind::SetDiscriminant { box place, variant_index } => {
-                state.flood_discr(place.as_ref(), self.map());
+                self.handle_set_discriminant(*place, *variant_index, state);
             }
             StatementKind::Intrinsic(box intrinsic) => {
                 self.handle_intrinsic(intrinsic, state);
@@ -74,11 +77,11 @@ pub trait ValueAnalysis<'tcx> {
             StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
                 // StorageLive leaves the local in an uninitialized state.
                 // StorageDead makes it UB to access the local afterwards.
-                state.flood_with(Place::from(*local).as_ref(), self.map(), Self::Value::bottom());
+                state.flood_with(Place::from(*local).as_ref(), self.map(), Self::Value::BOTTOM);
             }
             StatementKind::Deinit(box place) => {
                 // Deinit makes the place uninitialized.
-                state.flood_with(place.as_ref(), self.map(), Self::Value::bottom());
+                state.flood_with(place.as_ref(), self.map(), Self::Value::BOTTOM);
             }
             StatementKind::Retag(..) => {
                 // We don't track references.
@@ -92,6 +95,24 @@ pub trait ValueAnalysis<'tcx> {
         }
     }
 
+    fn handle_set_discriminant(
+        &self,
+        place: Place<'tcx>,
+        variant_index: VariantIdx,
+        state: &mut State<Self::Value>,
+    ) {
+        self.super_set_discriminant(place, variant_index, state)
+    }
+
+    fn super_set_discriminant(
+        &self,
+        place: Place<'tcx>,
+        _variant_index: VariantIdx,
+        state: &mut State<Self::Value>,
+    ) {
+        state.flood_discr(place.as_ref(), self.map());
+    }
+
     fn handle_intrinsic(
         &self,
         intrinsic: &NonDivergingIntrinsic<'tcx>,
@@ -103,16 +124,18 @@ pub trait ValueAnalysis<'tcx> {
     fn super_intrinsic(
         &self,
         intrinsic: &NonDivergingIntrinsic<'tcx>,
-        state: &mut State<Self::Value>,
+        _state: &mut State<Self::Value>,
     ) {
         match intrinsic {
             NonDivergingIntrinsic::Assume(..) => {
                 // Could use this, but ignoring it is sound.
             }
-            NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { dst, .. }) => {
+            NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
-                if let Some(place) = dst.place() {
+                dst: _,
-                    state.flood(place.as_ref(), self.map());
+                src: _,
-                }
+                count: _,
+            }) => {
+                // This statement represents `*dst = *src`, `count` times.
             }
         }
     }
@@ -154,7 +177,7 @@ pub trait ValueAnalysis<'tcx> {
             Rvalue::CopyForDeref(place) => self.handle_operand(&Operand::Copy(*place), state),
             Rvalue::Ref(..) | Rvalue::AddressOf(..) => {
                 // We don't track such places.
-                ValueOrPlace::top()
+                ValueOrPlace::TOP
             }
             Rvalue::Repeat(..)
             | Rvalue::ThreadLocalRef(..)
@@ -168,7 +191,7 @@ pub trait ValueAnalysis<'tcx> {
             | Rvalue::Aggregate(..)
             | Rvalue::ShallowInitBox(..) => {
                 // No modification is possible through these r-values.
-                ValueOrPlace::top()
+                ValueOrPlace::TOP
             }
         }
     }
@@ -196,7 +219,7 @@ pub trait ValueAnalysis<'tcx> {
                 self.map()
                     .find(place.as_ref())
                     .map(ValueOrPlace::Place)
-                    .unwrap_or(ValueOrPlace::top())
+                    .unwrap_or(ValueOrPlace::TOP)
             }
         }
     }
@@ -214,7 +237,7 @@ pub trait ValueAnalysis<'tcx> {
         _constant: &Constant<'tcx>,
         _state: &mut State<Self::Value>,
     ) -> Self::Value {
-        Self::Value::top()
+        Self::Value::TOP
     }
 
     /// The effect of a successful function call return should not be
@@ -229,7 +252,7 @@ pub trait ValueAnalysis<'tcx> {
                 // Effect is applied by `handle_call_return`.
             }
             TerminatorKind::Drop { place, .. } => {
-                state.flood_with(place.as_ref(), self.map(), Self::Value::bottom());
+                state.flood_with(place.as_ref(), self.map(), Self::Value::BOTTOM);
             }
             TerminatorKind::Yield { .. } => {
                 // They would have an effect, but are not allowed in this phase.
@@ -307,7 +330,7 @@ impl<'tcx, T: ValueAnalysis<'tcx>> AnalysisDomain<'tcx> for ValueAnalysisWrapper
     fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) {
         // The initial state maps all tracked places of argument projections to ⊤ and the rest to ⊥.
         assert!(matches!(state.0, StateData::Unreachable));
-        let values = IndexVec::from_elem_n(T::Value::bottom(), self.0.map().value_count);
+        let values = IndexVec::from_elem_n(T::Value::BOTTOM, self.0.map().value_count);
         *state = State(StateData::Reachable(values));
         for arg in body.args_iter() {
             state.flood(PlaceRef { local: arg, projection: &[] }, self.0.map());
@@ -437,7 +460,7 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
     }
 
     pub fn flood_all(&mut self) {
-        self.flood_all_with(V::top())
+        self.flood_all_with(V::TOP)
     }
 
     pub fn flood_all_with(&mut self, value: V) {
@@ -447,28 +470,24 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
 
     pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
         let StateData::Reachable(values) = &mut self.0 else { return };
-        map.for_each_aliasing_place(place, None, &mut |place| {
+        map.for_each_aliasing_place(place, None, &mut |vi| {
-            if let Some(vi) = map.places[place].value_index {
+            values[vi] = value.clone();
-                values[vi] = value.clone();
-            }
         });
     }
 
     pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) {
-        self.flood_with(place, map, V::top())
+        self.flood_with(place, map, V::TOP)
     }
 
     pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
         let StateData::Reachable(values) = &mut self.0 else { return };
-        map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |place| {
+        map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |vi| {
-            if let Some(vi) = map.places[place].value_index {
+            values[vi] = value.clone();
-                values[vi] = value.clone();
-            }
         });
     }
 
     pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) {
-        self.flood_discr_with(place, map, V::top())
+        self.flood_discr_with(place, map, V::TOP)
     }
 
     /// Low-level method that assigns to a place.
@@ -538,14 +557,14 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
 
     /// Retrieve the value stored for a place, or ⊤ if it is not tracked.
     pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V {
-        map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::top())
+        map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::TOP)
     }
 
     /// Retrieve the value stored for a place, or ⊤ if it is not tracked.
     pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V {
         match map.find_discr(place) {
             Some(place) => self.get_idx(place, map),
-            None => V::top(),
+            None => V::TOP,
         }
     }
 
@@ -553,11 +572,11 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
     pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V {
         match &self.0 {
             StateData::Reachable(values) => {
-                map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::top())
+                map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::TOP)
             }
             StateData::Unreachable => {
                 // Because this is unreachable, we can return any value we want.
-                V::bottom()
+                V::BOTTOM
             }
         }
     }
@@ -588,6 +607,9 @@ pub struct Map {
     projections: FxHashMap<(PlaceIndex, TrackElem), PlaceIndex>,
     places: IndexVec<PlaceIndex, PlaceInfo>,
     value_count: usize,
+    // The Range corresponds to a slice into `inner_values_buffer`.
+    inner_values: IndexVec<PlaceIndex, Range<usize>>,
+    inner_values_buffer: Vec<ValueIndex>,
 }
 
 impl Map {
@@ -597,6 +619,8 @@ impl Map {
             projections: FxHashMap::default(),
             places: IndexVec::new(),
             value_count: 0,
+            inner_values: IndexVec::new(),
+            inner_values_buffer: Vec::new(),
         }
     }
 
@@ -608,12 +632,12 @@ impl Map {
     pub fn from_filter<'tcx>(
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
-        filter: impl FnMut(Ty<'tcx>) -> bool,
+        filter: impl Fn(Ty<'tcx>) -> bool,
-        place_limit: Option<usize>,
+        value_limit: Option<usize>,
     ) -> Self {
         let mut map = Self::new();
         let exclude = excluded_locals(body);
-        map.register_with_filter(tcx, body, filter, exclude, place_limit);
+        map.register_with_filter(tcx, body, filter, exclude, value_limit);
         debug!("registered {} places ({} nodes in total)", map.value_count, map.places.len());
         map
     }
@@ -623,51 +647,90 @@ impl Map {
         &mut self,
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
-        mut filter: impl FnMut(Ty<'tcx>) -> bool,
+        filter: impl Fn(Ty<'tcx>) -> bool,
         exclude: BitSet<Local>,
-        place_limit: Option<usize>,
+        value_limit: Option<usize>,
     ) {
-        // We use this vector as stack, pushing and popping projections.
+        let mut worklist = VecDeque::with_capacity(value_limit.unwrap_or(body.local_decls.len()));
-        let mut projection = Vec::new();
+
+        // Start by constructing the places for each bare local.
+        self.locals = IndexVec::from_elem(None, &body.local_decls);
         for (local, decl) in body.local_decls.iter_enumerated() {
-            if !exclude.contains(local) {
+            if exclude.contains(local) {
-                self.register_with_filter_rec(
+                continue;
-                    tcx,
+            }
-                    local,
+
-                    &mut projection,
+            // Create a place for the local.
-                    decl.ty,
+            debug_assert!(self.locals[local].is_none());
-                    &mut filter,
+            let place = self.places.push(PlaceInfo::new(None));
-                    place_limit,
+            self.locals[local] = Some(place);
-                );
+
+            // And push the eventual children places to the worklist.
+            self.register_children(tcx, place, decl.ty, &filter, &mut worklist);
+        }
+
+        // `place.elem1.elem2` with type `ty`.
+        // `elem1` is either `Some(Variant(i))` or `None`.
+        while let Some((mut place, elem1, elem2, ty)) = worklist.pop_front() {
+            // The user requires a bound on the number of created values.
+            if let Some(value_limit) = value_limit && self.value_count >= value_limit {
+                break
+            }
+
+            // Create a place for this projection.
+            for elem in [elem1, Some(elem2)].into_iter().flatten() {
+                place = *self.projections.entry((place, elem)).or_insert_with(|| {
+                    // Prepend new child to the linked list.
+                    let next = self.places.push(PlaceInfo::new(Some(elem)));
+                    self.places[next].next_sibling = self.places[place].first_child;
+                    self.places[place].first_child = Some(next);
+                    next
+                });
+            }
+
+            // And push the eventual children places to the worklist.
+            self.register_children(tcx, place, ty, &filter, &mut worklist);
+        }
+
+        // Pre-compute the tree of ValueIndex nested in each PlaceIndex.
+        // `inner_values_buffer[inner_values[place]]` is the set of all the values
+        // reachable by projecting `place`.
+        self.inner_values_buffer = Vec::with_capacity(self.value_count);
+        self.inner_values = IndexVec::from_elem(0..0, &self.places);
+        for local in body.local_decls.indices() {
+            if let Some(place) = self.locals[local] {
+                self.cache_preorder_invoke(place);
             }
         }
+
+        // Trim useless places.
+        for opt_place in self.locals.iter_mut() {
+            if let Some(place) = *opt_place && self.inner_values[place].is_empty() {
+                *opt_place = None;
+            }
+        }
+        #[allow(rustc::potential_query_instability)]
+        self.projections.retain(|_, child| !self.inner_values[*child].is_empty());
     }
 
     /// Potentially register the (local, projection) place and its fields, recursively.
     ///
     /// Invariant: The projection must only contain trackable elements.
-    fn register_with_filter_rec<'tcx>(
+    fn register_children<'tcx>(
         &mut self,
         tcx: TyCtxt<'tcx>,
-        local: Local,
+        place: PlaceIndex,
-        projection: &mut Vec<PlaceElem<'tcx>>,
         ty: Ty<'tcx>,
-        filter: &mut impl FnMut(Ty<'tcx>) -> bool,
+        filter: &impl Fn(Ty<'tcx>) -> bool,
-        place_limit: Option<usize>,
+        worklist: &mut VecDeque<(PlaceIndex, Option<TrackElem>, TrackElem, Ty<'tcx>)>,
     ) {
-        if let Some(place_limit) = place_limit && self.value_count >= place_limit {
-            return
-        }
-
-        // We know that the projection only contains trackable elements.
-        let place = self.make_place(local, projection).unwrap();
-
         // Allocate a value slot if it doesn't have one, and the user requested one.
         if self.places[place].value_index.is_none() && filter(ty) {
             self.places[place].value_index = Some(self.value_count.into());
             self.value_count += 1;
         }
 
+        // For enums, directly create the `Discriminant`, as that's their main use.
         if ty.is_enum() {
             let discr_ty = ty.discriminant_ty(tcx);
             if filter(discr_ty) {
@@ -692,46 +755,32 @@ impl Map {
 
         // Recurse with all fields of this place.
         iter_fields(ty, tcx, ty::ParamEnv::reveal_all(), |variant, field, ty| {
-            if let Some(variant) = variant {
+            worklist.push_back((
-                projection.push(PlaceElem::Downcast(None, variant));
+                place,
-                let _ = self.make_place(local, projection);
+                variant.map(TrackElem::Variant),
-                projection.push(PlaceElem::Field(field, ty));
+                TrackElem::Field(field),
-                self.register_with_filter_rec(tcx, local, projection, ty, filter, place_limit);
+                ty,
-                projection.pop();
+            ))
-                projection.pop();
-                return;
-            }
-            projection.push(PlaceElem::Field(field, ty));
-            self.register_with_filter_rec(tcx, local, projection, ty, filter, place_limit);
-            projection.pop();
         });
     }
 
-    /// Tries to add the place to the map, without allocating a value slot.
+    /// Precompute the list of values inside `root` and store it inside
-    ///
+    /// as a slice within `inner_values_buffer`.
-    /// Can fail if the projection contains non-trackable elements.
+    fn cache_preorder_invoke(&mut self, root: PlaceIndex) {
-    fn make_place<'tcx>(
+        let start = self.inner_values_buffer.len();
-        &mut self,
+        if let Some(vi) = self.places[root].value_index {
-        local: Local,
+            self.inner_values_buffer.push(vi);
-        projection: &[PlaceElem<'tcx>],
-    ) -> Result<PlaceIndex, ()> {
-        // Get the base index of the local.
-        let mut index =
-            *self.locals.get_or_insert_with(local, || self.places.push(PlaceInfo::new(None)));
-
-        // Apply the projection.
-        for &elem in projection {
-            let elem = elem.try_into()?;
-            index = *self.projections.entry((index, elem)).or_insert_with(|| {
-                // Prepend new child to the linked list.
-                let next = self.places.push(PlaceInfo::new(Some(elem)));
-                self.places[next].next_sibling = self.places[index].first_child;
-                self.places[index].first_child = Some(next);
-                next
-            });
         }
 
-        Ok(index)
+        // We manually iterate instead of using `children` as we need to mutate `self`.
+        let mut next_child = self.places[root].first_child;
+        while let Some(child) = next_child {
+            ensure_sufficient_stack(|| self.cache_preorder_invoke(child));
+            next_child = self.places[child].next_sibling;
+        }
+
+        let end = self.inner_values_buffer.len();
+        self.inner_values[root] = start..end;
     }
 
     /// Returns the number of tracked places, i.e., those for which a value can be stored.
@@ -750,7 +799,7 @@ impl Map {
         place: PlaceRef<'_>,
         extra: impl IntoIterator<Item = TrackElem>,
     ) -> Option<PlaceIndex> {
-        let mut index = *self.locals.get(place.local)?.as_ref()?;
+        let mut index = *self.locals[place.local].as_ref()?;
 
         for &elem in place.projection {
             index = self.apply(index, elem.try_into().ok()?)?;
@@ -784,17 +833,17 @@ impl Map {
     ///
     /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track
     /// as such.
-    pub fn for_each_aliasing_place(
+    fn for_each_aliasing_place(
         &self,
         place: PlaceRef<'_>,
         tail_elem: Option<TrackElem>,
-        f: &mut impl FnMut(PlaceIndex),
+        f: &mut impl FnMut(ValueIndex),
     ) {
-        if place.is_indirect() {
+        if place.has_deref() {
             // We do not track indirect places.
             return;
         }
-        let Some(&Some(mut index)) = self.locals.get(place.local) else {
+        let Some(mut index) = self.locals[place.local] else {
             // The local is not tracked at all, so it does not alias anything.
             return;
         };
@@ -805,7 +854,9 @@ impl Map {
             .chain(tail_elem.map(Ok).into_iter());
         for elem in elems {
             // A field aliases the parent place.
-            f(index);
+            if let Some(vi) = self.places[index].value_index {
+                f(vi);
+            }
 
             let Ok(elem) = elem else { return };
             let sub = self.apply(index, elem);
@@ -819,7 +870,7 @@ impl Map {
                 return;
             }
         }
-        self.preorder_invoke(index, f);
+        self.for_each_value_inside(index, f);
     }
 
     /// Invoke the given function on all the descendants of the given place, except one branch.
@@ -827,7 +878,7 @@ impl Map {
         &self,
         parent: PlaceIndex,
         preserved_child: Option<PlaceIndex>,
-        f: &mut impl FnMut(PlaceIndex),
+        f: &mut impl FnMut(ValueIndex),
     ) {
         for sibling in self.children(parent) {
             let elem = self.places[sibling].proj_elem;
@@ -837,16 +888,17 @@ impl Map {
                 // Only invalidate the other variants, the current one is fine.
                 && Some(sibling) != preserved_child
             {
-                self.preorder_invoke(sibling, f);
+                self.for_each_value_inside(sibling, f);
             }
         }
     }
 
-    /// Invoke a function on the given place and all descendants.
+    /// Invoke a function on each value in the given place and all descendants.
-    fn preorder_invoke(&self, root: PlaceIndex, f: &mut impl FnMut(PlaceIndex)) {
+    fn for_each_value_inside(&self, root: PlaceIndex, f: &mut impl FnMut(ValueIndex)) {
-        f(root);
+        let range = self.inner_values[root].clone();
-        for child in self.children(root) {
+        let values = &self.inner_values_buffer[range];
-            self.preorder_invoke(child, f);
+        for &v in values {
+            f(v)
         }
     }
 }
@@ -909,9 +961,7 @@ pub enum ValueOrPlace<V> {
 }
 
 impl<V: HasTop> ValueOrPlace<V> {
-    pub fn top() -> Self {
+    pub const TOP: Self = ValueOrPlace::Value(V::TOP);
-        ValueOrPlace::Value(V::top())
-    }
 }
 
 /// The set of projection elements that can be used by a tracked place.

compiler/rustc_mir_transform/src/dataflow_const_prop.rs

@@ -79,22 +79,22 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
         &self.map
     }
 
-    fn handle_statement(&self, statement: &Statement<'tcx>, state: &mut State<Self::Value>) {
+    fn handle_set_discriminant(
-        match statement.kind {
+        &self,
-            StatementKind::SetDiscriminant { box ref place, variant_index } => {
+        place: Place<'tcx>,
-                state.flood_discr(place.as_ref(), &self.map);
+        variant_index: VariantIdx,
-                if self.map.find_discr(place.as_ref()).is_some() {
+        state: &mut State<Self::Value>,
-                    let enum_ty = place.ty(self.local_decls, self.tcx).ty;
+    ) {
-                    if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) {
+        state.flood_discr(place.as_ref(), &self.map);
-                        state.assign_discr(
+        if self.map.find_discr(place.as_ref()).is_some() {
-                            place.as_ref(),
+            let enum_ty = place.ty(self.local_decls, self.tcx).ty;
-                            ValueOrPlace::Value(FlatSet::Elem(discr)),
+            if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) {
-                            &self.map,
+                state.assign_discr(
-                        );
+                    place.as_ref(),
-                    }
+                    ValueOrPlace::Value(FlatSet::Elem(discr)),
-                }
+                    &self.map,
+                );
             }
-            _ => self.super_statement(statement, state),
         }
     }
 
@@ -208,8 +208,8 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
                     _ => unreachable!(),
                 }
                 .map(|result| ValueOrPlace::Value(self.wrap_immediate(result, *ty)))
-                .unwrap_or(ValueOrPlace::top()),
+                .unwrap_or(ValueOrPlace::TOP),
-                _ => ValueOrPlace::top(),
+                _ => ValueOrPlace::TOP,
             },
             Rvalue::BinaryOp(op, box (left, right)) => {
                 // Overflows must be ignored here.

compiler/rustc_mir_transform/src/ref_prop.rs

@@ -85,7 +85,9 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let ssa = SsaLocals::new(body);
 
     let mut replacer = compute_replacement(tcx, body, &ssa);
-    debug!(?replacer.targets, ?replacer.allowed_replacements, ?replacer.storage_to_remove);
+    debug!(?replacer.targets);
+    debug!(?replacer.allowed_replacements);
+    debug!(?replacer.storage_to_remove);
 
     replacer.visit_body_preserves_cfg(body);
 
@@ -190,8 +192,11 @@ fn compute_replacement<'tcx>(
             continue;
         }
 
+        // Whether the current local is subject to the uniqueness rule.
+        let needs_unique = ty.is_mutable_ptr();
+
         // If this a mutable reference that we cannot fully replace, mark it as unknown.
-        if ty.is_mutable_ptr() && !fully_replacable_locals.contains(local) {
+        if needs_unique && !fully_replacable_locals.contains(local) {
             debug!("not fully replaceable");
             continue;
         }
@@ -203,13 +208,14 @@ fn compute_replacement<'tcx>(
             // have been visited before.
             Rvalue::Use(Operand::Copy(place) | Operand::Move(place))
             | Rvalue::CopyForDeref(place) => {
-                if let Some(rhs) = place.as_local() {
+                if let Some(rhs) = place.as_local() && ssa.is_ssa(rhs) {
                     let target = targets[rhs];
-                    if matches!(target, Value::Pointer(..)) {
+                    // Only see through immutable reference and pointers, as we do not know yet if
+                    // mutable references are fully replaced.
+                    if !needs_unique && matches!(target, Value::Pointer(..)) {
                         targets[local] = target;
-                    } else if ssa.is_ssa(rhs) {
+                    } else {
-                        let refmut = body.local_decls[rhs].ty.is_mutable_ptr();
+                        targets[local] = Value::Pointer(tcx.mk_place_deref(rhs.into()), needs_unique);
-                        targets[local] = Value::Pointer(tcx.mk_place_deref(rhs.into()), refmut);
                     }
                 }
             }
@@ -217,10 +223,10 @@ fn compute_replacement<'tcx>(
                 let mut place = *place;
                 // Try to see through `place` in order to collapse reborrow chains.
                 if place.projection.first() == Some(&PlaceElem::Deref)
-                    && let Value::Pointer(target, refmut) = targets[place.local]
+                    && let Value::Pointer(target, inner_needs_unique) = targets[place.local]
                     // Only see through immutable reference and pointers, as we do not know yet if
                     // mutable references are fully replaced.
-                    && !refmut
+                    && !inner_needs_unique
                     // Only collapse chain if the pointee is definitely live.
                     && can_perform_opt(target, location)
                 {
@@ -228,7 +234,7 @@ fn compute_replacement<'tcx>(
                 }
                 assert_ne!(place.local, local);
                 if is_constant_place(place) {
-                    targets[local] = Value::Pointer(place, ty.is_mutable_ptr());
+                    targets[local] = Value::Pointer(place, needs_unique);
                 }
             }
             // We do not know what to do, so keep as not-a-pointer.
@@ -276,16 +282,35 @@ fn compute_replacement<'tcx>(
                 return;
             }
 
-            if let Value::Pointer(target, refmut) = self.targets[place.local]
+            if place.projection.first() != Some(&PlaceElem::Deref) {
-                && place.projection.first() == Some(&PlaceElem::Deref)
+                // This is not a dereference, nothing to do.
-            {
+                return;
-                let perform_opt = (self.can_perform_opt)(target, loc);
+            }
-                if perform_opt {
+
-                    self.allowed_replacements.insert((target.local, loc));
+            let mut place = place.as_ref();
-                } else if refmut {
+            loop {
-                    // This mutable reference is not fully replacable, so drop it.
+                if let Value::Pointer(target, needs_unique) = self.targets[place.local] {
-                    self.targets[place.local] = Value::Unknown;
+                    let perform_opt = (self.can_perform_opt)(target, loc);
+                    debug!(?place, ?target, ?needs_unique, ?perform_opt);
+
+                    // This a reborrow chain, recursively allow the replacement.
+                    //
+                    // This also allows to detect cases where `target.local` is not replacable,
+                    // and mark it as such.
+                    if let &[PlaceElem::Deref] = &target.projection[..] {
+                        assert!(perform_opt);
+                        self.allowed_replacements.insert((target.local, loc));
+                        place.local = target.local;
+                        continue;
+                    } else if perform_opt {
+                        self.allowed_replacements.insert((target.local, loc));
+                    } else if needs_unique {
+                        // This mutable reference is not fully replacable, so drop it.
+                        self.targets[place.local] = Value::Unknown;
+                    }
                 }
+
+                break;
             }
         }
     }
@@ -326,18 +351,23 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> {
     }
 
     fn visit_place(&mut self, place: &mut Place<'tcx>, ctxt: PlaceContext, loc: Location) {
-        if let Value::Pointer(target, _) = self.targets[place.local]
+        if place.projection.first() != Some(&PlaceElem::Deref) {
-            && place.projection.first() == Some(&PlaceElem::Deref)
+            return;
-        {
+        }
-            let perform_opt = matches!(ctxt, PlaceContext::NonUse(_))
-                || self.allowed_replacements.contains(&(target.local, loc));
 
-            if perform_opt {
+        loop {
-                *place = target.project_deeper(&place.projection[1..], self.tcx);
+            if let Value::Pointer(target, _) = self.targets[place.local] {
-                self.any_replacement = true;
+                let perform_opt = matches!(ctxt, PlaceContext::NonUse(_))
+                    || self.allowed_replacements.contains(&(target.local, loc));
+
+                if perform_opt {
+                    *place = target.project_deeper(&place.projection[1..], self.tcx);
+                    self.any_replacement = true;
+                    continue;
+                }
             }
-        } else {
+
-            self.super_place(place, ctxt, loc);
+            break;
         }
     }
 

compiler/rustc_mir_transform/src/ssa.rs

@@ -209,13 +209,6 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor {
         match ctxt {
             PlaceContext::MutatingUse(MutatingUseContext::Projection)
             | PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) => bug!(),
-            PlaceContext::MutatingUse(MutatingUseContext::Store) => {
-                self.assignments[local].insert(LocationExtended::Plain(loc));
-                if let Set1::One(_) = self.assignments[local] {
-                    // Only record if SSA-like, to avoid growing the vector needlessly.
-                    self.assignment_order.push(local);
-                }
-            }
             // Anything can happen with raw pointers, so remove them.
             // We do not verify that all uses of the borrow dominate the assignment to `local`,
             // so we have to remove them too.
@@ -252,6 +245,19 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor {
             self.visit_local(place.local, ctxt, loc);
         }
     }
+
+    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, loc: Location) {
+        if let Some(local) = place.as_local() {
+            self.assignments[local].insert(LocationExtended::Plain(loc));
+            if let Set1::One(_) = self.assignments[local] {
+                // Only record if SSA-like, to avoid growing the vector needlessly.
+                self.assignment_order.push(local);
+            }
+        } else {
+            self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), loc);
+        }
+        self.visit_rvalue(rvalue, loc);
+    }
 }
 
 #[instrument(level = "trace", skip(ssa, body))]

compiler/rustc_resolve/messages.ftl

@@ -199,6 +199,10 @@ resolve_invalid_asm_sym =
     .label = is a local variable
     .help = `sym` operands must refer to either a function or a static
 
+resolve_lowercase_self =
+    attempt to use a non-constant value in a constant
+    .suggestion = try using `Self`
+
 resolve_trait_impl_duplicate =
     duplicate definitions with name `{$name}`:
     .label = duplicate definition

compiler/rustc_resolve/src/diagnostics.rs

@@ -948,6 +948,9 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             ResolutionError::InvalidAsmSym => {
                 self.tcx.sess.create_err(errs::InvalidAsmSym { span })
             }
+            ResolutionError::LowercaseSelf => {
+                self.tcx.sess.create_err(errs::LowercaseSelf { span })
+            }
         }
     }
 

compiler/rustc_resolve/src/errors.rs

@@ -442,6 +442,14 @@ pub(crate) struct InvalidAsmSym {
     pub(crate) span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(resolve_lowercase_self)]
+pub(crate) struct LowercaseSelf {
+    #[primary_span]
+    #[suggestion(code = "Self", applicability = "maybe-incorrect", style = "short")]
+    pub(crate) span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(resolve_trait_impl_duplicate, code = "E0201")]
 pub(crate) struct TraitImplDuplicate {

compiler/rustc_resolve/src/ident.rs

@@ -15,8 +15,7 @@ use std::ptr;
 
 use crate::errors::{ParamKindInEnumDiscriminant, ParamKindInNonTrivialAnonConst};
 use crate::late::{
-    ConstantHasGenerics, ConstantItemKind, HasGenericParams, NoConstantGenericsReason, PathSource,
+    ConstantHasGenerics, HasGenericParams, NoConstantGenericsReason, PathSource, Rib, RibKind,
-    Rib, RibKind,
 };
 use crate::macros::{sub_namespace_match, MacroRulesScope};
 use crate::{errors, AmbiguityError, AmbiguityErrorMisc, AmbiguityKind, Determinacy, Finalize};
@@ -1127,28 +1126,25 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                         RibKind::ConstantItem(_, item) => {
                             // Still doesn't deal with upvars
                             if let Some(span) = finalize {
-                                let (span, resolution_error) =
+                                let (span, resolution_error) = match item {
-                                    if let Some((ident, constant_item_kind)) = item {
+                                    None if rib_ident.as_str() == "self" => (span, LowercaseSelf),
-                                        let kind_str = match constant_item_kind {
+                                    None => (
-                                            ConstantItemKind::Const => "const",
+                                        rib_ident.span,
-                                            ConstantItemKind::Static => "static",
+                                        AttemptToUseNonConstantValueInConstant(
-                                        };
+                                            original_rib_ident_def,
-                                        (
+                                            "const",
-                                            span,
+                                            "let",
-                                            AttemptToUseNonConstantValueInConstant(
+                                        ),
-                                                ident, "let", kind_str,
+                                    ),
-                                            ),
+                                    Some((ident, kind)) => (
-                                        )
+                                        span,
-                                    } else {
+                                        AttemptToUseNonConstantValueInConstant(
-                                        (
+                                            ident,
-                                            rib_ident.span,
+                                            "let",
-                                            AttemptToUseNonConstantValueInConstant(
+                                            kind.as_str(),
-                                                original_rib_ident_def,
+                                        ),
-                                                "const",
+                                    ),
-                                                "let",
+                                };
-                                            ),
-                                        )
-                                    };
                                 self.report_error(span, resolution_error);
                             }
                             return Res::Err;

compiler/rustc_resolve/src/late.rs

@@ -150,6 +150,15 @@ pub(crate) enum ConstantItemKind {
     Static,
 }
 
+impl ConstantItemKind {
+    pub(crate) fn as_str(&self) -> &'static str {
+        match self {
+            Self::Const => "const",
+            Self::Static => "static",
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 enum RecordPartialRes {
     Yes,
@@ -1482,7 +1491,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             if let Some(&(_, res)) = rib.bindings.get(&normalized_ident) {
                 self.record_lifetime_res(lifetime.id, res, LifetimeElisionCandidate::Named);
 
-                if let LifetimeRes::Param { param, .. } = res {
+                if let LifetimeRes::Param { param, binder } = res {
                     match self.lifetime_uses.entry(param) {
                         Entry::Vacant(v) => {
                             debug!("First use of {:?} at {:?}", res, ident.span);
@@ -1496,10 +1505,16 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                                     LifetimeRibKind::Item
                                     | LifetimeRibKind::AnonymousReportError
                                     | LifetimeRibKind::ElisionFailure => Some(LifetimeUseSet::Many),
-                                    // An anonymous lifetime is legal here, go ahead.
+                                    // An anonymous lifetime is legal here, and bound to the right
-                                    LifetimeRibKind::AnonymousCreateParameter { .. } => {
+                                    // place, go ahead.
-                                        Some(LifetimeUseSet::One { use_span: ident.span, use_ctxt })
+                                    LifetimeRibKind::AnonymousCreateParameter {
-                                    }
+                                        binder: anon_binder,
+                                        ..
+                                    } => Some(if binder == anon_binder {
+                                        LifetimeUseSet::One { use_span: ident.span, use_ctxt }
+                                    } else {
+                                        LifetimeUseSet::Many
+                                    }),
                                     // Only report if eliding the lifetime would have the same
                                     // semantics.
                                     LifetimeRibKind::Elided(r) => Some(if res == r {

compiler/rustc_resolve/src/lib.rs

@@ -251,6 +251,8 @@ enum ResolutionError<'a> {
     TraitImplDuplicate { name: Symbol, trait_item_span: Span, old_span: Span },
     /// Inline asm `sym` operand must refer to a `fn` or `static`.
     InvalidAsmSym,
+    /// `self` used instead of `Self` in a generic parameter
+    LowercaseSelf,
 }
 
 enum VisResolutionError<'a> {

compiler/rustc_session/Cargo.toml

@@ -25,7 +25,7 @@ termize = "0.1.1"
 libc = "0.2"
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.46.0"
+version = "0.48.0"
 features = [
     "Win32_Foundation",
     "Win32_System_LibraryLoader",

compiler/rustc_session/src/filesearch.rs

@@ -135,13 +135,13 @@ fn current_dll_path() -> Result<PathBuf, String> {
 
     use windows::{
         core::PCWSTR,
-        Win32::Foundation::HINSTANCE,
+        Win32::Foundation::HMODULE,
         Win32::System::LibraryLoader::{
             GetModuleFileNameW, GetModuleHandleExW, GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
         },
     };
 
-    let mut module = HINSTANCE::default();
+    let mut module = HMODULE::default();
     unsafe {
         GetModuleHandleExW(
             GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,

compiler/rustc_session/src/options.rs

@@ -1720,6 +1720,10 @@ options! {
     #[rustc_lint_opt_deny_field_access("use `Session::stack_protector` instead of this field")]
     stack_protector: StackProtector = (StackProtector::None, parse_stack_protector, [TRACKED],
         "control stack smash protection strategy (`rustc --print stack-protector-strategies` for details)"),
+    staticlib_allow_rdylib_deps: bool = (false, parse_bool, [TRACKED],
+        "allow staticlibs to have rust dylib dependencies"),
+    staticlib_prefer_dynamic: bool = (false, parse_bool, [TRACKED],
+        "prefer dynamic linking to static linking for staticlibs (default: no)"),
     strict_init_checks: bool = (false, parse_bool, [TRACKED],
         "control if mem::uninitialized and mem::zeroed panic on more UB"),
     strip: Strip = (Strip::None, parse_strip, [UNTRACKED],

compiler/rustc_smir/src/lib.rs

@@ -11,6 +11,8 @@
     test(attr(allow(unused_variables), deny(warnings)))
 )]
 #![cfg_attr(not(feature = "default"), feature(rustc_private))]
+#![feature(local_key_cell_methods)]
+#![feature(ptr_metadata)]
 
 pub mod rustc_internal;
 pub mod stable_mir;

compiler/rustc_smir/src/rustc_internal/mod.rs

@@ -3,30 +3,49 @@
 //! For that, we define APIs that will temporarily be public to 3P that exposes rustc internal APIs
 //! until stable MIR is complete.
 
-use std::sync::RwLock;
+use crate::{
-
+    rustc_smir::Tables,
-use crate::stable_mir;
+    stable_mir::{self, with},
+};
+use rustc_middle::ty::TyCtxt;
 pub use rustc_span::def_id::{CrateNum, DefId};
 
-static DEF_ID_MAP: RwLock<Vec<DefId>> = RwLock::new(Vec::new());
+fn with_tables<R>(mut f: impl FnMut(&mut Tables<'_>) -> R) -> R {
+    let mut ret = None;
+    with(|tables| tables.rustc_tables(&mut |t| ret = Some(f(t))));
+    ret.unwrap()
+}
 
 pub fn item_def_id(item: &stable_mir::CrateItem) -> DefId {
-    DEF_ID_MAP.read().unwrap()[item.0]
+    with_tables(|t| t.item_def_id(item))
 }
 
 pub fn crate_item(did: DefId) -> stable_mir::CrateItem {
-    // FIXME: this becomes inefficient when we have too many ids
+    with_tables(|t| t.crate_item(did))
-    let mut map = DEF_ID_MAP.write().unwrap();
+}
-    for (i, &d) in map.iter().enumerate() {
+
-        if d == did {
+impl<'tcx> Tables<'tcx> {
-            return stable_mir::CrateItem(i);
+    pub fn item_def_id(&self, item: &stable_mir::CrateItem) -> DefId {
-        }
+        self.def_ids[item.0]
+    }
+
+    pub fn crate_item(&mut self, did: DefId) -> stable_mir::CrateItem {
+        // FIXME: this becomes inefficient when we have too many ids
+        for (i, &d) in self.def_ids.iter().enumerate() {
+            if d == did {
+                return stable_mir::CrateItem(i);
+            }
+        }
+        let id = self.def_ids.len();
+        self.def_ids.push(did);
+        stable_mir::CrateItem(id)
     }
-    let id = map.len();
-    map.push(did);
-    stable_mir::CrateItem(id)
 }
 
 pub fn crate_num(item: &stable_mir::Crate) -> CrateNum {
     item.id.into()
 }
+
+pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) {
+    crate::stable_mir::run(Tables { tcx, def_ids: vec![], types: vec![] }, f);
+}

compiler/rustc_smir/src/rustc_smir/mod.rs

@@ -7,55 +7,36 @@
 //!
 //! For now, we are developing everything inside `rustc`, thus, we keep this module private.
 
-use crate::{
+use crate::stable_mir::{self, ty::TyKind, Context};
-    rustc_internal::{crate_item, item_def_id},
+use rustc_middle::ty::{self, Ty, TyCtxt};
-    stable_mir::{self},
+use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
-};
-use rustc_middle::ty::{tls::with, TyCtxt};
-use rustc_span::def_id::{CrateNum, LOCAL_CRATE};
 use tracing::debug;
 
-/// Get information about the local crate.
+impl<'tcx> Context for Tables<'tcx> {
-pub fn local_crate() -> stable_mir::Crate {
+    fn local_crate(&self) -> stable_mir::Crate {
-    with(|tcx| smir_crate(tcx, LOCAL_CRATE))
+        smir_crate(self.tcx, LOCAL_CRATE)
-}
+    }
 
-/// Retrieve a list of all external crates.
+    fn external_crates(&self) -> Vec<stable_mir::Crate> {
-pub fn external_crates() -> Vec<stable_mir::Crate> {
+        self.tcx.crates(()).iter().map(|crate_num| smir_crate(self.tcx, *crate_num)).collect()
-    with(|tcx| tcx.crates(()).iter().map(|crate_num| smir_crate(tcx, *crate_num)).collect())
+    }
-}
 
-/// Find a crate with the given name.
+    fn find_crate(&self, name: &str) -> Option<stable_mir::Crate> {
-pub fn find_crate(name: &str) -> Option<stable_mir::Crate> {
+        [LOCAL_CRATE].iter().chain(self.tcx.crates(()).iter()).find_map(|crate_num| {
-    with(|tcx| {
+            let crate_name = self.tcx.crate_name(*crate_num).to_string();
-        [LOCAL_CRATE].iter().chain(tcx.crates(()).iter()).find_map(|crate_num| {
+            (name == crate_name).then(|| smir_crate(self.tcx, *crate_num))
-            let crate_name = tcx.crate_name(*crate_num).to_string();
-            (name == crate_name).then(|| smir_crate(tcx, *crate_num))
         })
-    })
+    }
-}
 
-/// Retrieve all items of the local crate that have a MIR associated with them.
+    fn all_local_items(&mut self) -> stable_mir::CrateItems {
-pub fn all_local_items() -> stable_mir::CrateItems {
+        self.tcx.mir_keys(()).iter().map(|item| self.crate_item(item.to_def_id())).collect()
-    with(|tcx| tcx.mir_keys(()).iter().map(|item| crate_item(item.to_def_id())).collect())
+    }
-}
+    fn entry_fn(&mut self) -> Option<stable_mir::CrateItem> {
-
+        Some(self.crate_item(self.tcx.entry_fn(())?.0))
-pub fn entry_fn() -> Option<stable_mir::CrateItem> {
+    }
-    with(|tcx| Some(crate_item(tcx.entry_fn(())?.0)))
+    fn mir_body(&mut self, item: &stable_mir::CrateItem) -> stable_mir::mir::Body {
-}
+        let def_id = self.item_def_id(item);
-
+        let mir = self.tcx.optimized_mir(def_id);
-/// Build a stable mir crate from a given crate number.
-fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
-    let crate_name = tcx.crate_name(crate_num).to_string();
-    let is_local = crate_num == LOCAL_CRATE;
-    debug!(?crate_name, ?crate_num, "smir_crate");
-    stable_mir::Crate { id: crate_num.into(), name: crate_name, is_local }
-}
-
-pub fn mir_body(item: &stable_mir::CrateItem) -> stable_mir::mir::Body {
-    with(|tcx| {
-        let def_id = item_def_id(item);
-        let mir = tcx.optimized_mir(def_id);
         stable_mir::mir::Body {
             blocks: mir
                 .basic_blocks
@@ -65,8 +46,76 @@ pub fn mir_body(item: &stable_mir::CrateItem) -> stable_mir::mir::Body {
                     statements: block.statements.iter().map(rustc_statement_to_statement).collect(),
                 })
                 .collect(),
+            locals: mir.local_decls.iter().map(|decl| self.intern_ty(decl.ty)).collect(),
         }
-    })
+    }
+
+    fn rustc_tables(&mut self, f: &mut dyn FnMut(&mut Tables<'_>)) {
+        f(self)
+    }
+
+    fn ty_kind(&mut self, ty: crate::stable_mir::ty::Ty) -> TyKind {
+        self.rustc_ty_to_ty(self.types[ty.0])
+    }
+}
+
+pub struct Tables<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    pub def_ids: Vec<DefId>,
+    pub types: Vec<Ty<'tcx>>,
+}
+
+impl<'tcx> Tables<'tcx> {
+    fn rustc_ty_to_ty(&mut self, ty: Ty<'tcx>) -> TyKind {
+        match ty.kind() {
+            ty::Bool => TyKind::Bool,
+            ty::Char => todo!(),
+            ty::Int(_) => todo!(),
+            ty::Uint(_) => todo!(),
+            ty::Float(_) => todo!(),
+            ty::Adt(_, _) => todo!(),
+            ty::Foreign(_) => todo!(),
+            ty::Str => todo!(),
+            ty::Array(_, _) => todo!(),
+            ty::Slice(_) => todo!(),
+            ty::RawPtr(_) => todo!(),
+            ty::Ref(_, _, _) => todo!(),
+            ty::FnDef(_, _) => todo!(),
+            ty::FnPtr(_) => todo!(),
+            ty::Placeholder(..) => todo!(),
+            ty::Dynamic(_, _, _) => todo!(),
+            ty::Closure(_, _) => todo!(),
+            ty::Generator(_, _, _) => todo!(),
+            ty::GeneratorWitness(_) => todo!(),
+            ty::GeneratorWitnessMIR(_, _) => todo!(),
+            ty::Never => todo!(),
+            ty::Tuple(fields) => {
+                TyKind::Tuple(fields.iter().map(|ty| self.intern_ty(ty)).collect())
+            }
+            ty::Alias(_, _) => todo!(),
+            ty::Param(_) => todo!(),
+            ty::Bound(_, _) => todo!(),
+            ty::Infer(_) => todo!(),
+            ty::Error(_) => todo!(),
+        }
+    }
+
+    fn intern_ty(&mut self, ty: Ty<'tcx>) -> stable_mir::ty::Ty {
+        if let Some(id) = self.types.iter().position(|&t| t == ty) {
+            return stable_mir::ty::Ty(id);
+        }
+        let id = self.types.len();
+        self.types.push(ty);
+        stable_mir::ty::Ty(id)
+    }
+}
+
+/// Build a stable mir crate from a given crate number.
+fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
+    let crate_name = tcx.crate_name(crate_num).to_string();
+    let is_local = crate_num == LOCAL_CRATE;
+    debug!(?crate_name, ?crate_num, "smir_crate");
+    stable_mir::Crate { id: crate_num.into(), name: crate_name, is_local }
 }
 
 fn rustc_statement_to_statement(

compiler/rustc_smir/src/stable_mir/mir/body.rs

@@ -1,6 +1,9 @@
+use crate::stable_mir::ty::Ty;
+
 #[derive(Clone, Debug)]
 pub struct Body {
     pub blocks: Vec<BasicBlock>,
+    pub locals: Vec<Ty>,
 }
 
 #[derive(Clone, Debug)]

compiler/rustc_smir/src/stable_mir/mod.rs

@@ -11,7 +11,14 @@
 //! There shouldn't be any direct references to internal compiler constructs in this module.
 //! If you need an internal construct, consider using `rustc_internal` or `rustc_smir`.
 
+use std::cell::Cell;
+
+use crate::rustc_smir::Tables;
+
+use self::ty::{Ty, TyKind};
+
 pub mod mir;
+pub mod ty;
 
 /// Use String for now but we should replace it.
 pub type Symbol = String;
@@ -41,7 +48,7 @@ pub struct CrateItem(pub(crate) DefId);
 
 impl CrateItem {
     pub fn body(&self) -> mir::Body {
-        crate::rustc_smir::mir_body(self)
+        with(|cx| cx.mir_body(self))
     }
 }
 
@@ -49,25 +56,72 @@ impl CrateItem {
 /// crate defines that. This is usually `main`, but could be
 /// `start` if the crate is a no-std crate.
 pub fn entry_fn() -> Option<CrateItem> {
-    crate::rustc_smir::entry_fn()
+    with(|cx| cx.entry_fn())
 }
 
 /// Access to the local crate.
 pub fn local_crate() -> Crate {
-    crate::rustc_smir::local_crate()
+    with(|cx| cx.local_crate())
 }
 
 /// Try to find a crate with the given name.
 pub fn find_crate(name: &str) -> Option<Crate> {
-    crate::rustc_smir::find_crate(name)
+    with(|cx| cx.find_crate(name))
 }
 
 /// Try to find a crate with the given name.
 pub fn external_crates() -> Vec<Crate> {
-    crate::rustc_smir::external_crates()
+    with(|cx| cx.external_crates())
 }
 
 /// Retrieve all items in the local crate that have a MIR associated with them.
 pub fn all_local_items() -> CrateItems {
-    crate::rustc_smir::all_local_items()
+    with(|cx| cx.all_local_items())
+}
+
+pub trait Context {
+    fn entry_fn(&mut self) -> Option<CrateItem>;
+    /// Retrieve all items of the local crate that have a MIR associated with them.
+    fn all_local_items(&mut self) -> CrateItems;
+    fn mir_body(&mut self, item: &CrateItem) -> mir::Body;
+    /// Get information about the local crate.
+    fn local_crate(&self) -> Crate;
+    /// Retrieve a list of all external crates.
+    fn external_crates(&self) -> Vec<Crate>;
+
+    /// Find a crate with the given name.
+    fn find_crate(&self, name: &str) -> Option<Crate>;
+
+    /// Obtain the representation of a type.
+    fn ty_kind(&mut self, ty: Ty) -> TyKind;
+
+    /// HACK: Until we have fully stable consumers, we need an escape hatch
+    /// to get `DefId`s out of `CrateItem`s.
+    fn rustc_tables(&mut self, f: &mut dyn FnMut(&mut Tables<'_>));
+}
+
+thread_local! {
+    /// A thread local variable that stores a pointer to the tables mapping between TyCtxt
+    /// datastructures and stable MIR datastructures.
+    static TLV: Cell<*mut ()> = const { Cell::new(std::ptr::null_mut()) };
+}
+
+pub fn run(mut context: impl Context, f: impl FnOnce()) {
+    assert!(TLV.get().is_null());
+    fn g<'a>(mut context: &mut (dyn Context + 'a), f: impl FnOnce()) {
+        TLV.set(&mut context as *mut &mut _ as _);
+        f();
+        TLV.replace(std::ptr::null_mut());
+    }
+    g(&mut context, f);
+}
+
+/// Loads the current context and calls a function with it.
+/// Do not nest these, as that will ICE.
+pub(crate) fn with<R>(f: impl FnOnce(&mut dyn Context) -> R) -> R {
+    let ptr = TLV.replace(std::ptr::null_mut()) as *mut &mut dyn Context;
+    assert!(!ptr.is_null());
+    let ret = f(unsafe { *ptr });
+    TLV.set(ptr as _);
+    ret
 }

compiler/rustc_smir/src/stable_mir/ty.rs

@@ -0,0 +1,15 @@
+use super::with;
+
+#[derive(Copy, Clone, Debug)]
+pub struct Ty(pub usize);
+
+impl Ty {
+    pub fn kind(&self) -> TyKind {
+        with(|context| context.ty_kind(*self))
+    }
+}
+
+pub enum TyKind {
+    Bool,
+    Tuple(Vec<Ty>),
+}

compiler/rustc_symbol_mangling/src/typeid.rs

@@ -4,7 +4,7 @@
 /// For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler,
 /// see design document in the tracking issue #89653.
 use bitflags::bitflags;
-use rustc_middle::ty::{FnSig, Ty, TyCtxt};
+use rustc_middle::ty::{FnSig, Instance, Ty, TyCtxt};
 use rustc_target::abi::call::FnAbi;
 use std::hash::Hasher;
 use twox_hash::XxHash64;
@@ -38,6 +38,15 @@ pub fn typeid_for_fnsig<'tcx>(
     typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, options)
 }
 
+/// Returns a type metadata identifier for the specified Instance.
+pub fn typeid_for_instance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: &Instance<'tcx>,
+    options: TypeIdOptions,
+) -> String {
+    typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options)
+}
+
 /// Returns a KCFI type metadata identifier for the specified FnAbi.
 pub fn kcfi_typeid_for_fnabi<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -63,3 +72,16 @@ pub fn kcfi_typeid_for_fnsig<'tcx>(
     hash.write(typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, options).as_bytes());
     hash.finish() as u32
 }
+
+/// Returns a KCFI type metadata identifier for the specified Instance.
+pub fn kcfi_typeid_for_instance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: &Instance<'tcx>,
+    options: TypeIdOptions,
+) -> u32 {
+    // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the
+    // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.)
+    let mut hash: XxHash64 = Default::default();
+    hash.write(typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options).as_bytes());
+    hash.finish() as u32
+}

compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs

@@ -14,8 +14,8 @@ use rustc_errors::DiagnosticMessage;
 use rustc_hir as hir;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind, SubstsRef};
 use rustc_middle::ty::{
-    self, Const, ExistentialPredicate, FloatTy, FnSig, IntTy, List, Region, RegionKind, TermKind,
+    self, Const, ExistentialPredicate, FloatTy, FnSig, Instance, IntTy, List, Region, RegionKind,
-    Ty, TyCtxt, UintTy,
+    TermKind, Ty, TyCtxt, UintTy,
 };
 use rustc_span::def_id::DefId;
 use rustc_span::sym;
@@ -1010,3 +1010,56 @@ pub fn typeid_for_fnsig<'tcx>(
 
     typeid
 }
+
+/// Returns a type metadata identifier for the specified Instance using the Itanium C++ ABI with
+/// vendor extended type qualifiers and types for Rust types that are not used at the FFI boundary.
+pub fn typeid_for_instance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: &Instance<'tcx>,
+    options: TypeIdOptions,
+) -> String {
+    let fn_abi = tcx
+        .fn_abi_of_instance(tcx.param_env(instance.def_id()).and((*instance, ty::List::empty())))
+        .unwrap_or_else(|instance| {
+            bug!("typeid_for_instance: couldn't get fn_abi of instance {:?}", instance)
+        });
+
+    // If this instance is a method and self is a reference, get the impl it belongs to
+    let impl_def_id = tcx.impl_of_method(instance.def_id());
+    if impl_def_id.is_some() && !fn_abi.args.is_empty() && fn_abi.args[0].layout.ty.is_ref() {
+        // If this impl is not an inherent impl, get the trait it implements
+        if let Some(trait_ref) = tcx.impl_trait_ref(impl_def_id.unwrap()) {
+            // Transform the concrete self into a reference to a trait object
+            let existential_predicate = trait_ref.map_bound(|trait_ref| {
+                ty::ExistentialPredicate::Trait(ty::ExistentialTraitRef::erase_self_ty(
+                    tcx, trait_ref,
+                ))
+            });
+            let existential_predicates = tcx.mk_poly_existential_predicates(&[ty::Binder::dummy(
+                existential_predicate.skip_binder(),
+            )]);
+            // Is the concrete self mutable?
+            let self_ty = if fn_abi.args[0].layout.ty.is_mutable_ptr() {
+                tcx.mk_mut_ref(
+                    tcx.lifetimes.re_erased,
+                    tcx.mk_dynamic(existential_predicates, tcx.lifetimes.re_erased, ty::Dyn),
+                )
+            } else {
+                tcx.mk_imm_ref(
+                    tcx.lifetimes.re_erased,
+                    tcx.mk_dynamic(existential_predicates, tcx.lifetimes.re_erased, ty::Dyn),
+                )
+            };
+
+            // Replace the concrete self in an fn_abi clone by the reference to a trait object
+            let mut fn_abi = fn_abi.clone();
+            // HACK(rcvalle): It is okay to not replace or update the entire ArgAbi here because the
+            //   other fields are never used.
+            fn_abi.args[0].layout.ty = self_ty;
+
+            return typeid_for_fnabi(tcx, &fn_abi, options);
+        }
+    }
+
+    typeid_for_fnabi(tcx, &fn_abi, options)
+}

compiler/rustc_target/src/abi/call/mod.rs

@@ -28,7 +28,7 @@ mod x86;
 mod x86_64;
 mod x86_win64;
 
-#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
 pub enum PassMode {
     /// Ignore the argument.
     ///
@@ -211,7 +211,7 @@ impl Uniform {
     }
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
 pub struct CastTarget {
     pub prefix: [Option<Reg>; 8],
     pub rest: Uniform,
@@ -458,7 +458,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
 
 /// Information about how to pass an argument to,
 /// or return a value from, a function, under some ABI.
-#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
 pub struct ArgAbi<'a, Ty> {
     pub layout: TyAndLayout<'a, Ty>,
     pub mode: PassMode,
@@ -605,7 +605,7 @@ pub enum Conv {
 ///
 /// I will do my best to describe this structure, but these
 /// comments are reverse-engineered and may be inaccurate. -NDM
-#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
 pub struct FnAbi<'a, Ty> {
     /// The LLVM types of each argument.
     pub args: Box<[ArgAbi<'a, Ty>]>,

compiler/rustc_trait_selection/src/solve/assembly/mod.rs

@@ -8,6 +8,7 @@ use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir::def_id::DefId;
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::util::elaborate;
+use rustc_infer::traits::Reveal;
 use rustc_middle::traits::solve::{CanonicalResponse, Certainty, Goal, MaybeCause, QueryResult};
 use rustc_middle::ty::fast_reject::TreatProjections;
 use rustc_middle::ty::TypeFoldable;
@@ -87,7 +88,9 @@ pub(super) enum CandidateSource {
 }
 
 /// Methods used to assemble candidates for either trait or projection goals.
-pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
+pub(super) trait GoalKind<'tcx>:
+    TypeFoldable<TyCtxt<'tcx>> + Copy + Eq + std::fmt::Display
+{
     fn self_ty(self) -> Ty<'tcx>;
 
     fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx>;
@@ -96,6 +99,17 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
 
     fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId;
 
+    // Try equating an assumption predicate against a goal's predicate. If it
+    // holds, then execute the `then` callback, which should do any additional
+    // work, then produce a response (typically by executing
+    // [`EvalCtxt::evaluate_added_goals_and_make_canonical_response`]).
+    fn probe_and_match_goal_against_assumption(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        assumption: ty::Predicate<'tcx>,
+        then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>,
+    ) -> QueryResult<'tcx>;
+
     // Consider a clause, which consists of a "assumption" and some "requirements",
     // to satisfy a goal. If the requirements hold, then attempt to satisfy our
     // goal by equating it with the assumption.
@@ -104,7 +118,26 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
         goal: Goal<'tcx, Self>,
         assumption: ty::Predicate<'tcx>,
         requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
-    ) -> QueryResult<'tcx>;
+    ) -> QueryResult<'tcx> {
+        Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| {
+            ecx.add_goals(requirements);
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
+    /// Consider a bound originating from the item bounds of an alias. For this we
+    /// require that the well-formed requirements of the self type of the goal
+    /// are "satisfied from the param-env".
+    /// See [`EvalCtxt::validate_alias_bound_self_from_param_env`].
+    fn consider_alias_bound_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        assumption: ty::Predicate<'tcx>,
+    ) -> QueryResult<'tcx> {
+        Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| {
+            ecx.validate_alias_bound_self_from_param_env(goal)
+        })
+    }
 
     // Consider a clause specifically for a `dyn Trait` self type. This requires
     // additionally checking all of the supertraits and object bounds to hold,
@@ -113,7 +146,25 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         assumption: ty::Predicate<'tcx>,
-    ) -> QueryResult<'tcx>;
+    ) -> QueryResult<'tcx> {
+        Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| {
+            let tcx = ecx.tcx();
+            let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
+                    bug!("expected object type in `consider_object_bound_candidate`");
+                };
+            ecx.add_goals(
+                structural_traits::predicates_for_object_candidate(
+                    &ecx,
+                    goal.param_env,
+                    goal.predicate.trait_ref(tcx),
+                    bounds,
+                )
+                .into_iter()
+                .map(|pred| goal.with(tcx, pred)),
+            );
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
 
     fn consider_impl_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
@@ -463,7 +514,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
         for assumption in self.tcx().item_bounds(alias_ty.def_id).subst(self.tcx(), alias_ty.substs)
         {
-            match G::consider_implied_clause(self, goal, assumption, []) {
+            match G::consider_alias_bound_candidate(self, goal, assumption) {
                 Ok(result) => {
                     candidates.push(Candidate { source: CandidateSource::AliasBound, result })
                 }
@@ -472,6 +523,105 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
+    /// Check that we are allowed to use an alias bound originating from the self
+    /// type of this goal. This means something different depending on the self type's
+    /// alias kind.
+    ///
+    /// * Projection: Given a goal with a self type such as `<Ty as Trait>::Assoc`,
+    /// we require that the bound `Ty: Trait` can be proven using either a nested alias
+    /// bound candidate, or a param-env candidate.
+    ///
+    /// * Opaque: The param-env must be in `Reveal::UserFacing` mode. Otherwise,
+    /// the goal should be proven by using the hidden type instead.
+    #[instrument(level = "debug", skip(self), ret)]
+    pub(super) fn validate_alias_bound_self_from_param_env<G: GoalKind<'tcx>>(
+        &mut self,
+        goal: Goal<'tcx, G>,
+    ) -> QueryResult<'tcx> {
+        match *goal.predicate.self_ty().kind() {
+            ty::Alias(ty::Projection, projection_ty) => {
+                let mut param_env_candidates = vec![];
+                let self_trait_ref = projection_ty.trait_ref(self.tcx());
+
+                if self_trait_ref.self_ty().is_ty_var() {
+                    return self
+                        .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS);
+                }
+
+                let trait_goal: Goal<'_, ty::TraitPredicate<'tcx>> = goal.with(
+                    self.tcx(),
+                    ty::TraitPredicate {
+                        trait_ref: self_trait_ref,
+                        constness: ty::BoundConstness::NotConst,
+                        polarity: ty::ImplPolarity::Positive,
+                    },
+                );
+
+                self.assemble_param_env_candidates(trait_goal, &mut param_env_candidates);
+                // FIXME: We probably need some sort of recursion depth check here.
+                // Can't come up with an example yet, though, and the worst case
+                // we can have is a compiler stack overflow...
+                self.assemble_alias_bound_candidates(trait_goal, &mut param_env_candidates);
+
+                // FIXME: We must also consider alias-bound candidates for a peculiar
+                // class of built-in candidates that I'll call "defaulted" built-ins.
+                //
+                // For example, we always know that `T: Pointee` is implemented, but
+                // we do not always know what `<T as Pointee>::Metadata` actually is,
+                // similar to if we had a user-defined impl with a `default type ...`.
+                // For these traits, since we're not able to always normalize their
+                // associated types to a concrete type, we must consider their alias bounds
+                // instead, so we can prove bounds such as `<T as Pointee>::Metadata: Copy`.
+                self.assemble_alias_bound_candidates_for_builtin_impl_default_items(
+                    trait_goal,
+                    &mut param_env_candidates,
+                );
+
+                self.merge_candidates(param_env_candidates)
+            }
+            ty::Alias(ty::Opaque, _opaque_ty) => match goal.param_env.reveal() {
+                Reveal::UserFacing => {
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                }
+                Reveal::All => return Err(NoSolution),
+            },
+            _ => bug!("only expected to be called on alias tys"),
+        }
+    }
+
+    /// Assemble a subset of builtin impl candidates for a class of candidates called
+    /// "defaulted" built-in traits.
+    ///
+    /// For example, we always know that `T: Pointee` is implemented, but we do not
+    /// always know what `<T as Pointee>::Metadata` actually is! See the comment in
+    /// [`EvalCtxt::validate_alias_bound_self_from_param_env`] for more detail.
+    #[instrument(level = "debug", skip_all)]
+    fn assemble_alias_bound_candidates_for_builtin_impl_default_items<G: GoalKind<'tcx>>(
+        &mut self,
+        goal: Goal<'tcx, G>,
+        candidates: &mut Vec<Candidate<'tcx>>,
+    ) {
+        let lang_items = self.tcx().lang_items();
+        let trait_def_id = goal.predicate.trait_def_id(self.tcx());
+
+        // You probably shouldn't add anything to this list unless you
+        // know what you're doing.
+        let result = if lang_items.pointee_trait() == Some(trait_def_id) {
+            G::consider_builtin_pointee_candidate(self, goal)
+        } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
+            G::consider_builtin_discriminant_kind_candidate(self, goal)
+        } else {
+            Err(NoSolution)
+        };
+
+        match result {
+            Ok(result) => {
+                candidates.push(Candidate { source: CandidateSource::BuiltinImpl, result })
+            }
+            Err(NoSolution) => (),
+        }
+    }
+
     #[instrument(level = "debug", skip_all)]
     fn assemble_object_bound_candidates<G: GoalKind<'tcx>>(
         &mut self,

compiler/rustc_trait_selection/src/solve/project_goals.rs

@@ -56,11 +56,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         self.trait_def_id(tcx)
     }
 
-    fn consider_implied_clause(
+    fn probe_and_match_goal_against_assumption(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         assumption: ty::Predicate<'tcx>,
-        requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
+        then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>,
     ) -> QueryResult<'tcx> {
         if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred()
             && poly_projection_pred.projection_def_id() == goal.predicate.def_id()
@@ -75,49 +75,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 )?;
                 ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)
                     .expect("expected goal term to be fully unconstrained");
-                ecx.add_goals(requirements);
+                then(ecx)
-                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-            })
-        } else {
-            Err(NoSolution)
-        }
-    }
-
-    fn consider_object_bound_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-        assumption: ty::Predicate<'tcx>,
-    ) -> QueryResult<'tcx> {
-        if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred()
-            && poly_projection_pred.projection_def_id() == goal.predicate.def_id()
-        {
-            ecx.probe(|ecx| {
-                let tcx = ecx.tcx();
-
-                let assumption_projection_pred =
-                    ecx.instantiate_binder_with_infer(poly_projection_pred);
-                ecx.eq(
-                    goal.param_env,
-                    goal.predicate.projection_ty,
-                    assumption_projection_pred.projection_ty,
-                )?;
-
-                let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
-                    bug!("expected object type in `consider_object_bound_candidate`");
-                };
-                ecx.add_goals(
-                    structural_traits::predicates_for_object_candidate(
-                        &ecx,
-                        goal.param_env,
-                        goal.predicate.projection_ty.trait_ref(tcx),
-                        bounds,
-                    )
-                    .into_iter()
-                    .map(|pred| goal.with(tcx, pred)),
-                );
-                ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)
-                    .expect("expected goal term to be fully unconstrained");
-                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         } else {
             Err(NoSolution)

compiler/rustc_trait_selection/src/solve/trait_goals.rs

@@ -78,11 +78,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         })
     }
 
-    fn consider_implied_clause(
+    fn probe_and_match_goal_against_assumption(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         assumption: ty::Predicate<'tcx>,
-        requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
+        then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>,
     ) -> QueryResult<'tcx> {
         if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred()
             && poly_trait_pred.def_id() == goal.predicate.def_id()
@@ -97,48 +97,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     goal.predicate.trait_ref,
                     assumption_trait_pred.trait_ref,
                 )?;
-                ecx.add_goals(requirements);
+                then(ecx)
-                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-            })
-        } else {
-            Err(NoSolution)
-        }
-    }
-
-    fn consider_object_bound_candidate(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-        assumption: ty::Predicate<'tcx>,
-    ) -> QueryResult<'tcx> {
-        if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred()
-            && poly_trait_pred.def_id() == goal.predicate.def_id()
-            && poly_trait_pred.polarity() == goal.predicate.polarity
-        {
-            // FIXME: Constness and polarity
-            ecx.probe(|ecx| {
-                let assumption_trait_pred =
-                    ecx.instantiate_binder_with_infer(poly_trait_pred);
-                ecx.eq(
-                    goal.param_env,
-                    goal.predicate.trait_ref,
-                    assumption_trait_pred.trait_ref,
-                )?;
-
-                let tcx = ecx.tcx();
-                let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
-                    bug!("expected object type in `consider_object_bound_candidate`");
-                };
-                ecx.add_goals(
-                    structural_traits::predicates_for_object_candidate(
-                        &ecx,
-                        goal.param_env,
-                        goal.predicate.trait_ref,
-                        bounds,
-                    )
-                    .into_iter()
-                    .map(|pred| goal.with(tcx, pred)),
-                );
-                ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
         } else {
             Err(NoSolution)

compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs

@@ -28,6 +28,7 @@ use rustc_hir::{GenericParam, Item, Node};
 use rustc_infer::infer::error_reporting::TypeErrCtxt;
 use rustc_infer::infer::{InferOk, TypeTrace};
 use rustc_middle::traits::select::OverflowError;
+use rustc_middle::traits::SelectionOutputTypeParameterMismatch;
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable};
@@ -1087,17 +1088,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 }
             }
 
-            OutputTypeParameterMismatch(
+            OutputTypeParameterMismatch(box SelectionOutputTypeParameterMismatch {
                 found_trait_ref,
                 expected_trait_ref,
-                terr @ TypeError::CyclicTy(_),
+                terr: terr @ TypeError::CyclicTy(_),
-            ) => self.report_type_parameter_mismatch_cyclic_type_error(
+            }) => self.report_type_parameter_mismatch_cyclic_type_error(
                 &obligation,
                 found_trait_ref,
                 expected_trait_ref,
                 terr,
             ),
-            OutputTypeParameterMismatch(found_trait_ref, expected_trait_ref, _) => {
+            OutputTypeParameterMismatch(box SelectionOutputTypeParameterMismatch {
+                found_trait_ref,
+                expected_trait_ref,
+                terr: _,
+            }) => {
                 match self.report_type_parameter_mismatch_error(
                     &obligation,
                     span,

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -10,6 +10,7 @@ use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::lang_items::LangItem;
 use rustc_infer::infer::LateBoundRegionConversionTime::HigherRankedType;
 use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
+use rustc_middle::traits::SelectionOutputTypeParameterMismatch;
 use rustc_middle::ty::{
     self, Binder, GenericParamDefKind, InternalSubsts, SubstsRef, ToPolyTraitRef, ToPredicate,
     TraitRef, Ty, TyCtxt, TypeVisitableExt,
@@ -811,7 +812,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     fn confirm_poly_trait_refs(
         &mut self,
         obligation: &TraitObligation<'tcx>,
-        expected_trait_ref: ty::PolyTraitRef<'tcx>,
+        self_ty_trait_ref: ty::PolyTraitRef<'tcx>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
         let obligation_trait_ref = obligation.predicate.to_poly_trait_ref();
         // Normalize the obligation and expected trait refs together, because why not
@@ -822,7 +823,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     obligation.param_env,
                     obligation.cause.clone(),
                     obligation.recursion_depth + 1,
-                    (obligation_trait_ref, expected_trait_ref),
+                    (obligation_trait_ref, self_ty_trait_ref),
                 )
             });
 
@@ -834,7 +835,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 obligations.extend(nested);
                 obligations
             })
-            .map_err(|e| OutputTypeParameterMismatch(expected_trait_ref, obligation_trait_ref, e))
+            .map_err(|terr| {
+                OutputTypeParameterMismatch(Box::new(SelectionOutputTypeParameterMismatch {
+                    expected_trait_ref: obligation_trait_ref,
+                    found_trait_ref: expected_trait_ref,
+                    terr,
+                }))
+            })
     }
 
     fn confirm_trait_upcasting_unsize_candidate(

compiler/rustc_ty_utils/src/consts.rs

@@ -394,7 +394,7 @@ impl<'a, 'tcx> visit::Visitor<'a, 'tcx> for IsThirPolymorphic<'a, 'tcx> {
 pub fn thir_abstract_const(
     tcx: TyCtxt<'_>,
     def: LocalDefId,
-) -> Result<Option<ty::Const<'_>>, ErrorGuaranteed> {
+) -> Result<Option<ty::EarlyBinder<ty::Const<'_>>>, ErrorGuaranteed> {
     if !tcx.features().generic_const_exprs {
         return Ok(None);
     }
@@ -420,7 +420,7 @@ pub fn thir_abstract_const(
 
     let root_span = body.exprs[body_id].span;
 
-    Some(recurse_build(tcx, body, body_id, root_span)).transpose()
+    Ok(Some(ty::EarlyBinder(recurse_build(tcx, body, body_id, root_span)?)))
 }
 
 pub fn provide(providers: &mut ty::query::Providers) {

compiler/rustc_ty_utils/src/implied_bounds.rs

@@ -31,6 +31,18 @@ fn assumed_wf_types(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List<Ty<'_>> {
             }
         }
         DefKind::AssocConst | DefKind::AssocTy => tcx.assumed_wf_types(tcx.parent(def_id)),
+        DefKind::OpaqueTy => match tcx.def_kind(tcx.parent(def_id)) {
+            DefKind::TyAlias => ty::List::empty(),
+            DefKind::AssocTy => tcx.assumed_wf_types(tcx.parent(def_id)),
+            // Nested opaque types only occur in associated types:
+            // ` type Opaque<T> = impl Trait<&'static T, AssocTy = impl Nested>; `
+            // assumed_wf_types should include those of `Opaque<T>`, `Opaque<T>` itself
+            // and `&'static T`.
+            DefKind::OpaqueTy => bug!("unimplemented implied bounds for neseted opaque types"),
+            def_kind @ _ => {
+                bug!("unimplemented implied bounds for opaque types with parent {def_kind:?}")
+            }
+        },
         DefKind::Mod
         | DefKind::Struct
         | DefKind::Union
@@ -51,7 +63,6 @@ fn assumed_wf_types(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List<Ty<'_>> {
         | DefKind::ForeignMod
         | DefKind::AnonConst
         | DefKind::InlineConst
-        | DefKind::OpaqueTy
         | DefKind::ImplTraitPlaceholder
         | DefKind::Field
         | DefKind::LifetimeParam

compiler/rustc_type_ir/src/lib.rs

@@ -643,7 +643,7 @@ impl UnifyKey for FloatVid {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Decodable, Encodable, Hash, HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Decodable, Encodable, Hash, HashStable_Generic)]
 #[rustc_pass_by_value]
 pub enum Variance {
     Covariant,     // T<A> <: T<B> iff A <: B -- e.g., function return type

library/core/src/intrinsics.rs

@@ -1187,7 +1187,7 @@ extern "rust-intrinsic" {
     /// Below are common applications of `transmute` which can be replaced with safer
     /// constructs.
     ///
-    /// Turning raw bytes (`&[u8]`) into `u32`, `f64`, etc.:
+    /// Turning raw bytes (`[u8; SZ]`) into `u32`, `f64`, etc.:
     ///
     /// ```
     /// let raw_bytes = [0x78, 0x56, 0x34, 0x12];

library/core/src/ptr/non_null.rs

@@ -449,6 +449,19 @@ impl<T: ?Sized> NonNull<T> {
         // SAFETY: `self` is a `NonNull` pointer which is necessarily non-null
         unsafe { NonNull::new_unchecked(self.as_ptr() as *mut U) }
     }
+
+    /// See [`pointer::add`] for semantics and safety requirements.
+    #[inline]
+    pub(crate) const unsafe fn add(self, delta: usize) -> Self
+    where
+        T: Sized,
+    {
+        // SAFETY: We require that the delta stays in-bounds of the object, and
+        // thus it cannot become null, as that would require wrapping the
+        // address space, which no legal objects are allowed to do.
+        // And the caller promised the `delta` is sound to add.
+        unsafe { NonNull { pointer: self.pointer.add(delta) } }
+    }
 }
 
 impl<T> NonNull<[T]> {

library/core/src/slice/iter.rs

@@ -13,7 +13,7 @@ use crate::iter::{
 use crate::marker::{PhantomData, Send, Sized, Sync};
 use crate::mem::{self, SizedTypeProperties};
 use crate::num::NonZeroUsize;
-use crate::ptr::NonNull;
+use crate::ptr::{invalid, invalid_mut, NonNull};
 
 use super::{from_raw_parts, from_raw_parts_mut};
 
@@ -67,9 +67,7 @@ pub struct Iter<'a, T: 'a> {
     ptr: NonNull<T>,
     /// For non-ZSTs, the non-null pointer to the past-the-end element.
     ///
-    /// For ZSTs, this is `ptr.wrapping_byte_add(len)`.
+    /// For ZSTs, this is `ptr::invalid(len)`.
-    ///
-    /// For all types, `ptr == end` tests whether the iterator is empty.
     end: *const T,
     _marker: PhantomData<&'a T>,
 }
@@ -94,8 +92,7 @@ impl<'a, T> Iter<'a, T> {
         unsafe {
             assume(!ptr.is_null());
 
-            let end =
+            let end = if T::IS_ZST { invalid(slice.len()) } else { ptr.add(slice.len()) };
-                if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) };
 
             Self { ptr: NonNull::new_unchecked(ptr as *mut T), end, _marker: PhantomData }
         }
@@ -193,9 +190,7 @@ pub struct IterMut<'a, T: 'a> {
     ptr: NonNull<T>,
     /// For non-ZSTs, the non-null pointer to the past-the-end element.
     ///
-    /// For ZSTs, this is `ptr.wrapping_byte_add(len)`.
+    /// For ZSTs, this is `ptr::invalid_mut(len)`.
-    ///
-    /// For all types, `ptr == end` tests whether the iterator is empty.
     end: *mut T,
     _marker: PhantomData<&'a mut T>,
 }
@@ -235,8 +230,7 @@ impl<'a, T> IterMut<'a, T> {
         unsafe {
             assume(!ptr.is_null());
 
-            let end =
+            let end = if T::IS_ZST { invalid_mut(slice.len()) } else { ptr.add(slice.len()) };
-                if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) };
 
             Self { ptr: NonNull::new_unchecked(ptr), end, _marker: PhantomData }
         }

library/core/src/slice/iter/macros.rs

@@ -1,11 +1,30 @@
 //! Macros used by iterators of slice.
 
+// Shrinks the iterator when T is a ZST, setting the length to `new_len`.
+// `new_len` must not exceed `self.len()`.
+macro_rules! zst_set_len {
+    ($self: ident, $new_len: expr) => {{
+        #![allow(unused_unsafe)] // we're sometimes used within an unsafe block
+
+        // SAFETY: same as `invalid(_mut)`, but the macro doesn't know
+        // which versions of that function to call, so open-code it.
+        $self.end = unsafe { mem::transmute::<usize, _>($new_len) };
+    }};
+}
+
+// Shrinks the iterator when T is a ZST, reducing the length by `n`.
+// `n` must not exceed `self.len()`.
+macro_rules! zst_shrink {
+    ($self: ident, $n: ident) => {
+        let new_len = $self.end.addr() - $n;
+        zst_set_len!($self, new_len);
+    };
+}
+
 // Inlining is_empty and len makes a huge performance difference
 macro_rules! is_empty {
-    // The way we encode the length of a ZST iterator, this works both for ZST
-    // and non-ZST.
     ($self: ident) => {
-        $self.ptr.as_ptr() as *const T == $self.end
+        if T::IS_ZST { $self.end.addr() == 0 } else { $self.ptr.as_ptr() as *const _ == $self.end }
     };
 }
 
@@ -13,16 +32,13 @@ macro_rules! len {
     ($self: ident) => {{
         #![allow(unused_unsafe)] // we're sometimes used within an unsafe block
 
-        let start = $self.ptr;
         if T::IS_ZST {
-            // This _cannot_ use `ptr_sub` because we depend on wrapping
+            $self.end.addr()
-            // to represent the length of long ZST slice iterators.
-            $self.end.addr().wrapping_sub(start.as_ptr().addr())
         } else {
             // To get rid of some bounds checks (see `position`), we use ptr_sub instead of
             // offset_from (Tested by `codegen/slice-position-bounds-check`.)
             // SAFETY: by the type invariant pointers are aligned and `start <= end`
-            unsafe { $self.end.sub_ptr(start.as_ptr()) }
+            unsafe { $self.end.sub_ptr($self.ptr.as_ptr()) }
         }
     }};
 }
@@ -50,14 +66,6 @@ macro_rules! iterator {
             ($self: ident) => {& $( $mut_ )? *$self.pre_dec_end(1)}
         }
 
-        // Shrinks the iterator when T is a ZST, by moving the end of the iterator
-        // backwards by `n`. `n` must not exceed `self.len()`.
-        macro_rules! zst_shrink {
-            ($self: ident, $n: ident) => {
-                $self.end = $self.end.wrapping_byte_sub($n);
-            }
-        }
-
         impl<'a, T> $name<'a, T> {
             // Helper function for creating a slice from the iterator.
             #[inline(always)]
@@ -73,16 +81,15 @@ macro_rules! iterator {
             // Unsafe because the offset must not exceed `self.len()`.
             #[inline(always)]
             unsafe fn post_inc_start(&mut self, offset: usize) -> * $raw_mut T {
+                let old = self.ptr;
                 if T::IS_ZST {
                     zst_shrink!(self, offset);
-                    self.ptr.as_ptr()
                 } else {
-                    let old = self.ptr.as_ptr();
                     // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`,
                     // so this new pointer is inside `self` and thus guaranteed to be non-null.
-                    self.ptr = unsafe { NonNull::new_unchecked(self.ptr.as_ptr().add(offset)) };
+                    self.ptr = unsafe { self.ptr.add(offset) };
-                    old
                 }
+                old.as_ptr()
             }
 
             // Helper function for moving the end of the iterator backwards by `offset` elements,
@@ -155,9 +162,7 @@ macro_rules! iterator {
                 if n >= len!(self) {
                     // This iterator is now empty.
                     if T::IS_ZST {
-                        // We have to do it this way as `ptr` may never be 0, but `end`
+                        zst_set_len!(self, 0);
-                        // could be (due to wrapping).
-                        self.end = self.ptr.as_ptr();
                     } else {
                         // SAFETY: end can't be 0 if T isn't ZST because ptr isn't 0 and end >= ptr
                         unsafe {
@@ -356,7 +361,11 @@ macro_rules! iterator {
             fn nth_back(&mut self, n: usize) -> Option<$elem> {
                 if n >= len!(self) {
                     // This iterator is now empty.
-                    self.end = self.ptr.as_ptr();
+                    if T::IS_ZST {
+                        zst_set_len!(self, 0);
+                    } else {
+                        self.end = self.ptr.as_ptr();
+                    }
                     return None;
                 }
                 // SAFETY: We are in bounds. `pre_dec_end` does the right thing even for ZSTs.

library/portable-simd/.github/workflows/ci.yml

@@ -241,6 +241,10 @@ jobs:
           - "--features std"
           - "--features generic_const_exprs"
           - "--features std --features generic_const_exprs"
+          - "--features all_lane_counts"
+          - "--features all_lane_counts --features std"
+          - "--features all_lane_counts --features generic_const_exprs"
+          - "--features all_lane_counts --features std --features generic_const_exprs"
 
     steps:
       - uses: actions/checkout@v2

library/portable-simd/README.md

@@ -24,19 +24,10 @@ or by setting up `rustup default nightly` or else with `cargo +nightly {build,te
 ```bash
 cargo new hellosimd
 ```
-to create a new crate. Edit `hellosimd/Cargo.toml` to be 
+to create a new crate. Finally write this in `src/main.rs`:
-```toml
-[package]
-name = "hellosimd"
-version = "0.1.0"
-edition = "2018"
-[dependencies]
-core_simd = { git = "https://github.com/rust-lang/portable-simd" }
-```
-
-and finally write this in `src/main.rs`:
 ```rust
-use core_simd::*;
+#![feature(portable_simd)]
+use std::simd::f32x4;
 fn main() {
     let a = f32x4::splat(10.0);
     let b = f32x4::from_array([1.0, 2.0, 3.0, 4.0]);
@@ -44,24 +35,23 @@ fn main() {
 }
 ```
 
-Explanation: We import all the bindings from the crate with the first line. Then, we construct our SIMD vectors with methods like `splat` or `from_array`. Finally, we can use operators on them like `+` and the appropriate SIMD instructions will be carried out. When we run `cargo run` you should get `[11.0, 12.0, 13.0, 14.0]`.
+Explanation: We construct our SIMD vectors with methods like `splat` or `from_array`. Next, we can use operators like `+` on them, and the appropriate SIMD instructions will be carried out. When we run `cargo run` you should get `[11.0, 12.0, 13.0, 14.0]`.
 
-## Code Organization
+## Supported vectors
 
-Currently the crate is organized so that each element type is a file, and then the 64-bit, 128-bit, 256-bit, and 512-bit vectors using those types are contained in said file.
+Currently, vectors may have up to 64 elements, but aliases are provided only up to 512-bit vectors.
-
-All types are then exported as a single, flat module.
 
 Depending on the size of the primitive type, the number of lanes the vector will have varies. For example, 128-bit vectors have four `f32` lanes and two `f64` lanes.
 
 The supported element types are as follows:
 * **Floating Point:** `f32`, `f64`
-* **Signed Integers:** `i8`, `i16`, `i32`, `i64`, `i128`, `isize`
+* **Signed Integers:** `i8`, `i16`, `i32`, `i64`, `isize` (`i128` excluded)
-* **Unsigned Integers:** `u8`, `u16`, `u32`, `u64`, `u128`, `usize`
+* **Unsigned Integers:** `u8`, `u16`, `u32`, `u64`, `usize` (`u128` excluded)
-* **Masks:** `mask8`, `mask16`, `mask32`, `mask64`, `mask128`, `masksize`
+* **Pointers:** `*const T` and `*mut T` (zero-sized metadata only)
+* **Masks:** 8-bit, 16-bit, 32-bit, 64-bit, and `usize`-sized masks
 
-Floating point, signed integers, and unsigned integers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to.
+Floating point, signed integers, unsigned integers, and pointers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to.
-The `mask` types are "truthy" values, but they use the number of bits in their name instead of just 1 bit like a normal `bool` uses.
+The mask types have elements that are "truthy" values, like `bool`, but have an unspecified layout because different architectures prefer different layouts for mask types.
 
 [simd-guide]: ./beginners-guide.md
 [zulip-project-portable-simd]: https://rust-lang.zulipchat.com/#narrow/stream/257879-project-portable-simd

library/portable-simd/crates/core_simd/Cargo.toml

@@ -13,12 +13,11 @@ default = ["as_crate"]
 as_crate = []
 std = []
 generic_const_exprs = []
+all_lane_counts = []
 
-[target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen]
+[target.'cfg(target_arch = "wasm32")'.dev-dependencies]
-version = "0.2"
+wasm-bindgen = "0.2"
-
+wasm-bindgen-test = "0.3"
-[dev-dependencies.wasm-bindgen-test]
-version = "0.3"
 
 [dev-dependencies.proptest]
 version = "0.10"

library/portable-simd/crates/core_simd/examples/README.md

@@ -0,0 +1,13 @@
+### `stdsimd` examples
+
+This crate is a port of example uses of `stdsimd`, mostly taken from the `packed_simd` crate.
+
+The examples contain, as in the case of `dot_product.rs`, multiple ways of solving the problem, in order to show idiomatic uses of SIMD and iteration of performance designs.
+
+Run the tests with the command 
+
+```
+cargo run --example dot_product
+```
+
+and verify the code for `dot_product.rs` on your machine.

library/portable-simd/crates/core_simd/examples/dot_product.rs

@@ -0,0 +1,169 @@
+// Code taken from the `packed_simd` crate
+// Run this code with `cargo test --example dot_product`
+//use std::iter::zip;
+
+#![feature(array_chunks)]
+#![feature(slice_as_chunks)]
+// Add these imports to use the stdsimd library
+#![feature(portable_simd)]
+use core_simd::simd::*;
+
+// This is your barebones dot product implementation:
+// Take 2 vectors, multiply them element wise and *then*
+// go along the resulting array and add up the result.
+// In the next example we will see if there
+//  is any difference to adding and multiplying in tandem.
+pub fn dot_prod_scalar_0(a: &[f32], b: &[f32]) -> f32 {
+    assert_eq!(a.len(), b.len());
+
+    a.iter().zip(b.iter()).map(|(a, b)| a * b).sum()
+}
+
+// When dealing with SIMD, it is very important to think about the amount
+// of data movement and when it happens. We're going over simple computation examples here, and yet
+// it is not trivial to understand what may or may not contribute to performance
+// changes. Eventually, you will need tools to inspect the generated assembly and confirm your
+// hypothesis and benchmarks - we will mention them later on.
+// With the use of `fold`, we're doing a multiplication,
+// and then adding it to the sum, one element from both vectors at a time.
+pub fn dot_prod_scalar_1(a: &[f32], b: &[f32]) -> f32 {
+    assert_eq!(a.len(), b.len());
+    a.iter()
+        .zip(b.iter())
+        .fold(0.0, |a, zipped| a + zipped.0 * zipped.1)
+}
+
+// We now move on to the SIMD implementations: notice the following constructs:
+// `array_chunks::<4>`: mapping this over the vector will let use construct SIMD vectors
+// `f32x4::from_array`: construct the SIMD vector from a slice
+// `(a * b).reduce_sum()`: Multiply both f32x4 vectors together, and then reduce them.
+// This approach essentially uses SIMD to produce a vector of length N/4 of all the products,
+// and then add those with `sum()`. This is suboptimal.
+// TODO: ASCII diagrams
+pub fn dot_prod_simd_0(a: &[f32], b: &[f32]) -> f32 {
+    assert_eq!(a.len(), b.len());
+    // TODO handle remainder when a.len() % 4 != 0
+    a.array_chunks::<4>()
+        .map(|&a| f32x4::from_array(a))
+        .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b)))
+        .map(|(a, b)| (a * b).reduce_sum())
+        .sum()
+}
+
+// There's some simple ways to improve the previous code:
+// 1. Make a `zero` `f32x4` SIMD vector that we will be accumulating into
+// So that there is only one `sum()` reduction when the last `f32x4` has been processed
+// 2. Exploit Fused Multiply Add so that the multiplication, addition and sinking into the reduciton
+// happen in the same step.
+// If the arrays are large, minimizing the data shuffling will lead to great perf.
+// If the arrays are small, handling the remainder elements when the length isn't a multiple of 4
+// Can become a problem.
+pub fn dot_prod_simd_1(a: &[f32], b: &[f32]) -> f32 {
+    assert_eq!(a.len(), b.len());
+    // TODO handle remainder when a.len() % 4 != 0
+    a.array_chunks::<4>()
+        .map(|&a| f32x4::from_array(a))
+        .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b)))
+        .fold(f32x4::splat(0.0), |acc, zipped| acc + zipped.0 * zipped.1)
+        .reduce_sum()
+}
+
+// A lot of knowledgeable use of SIMD comes from knowing specific instructions that are
+// available - let's try to use the `mul_add` instruction, which is the fused-multiply-add we were looking for.
+use std_float::StdFloat;
+pub fn dot_prod_simd_2(a: &[f32], b: &[f32]) -> f32 {
+    assert_eq!(a.len(), b.len());
+    // TODO handle remainder when a.len() % 4 != 0
+    let mut res = f32x4::splat(0.0);
+    a.array_chunks::<4>()
+        .map(|&a| f32x4::from_array(a))
+        .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b)))
+        .for_each(|(a, b)| {
+            res = a.mul_add(b, res);
+        });
+    res.reduce_sum()
+}
+
+// Finally, we will write the same operation but handling the loop remainder.
+const LANES: usize = 4;
+pub fn dot_prod_simd_3(a: &[f32], b: &[f32]) -> f32 {
+    assert_eq!(a.len(), b.len());
+
+    let (a_extra, a_chunks) = a.as_rchunks();
+    let (b_extra, b_chunks) = b.as_rchunks();
+
+    // These are always true, but for emphasis:
+    assert_eq!(a_chunks.len(), b_chunks.len());
+    assert_eq!(a_extra.len(), b_extra.len());
+
+    let mut sums = [0.0; LANES];
+    for ((x, y), d) in std::iter::zip(a_extra, b_extra).zip(&mut sums) {
+        *d = x * y;
+    }
+
+    let mut sums = f32x4::from_array(sums);
+    std::iter::zip(a_chunks, b_chunks).for_each(|(x, y)| {
+        sums += f32x4::from_array(*x) * f32x4::from_array(*y);
+    });
+
+    sums.reduce_sum()
+}
+
+// Finally, we present an iterator version for handling remainders in a scalar fashion at the end of the loop.
+// Unfortunately, this is allocating 1 `XMM` register on the order of `~len(a)` - we'll see how we can get around it in the
+// next example.
+pub fn dot_prod_simd_4(a: &[f32], b: &[f32]) -> f32 {
+    let mut sum = a
+        .array_chunks::<4>()
+        .map(|&a| f32x4::from_array(a))
+        .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b)))
+        .map(|(a, b)| a * b)
+        .fold(f32x4::splat(0.0), std::ops::Add::add)
+        .reduce_sum();
+    let remain = a.len() - (a.len() % 4);
+    sum += a[remain..]
+        .iter()
+        .zip(&b[remain..])
+        .map(|(a, b)| a * b)
+        .sum::<f32>();
+    sum
+}
+
+// This version allocates a single `XMM` register for accumulation, and the folds don't allocate on top of that.
+// Notice the the use of `mul_add`, which can do a multiply and an add operation ber iteration.
+pub fn dot_prod_simd_5(a: &[f32], b: &[f32]) -> f32 {
+    a.array_chunks::<4>()
+        .map(|&a| f32x4::from_array(a))
+        .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b)))
+        .fold(f32x4::splat(0.), |acc, (a, b)| a.mul_add(b, acc))
+        .reduce_sum()
+}
+
+fn main() {
+    // Empty main to make cargo happy
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn smoke_test() {
+        use super::*;
+        let a: Vec<f32> = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0];
+        let b: Vec<f32> = vec![-8.0, -7.0, -6.0, -5.0, 4.0, 3.0, 2.0, 1.0];
+        let x: Vec<f32> = [0.5; 1003].to_vec();
+        let y: Vec<f32> = [2.0; 1003].to_vec();
+
+        // Basic check
+        assert_eq!(0.0, dot_prod_scalar_0(&a, &b));
+        assert_eq!(0.0, dot_prod_scalar_1(&a, &b));
+        assert_eq!(0.0, dot_prod_simd_0(&a, &b));
+        assert_eq!(0.0, dot_prod_simd_1(&a, &b));
+        assert_eq!(0.0, dot_prod_simd_2(&a, &b));
+        assert_eq!(0.0, dot_prod_simd_3(&a, &b));
+        assert_eq!(0.0, dot_prod_simd_4(&a, &b));
+        assert_eq!(0.0, dot_prod_simd_5(&a, &b));
+
+        // We can handle vectors that are non-multiples of 4
+        assert_eq!(1003.0, dot_prod_simd_3(&x, &y));
+    }
+}

library/portable-simd/crates/core_simd/src/alias.rs

@@ -0,0 +1,227 @@
+macro_rules! number {
+    { 1 } => { "one" };
+    { 2 } => { "two" };
+    { 4 } => { "four" };
+    { 8 } => { "eight" };
+    { $x:literal } => { stringify!($x) };
+}
+
+macro_rules! plural {
+    { 1 } => { "" };
+    { $x:literal } => { "s" };
+}
+
+macro_rules! alias {
+    {
+        $(
+            $element_ty:ty = {
+                $($alias:ident $num_elements:tt)*
+            }
+        )*
+    } => {
+        $(
+            $(
+            #[doc = concat!("A SIMD vector with ", number!($num_elements), " element", plural!($num_elements), " of type [`", stringify!($element_ty), "`].")]
+            #[allow(non_camel_case_types)]
+            pub type $alias = $crate::simd::Simd<$element_ty, $num_elements>;
+            )*
+        )*
+    }
+}
+
+macro_rules! mask_alias {
+    {
+        $(
+            $element_ty:ty : $size:literal = {
+                $($alias:ident $num_elements:tt)*
+            }
+        )*
+    } => {
+        $(
+            $(
+            #[doc = concat!("A SIMD mask with ", number!($num_elements), " element", plural!($num_elements), " for vectors with ", $size, " element types.")]
+            ///
+            #[doc = concat!(
+                "The layout of this type is unspecified, and may change between platforms and/or Rust versions, and code should not assume that it is equivalent to `[",
+                stringify!($element_ty), "; ", $num_elements, "]`."
+            )]
+            #[allow(non_camel_case_types)]
+            pub type $alias = $crate::simd::Mask<$element_ty, $num_elements>;
+            )*
+        )*
+    }
+}
+
+alias! {
+    i8 = {
+        i8x1 1
+        i8x2 2
+        i8x4 4
+        i8x8 8
+        i8x16 16
+        i8x32 32
+        i8x64 64
+    }
+
+    i16 = {
+        i16x1 1
+        i16x2 2
+        i16x4 4
+        i16x8 8
+        i16x16 16
+        i16x32 32
+        i16x64 64
+    }
+
+    i32 = {
+        i32x1 1
+        i32x2 2
+        i32x4 4
+        i32x8 8
+        i32x16 16
+        i32x32 32
+        i32x64 64
+    }
+
+    i64 = {
+        i64x1 1
+        i64x2 2
+        i64x4 4
+        i64x8 8
+        i64x16 16
+        i64x32 32
+        i64x64 64
+    }
+
+    isize = {
+        isizex1 1
+        isizex2 2
+        isizex4 4
+        isizex8 8
+        isizex16 16
+        isizex32 32
+        isizex64 64
+    }
+
+    u8 = {
+        u8x1 1
+        u8x2 2
+        u8x4 4
+        u8x8 8
+        u8x16 16
+        u8x32 32
+        u8x64 64
+    }
+
+    u16 = {
+        u16x1 1
+        u16x2 2
+        u16x4 4
+        u16x8 8
+        u16x16 16
+        u16x32 32
+        u16x64 64
+    }
+
+    u32 = {
+        u32x1 1
+        u32x2 2
+        u32x4 4
+        u32x8 8
+        u32x16 16
+        u32x32 32
+        u32x64 64
+    }
+
+    u64 = {
+        u64x1 1
+        u64x2 2
+        u64x4 4
+        u64x8 8
+        u64x16 16
+        u64x32 32
+        u64x64 64
+    }
+
+    usize = {
+        usizex1 1
+        usizex2 2
+        usizex4 4
+        usizex8 8
+        usizex16 16
+        usizex32 32
+        usizex64 64
+    }
+
+    f32 = {
+        f32x1 1
+        f32x2 2
+        f32x4 4
+        f32x8 8
+        f32x16 16
+        f32x32 32
+        f32x64 64
+    }
+
+    f64 = {
+        f64x1 1
+        f64x2 2
+        f64x4 4
+        f64x8 8
+        f64x16 16
+        f64x32 32
+        f64x64 64
+    }
+}
+
+mask_alias! {
+    i8 : "8-bit" = {
+        mask8x1 1
+        mask8x2 2
+        mask8x4 4
+        mask8x8 8
+        mask8x16 16
+        mask8x32 32
+        mask8x64 64
+    }
+
+    i16 : "16-bit" = {
+        mask16x1 1
+        mask16x2 2
+        mask16x4 4
+        mask16x8 8
+        mask16x16 16
+        mask16x32 32
+        mask16x64 64
+    }
+
+    i32 : "32-bit" = {
+        mask32x1 1
+        mask32x2 2
+        mask32x4 4
+        mask32x8 8
+        mask32x16 16
+        mask32x32 32
+        mask32x64 64
+    }
+
+    i64 : "64-bit" = {
+        mask64x1 1
+        mask64x2 2
+        mask64x4 4
+        mask64x8 8
+        mask64x16 16
+        mask64x32 32
+        mask64x64 64
+    }
+
+    isize : "pointer-sized" = {
+        masksizex1 1
+        masksizex2 2
+        masksizex4 4
+        masksizex8 8
+        masksizex16 16
+        masksizex32 32
+        masksizex64 64
+    }
+}

library/portable-simd/crates/core_simd/src/cast.rs

@@ -0,0 +1,55 @@
+use crate::simd::SimdElement;
+
+/// Supporting trait for `Simd::cast`.  Typically doesn't need to be used directly.
+///
+/// # Safety
+/// Implementing this trait asserts that the type is a valid vector element for the `simd_cast` or
+/// `simd_as` intrinsics.
+pub unsafe trait SimdCast: SimdElement {}
+
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for i8 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for i16 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for i32 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for i64 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for isize {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for u8 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for u16 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for u32 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for u64 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for usize {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for f32 {}
+// Safety: primitive number types can be cast to other primitive number types
+unsafe impl SimdCast for f64 {}
+
+/// Supporting trait for `Simd::cast_ptr`.  Typically doesn't need to be used directly.
+///
+/// # Safety
+/// Implementing this trait asserts that the type is a valid vector element for the `simd_cast_ptr`
+/// intrinsic.
+pub unsafe trait SimdCastPtr<T> {}
+
+// Safety: pointers can be cast to other pointer types
+unsafe impl<T, U> SimdCastPtr<T> for *const U
+where
+    U: core::ptr::Pointee,
+    T: core::ptr::Pointee<Metadata = U::Metadata>,
+{
+}
+// Safety: pointers can be cast to other pointer types
+unsafe impl<T, U> SimdCastPtr<T> for *mut U
+where
+    U: core::ptr::Pointee,
+    T: core::ptr::Pointee<Metadata = U::Metadata>,
+{
+}

library/portable-simd/crates/core_simd/src/elements.rs

@@ -1,11 +1,15 @@
+mod const_ptr;
 mod float;
 mod int;
+mod mut_ptr;
 mod uint;
 
 mod sealed {
     pub trait Sealed {}
 }
 
+pub use const_ptr::*;
 pub use float::*;
 pub use int::*;
+pub use mut_ptr::*;
 pub use uint::*;

library/portable-simd/crates/core_simd/src/elements/const_ptr.rs

@@ -0,0 +1,141 @@
+use super::sealed::Sealed;
+use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount};
+
+/// Operations on SIMD vectors of constant pointers.
+pub trait SimdConstPtr: Copy + Sealed {
+    /// Vector of `usize` with the same number of lanes.
+    type Usize;
+
+    /// Vector of `isize` with the same number of lanes.
+    type Isize;
+
+    /// Vector of mutable pointers to the same type.
+    type MutPtr;
+
+    /// Mask type used for manipulating this SIMD vector type.
+    type Mask;
+
+    /// Returns `true` for each lane that is null.
+    fn is_null(self) -> Self::Mask;
+
+    /// Changes constness without changing the type.
+    ///
+    /// Equivalent to calling [`pointer::cast_mut`] on each lane.
+    fn cast_mut(self) -> Self::MutPtr;
+
+    /// Gets the "address" portion of the pointer.
+    ///
+    /// This method discards pointer semantic metadata, so the result cannot be
+    /// directly cast into a valid pointer.
+    ///
+    /// This method semantically discards *provenance* and
+    /// *address-space* information. To properly restore that information, use [`Self::with_addr`].
+    ///
+    /// Equivalent to calling [`pointer::addr`] on each lane.
+    fn addr(self) -> Self::Usize;
+
+    /// Creates a new pointer with the given address.
+    ///
+    /// This performs the same operation as a cast, but copies the *address-space* and
+    /// *provenance* of `self` to the new pointer.
+    ///
+    /// Equivalent to calling [`pointer::with_addr`] on each lane.
+    fn with_addr(self, addr: Self::Usize) -> Self;
+
+    /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use
+    /// in [`Self::from_exposed_addr`].
+    fn expose_addr(self) -> Self::Usize;
+
+    /// Convert an address back to a pointer, picking up a previously "exposed" provenance.
+    ///
+    /// Equivalent to calling [`core::ptr::from_exposed_addr`] on each lane.
+    fn from_exposed_addr(addr: Self::Usize) -> Self;
+
+    /// Calculates the offset from a pointer using wrapping arithmetic.
+    ///
+    /// Equivalent to calling [`pointer::wrapping_offset`] on each lane.
+    fn wrapping_offset(self, offset: Self::Isize) -> Self;
+
+    /// Calculates the offset from a pointer using wrapping arithmetic.
+    ///
+    /// Equivalent to calling [`pointer::wrapping_add`] on each lane.
+    fn wrapping_add(self, count: Self::Usize) -> Self;
+
+    /// Calculates the offset from a pointer using wrapping arithmetic.
+    ///
+    /// Equivalent to calling [`pointer::wrapping_sub`] on each lane.
+    fn wrapping_sub(self, count: Self::Usize) -> Self;
+}
+
+impl<T, const LANES: usize> Sealed for Simd<*const T, LANES> where
+    LaneCount<LANES>: SupportedLaneCount
+{
+}
+
+impl<T, const LANES: usize> SimdConstPtr for Simd<*const T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    type Usize = Simd<usize, LANES>;
+    type Isize = Simd<isize, LANES>;
+    type MutPtr = Simd<*mut T, LANES>;
+    type Mask = Mask<isize, LANES>;
+
+    #[inline]
+    fn is_null(self) -> Self::Mask {
+        Simd::splat(core::ptr::null()).simd_eq(self)
+    }
+
+    #[inline]
+    fn cast_mut(self) -> Self::MutPtr {
+        self.cast_ptr()
+    }
+
+    #[inline]
+    fn addr(self) -> Self::Usize {
+        // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic.
+        // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the
+        // provenance).
+        unsafe { core::mem::transmute_copy(&self) }
+    }
+
+    #[inline]
+    fn with_addr(self, addr: Self::Usize) -> Self {
+        // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic.
+        //
+        // In the mean-time, this operation is defined to be "as if" it was
+        // a wrapping_offset, so we can emulate it as such. This should properly
+        // restore pointer provenance even under today's compiler.
+        self.cast_ptr::<*const u8>()
+            .wrapping_offset(addr.cast::<isize>() - self.addr().cast::<isize>())
+            .cast_ptr()
+    }
+
+    #[inline]
+    fn expose_addr(self) -> Self::Usize {
+        // Safety: `self` is a pointer vector
+        unsafe { intrinsics::simd_expose_addr(self) }
+    }
+
+    #[inline]
+    fn from_exposed_addr(addr: Self::Usize) -> Self {
+        // Safety: `self` is a pointer vector
+        unsafe { intrinsics::simd_from_exposed_addr(addr) }
+    }
+
+    #[inline]
+    fn wrapping_offset(self, count: Self::Isize) -> Self {
+        // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets
+        unsafe { intrinsics::simd_arith_offset(self, count) }
+    }
+
+    #[inline]
+    fn wrapping_add(self, count: Self::Usize) -> Self {
+        self.wrapping_offset(count.cast())
+    }
+
+    #[inline]
+    fn wrapping_sub(self, count: Self::Usize) -> Self {
+        self.wrapping_offset(-count.cast::<isize>())
+    }
+}

library/portable-simd/crates/core_simd/src/elements/mut_ptr.rs

@@ -0,0 +1,136 @@
+use super::sealed::Sealed;
+use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount};
+
+/// Operations on SIMD vectors of mutable pointers.
+pub trait SimdMutPtr: Copy + Sealed {
+    /// Vector of `usize` with the same number of lanes.
+    type Usize;
+
+    /// Vector of `isize` with the same number of lanes.
+    type Isize;
+
+    /// Vector of constant pointers to the same type.
+    type ConstPtr;
+
+    /// Mask type used for manipulating this SIMD vector type.
+    type Mask;
+
+    /// Returns `true` for each lane that is null.
+    fn is_null(self) -> Self::Mask;
+
+    /// Changes constness without changing the type.
+    ///
+    /// Equivalent to calling [`pointer::cast_const`] on each lane.
+    fn cast_const(self) -> Self::ConstPtr;
+
+    /// Gets the "address" portion of the pointer.
+    ///
+    /// This method discards pointer semantic metadata, so the result cannot be
+    /// directly cast into a valid pointer.
+    ///
+    /// Equivalent to calling [`pointer::addr`] on each lane.
+    fn addr(self) -> Self::Usize;
+
+    /// Creates a new pointer with the given address.
+    ///
+    /// This performs the same operation as a cast, but copies the *address-space* and
+    /// *provenance* of `self` to the new pointer.
+    ///
+    /// Equivalent to calling [`pointer::with_addr`] on each lane.
+    fn with_addr(self, addr: Self::Usize) -> Self;
+
+    /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use
+    /// in [`Self::from_exposed_addr`].
+    fn expose_addr(self) -> Self::Usize;
+
+    /// Convert an address back to a pointer, picking up a previously "exposed" provenance.
+    ///
+    /// Equivalent to calling [`core::ptr::from_exposed_addr_mut`] on each lane.
+    fn from_exposed_addr(addr: Self::Usize) -> Self;
+
+    /// Calculates the offset from a pointer using wrapping arithmetic.
+    ///
+    /// Equivalent to calling [`pointer::wrapping_offset`] on each lane.
+    fn wrapping_offset(self, offset: Self::Isize) -> Self;
+
+    /// Calculates the offset from a pointer using wrapping arithmetic.
+    ///
+    /// Equivalent to calling [`pointer::wrapping_add`] on each lane.
+    fn wrapping_add(self, count: Self::Usize) -> Self;
+
+    /// Calculates the offset from a pointer using wrapping arithmetic.
+    ///
+    /// Equivalent to calling [`pointer::wrapping_sub`] on each lane.
+    fn wrapping_sub(self, count: Self::Usize) -> Self;
+}
+
+impl<T, const LANES: usize> Sealed for Simd<*mut T, LANES> where LaneCount<LANES>: SupportedLaneCount
+{}
+
+impl<T, const LANES: usize> SimdMutPtr for Simd<*mut T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    type Usize = Simd<usize, LANES>;
+    type Isize = Simd<isize, LANES>;
+    type ConstPtr = Simd<*const T, LANES>;
+    type Mask = Mask<isize, LANES>;
+
+    #[inline]
+    fn is_null(self) -> Self::Mask {
+        Simd::splat(core::ptr::null_mut()).simd_eq(self)
+    }
+
+    #[inline]
+    fn cast_const(self) -> Self::ConstPtr {
+        self.cast_ptr()
+    }
+
+    #[inline]
+    fn addr(self) -> Self::Usize {
+        // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic.
+        // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the
+        // provenance).
+        unsafe { core::mem::transmute_copy(&self) }
+    }
+
+    #[inline]
+    fn with_addr(self, addr: Self::Usize) -> Self {
+        // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic.
+        //
+        // In the mean-time, this operation is defined to be "as if" it was
+        // a wrapping_offset, so we can emulate it as such. This should properly
+        // restore pointer provenance even under today's compiler.
+        self.cast_ptr::<*mut u8>()
+            .wrapping_offset(addr.cast::<isize>() - self.addr().cast::<isize>())
+            .cast_ptr()
+    }
+
+    #[inline]
+    fn expose_addr(self) -> Self::Usize {
+        // Safety: `self` is a pointer vector
+        unsafe { intrinsics::simd_expose_addr(self) }
+    }
+
+    #[inline]
+    fn from_exposed_addr(addr: Self::Usize) -> Self {
+        // Safety: `self` is a pointer vector
+        unsafe { intrinsics::simd_from_exposed_addr(addr) }
+    }
+
+    #[inline]
+    fn wrapping_offset(self, count: Self::Isize) -> Self {
+        // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets
+        unsafe { intrinsics::simd_arith_offset(self, count) }
+    }
+
+    #[inline]
+    fn wrapping_add(self, count: Self::Usize) -> Self {
+        self.wrapping_offset(count.cast())
+    }
+
+    #[inline]
+    fn wrapping_sub(self, count: Self::Usize) -> Self {
+        self.wrapping_offset(-count.cast::<isize>())
+    }
+}

library/portable-simd/crates/core_simd/src/eq.rs

@@ -1,4 +1,6 @@
-use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdElement, SupportedLaneCount};
+use crate::simd::{
+    intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdElement, SimdMutPtr, SupportedLaneCount,
+};
 
 /// Parallel `PartialEq`.
 pub trait SimdPartialEq {
@@ -71,3 +73,37 @@ macro_rules! impl_mask {
 }
 
 impl_mask! { i8, i16, i32, i64, isize }
+
+impl<T, const LANES: usize> SimdPartialEq for Simd<*const T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    type Mask = Mask<isize, LANES>;
+
+    #[inline]
+    fn simd_eq(self, other: Self) -> Self::Mask {
+        self.addr().simd_eq(other.addr())
+    }
+
+    #[inline]
+    fn simd_ne(self, other: Self) -> Self::Mask {
+        self.addr().simd_ne(other.addr())
+    }
+}
+
+impl<T, const LANES: usize> SimdPartialEq for Simd<*mut T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    type Mask = Mask<isize, LANES>;
+
+    #[inline]
+    fn simd_eq(self, other: Self) -> Self::Mask {
+        self.addr().simd_eq(other.addr())
+    }
+
+    #[inline]
+    fn simd_ne(self, other: Self) -> Self::Mask {
+        self.addr().simd_ne(other.addr())
+    }
+}

library/portable-simd/crates/core_simd/src/fmt.rs

@@ -1,39 +1,21 @@
 use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
 use core::fmt;
 
-macro_rules! impl_fmt_trait {
+impl<T, const LANES: usize> fmt::Debug for Simd<T, LANES>
-    { $($trait:ident,)* } => {
+where
-        $(
+    LaneCount<LANES>: SupportedLaneCount,
-            impl<T, const LANES: usize> fmt::$trait for Simd<T, LANES>
+    T: SimdElement + fmt::Debug,
-            where
+{
-                LaneCount<LANES>: SupportedLaneCount,
+    /// A `Simd<T, N>` has a debug format like the one for `[T]`:
-                T: SimdElement + fmt::$trait,
+    /// ```
-            {
+    /// # #![feature(portable_simd)]
-                fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    /// # #[cfg(feature = "as_crate")] use core_simd::simd::Simd;
-                    #[repr(transparent)]
+    /// # #[cfg(not(feature = "as_crate"))] use core::simd::Simd;
-                    struct Wrapper<'a, T: fmt::$trait>(&'a T);
+    /// let floats = Simd::<f32, 4>::splat(-1.0);
-
+    /// assert_eq!(format!("{:?}", [-1.0; 4]), format!("{:?}", floats));
-                    impl<T: fmt::$trait> fmt::Debug for Wrapper<'_, T> {
+    /// ```
-                        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    #[inline]
-                            self.0.fmt(f)
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                        }
+        <[T] as fmt::Debug>::fmt(self.as_array(), f)
-                    }
-
-                    f.debug_list()
-                        .entries(self.as_array().iter().map(|x| Wrapper(x)))
-                        .finish()
-                }
-            }
-        )*
     }
 }
-
-impl_fmt_trait! {
-    Debug,
-    Binary,
-    LowerExp,
-    UpperExp,
-    Octal,
-    LowerHex,
-    UpperHex,
-}

library/portable-simd/crates/core_simd/src/intrinsics.rs

@@ -61,9 +61,6 @@ extern "platform-intrinsic" {
     /// xor
     pub(crate) fn simd_xor<T>(x: T, y: T) -> T;
 
-    /// getelementptr (without inbounds)
-    pub(crate) fn simd_arith_offset<T, U>(ptrs: T, offsets: U) -> T;
-
     /// fptoui/fptosi/uitofp/sitofp
     /// casting floats to integers is truncating, so it is safe to convert values like e.g. 1.5
     /// but the truncated value must fit in the target type or the result is poison.
@@ -150,4 +147,17 @@ extern "platform-intrinsic" {
     pub(crate) fn simd_select<M, T>(m: M, yes: T, no: T) -> T;
     #[allow(unused)]
     pub(crate) fn simd_select_bitmask<M, T>(m: M, yes: T, no: T) -> T;
+
+    /// getelementptr (without inbounds)
+    /// equivalent to wrapping_offset
+    pub(crate) fn simd_arith_offset<T, U>(ptr: T, offset: U) -> T;
+
+    /// equivalent to `T as U` semantics, specifically for pointers
+    pub(crate) fn simd_cast_ptr<T, U>(ptr: T) -> U;
+
+    /// expose a pointer as an address
+    pub(crate) fn simd_expose_addr<T, U>(ptr: T) -> U;
+
+    /// convert an exposed address back to a pointer
+    pub(crate) fn simd_from_exposed_addr<T, U>(addr: T) -> U;
 }

library/portable-simd/crates/core_simd/src/lane_count.rs

@@ -23,24 +23,20 @@ pub trait SupportedLaneCount: Sealed {
 
 impl<const LANES: usize> Sealed for LaneCount<LANES> {}
 
-impl SupportedLaneCount for LaneCount<1> {
+macro_rules! supported_lane_count {
-    type BitMask = [u8; 1];
+    ($($lanes:literal),+) => {
-}
+        $(
-impl SupportedLaneCount for LaneCount<2> {
+            impl SupportedLaneCount for LaneCount<$lanes> {
-    type BitMask = [u8; 1];
+                type BitMask = [u8; ($lanes + 7) / 8];
-}
+            }
-impl SupportedLaneCount for LaneCount<4> {
+        )+
-    type BitMask = [u8; 1];
+    };
-}
-impl SupportedLaneCount for LaneCount<8> {
-    type BitMask = [u8; 1];
-}
-impl SupportedLaneCount for LaneCount<16> {
-    type BitMask = [u8; 2];
-}
-impl SupportedLaneCount for LaneCount<32> {
-    type BitMask = [u8; 4];
-}
-impl SupportedLaneCount for LaneCount<64> {
-    type BitMask = [u8; 8];
 }
+
+supported_lane_count!(1, 2, 4, 8, 16, 32, 64);
+#[cfg(feature = "all_lane_counts")]
+supported_lane_count!(
+    3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+    31, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,
+    56, 57, 58, 59, 60, 61, 62, 63
+);

library/portable-simd/crates/core_simd/src/lib.rs

@@ -1,5 +1,8 @@
 #![no_std]
 #![feature(
+    const_refs_to_cell,
+    const_maybe_uninit_as_mut_ptr,
+    const_mut_refs,
     convert_float_to_int,
     decl_macro,
     intra_doc_pointers,
@@ -7,7 +10,9 @@
     repr_simd,
     simd_ffi,
     staged_api,
-    stdsimd
+    stdsimd,
+    strict_provenance,
+    ptr_metadata
 )]
 #![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))]
 #![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))]
@@ -19,4 +24,3 @@
 #[path = "mod.rs"]
 mod core_simd;
 pub use self::core_simd::simd;
-pub use simd::*;

library/portable-simd/crates/core_simd/src/masks.rs

@@ -55,6 +55,7 @@ pub unsafe trait MaskElement: SimdElement + Sealed {}
 macro_rules! impl_element {
     { $ty:ty } => {
         impl Sealed for $ty {
+            #[inline]
             fn valid<const LANES: usize>(value: Simd<Self, LANES>) -> bool
             where
                 LaneCount<LANES>: SupportedLaneCount,
@@ -62,6 +63,7 @@ macro_rules! impl_element {
                 (value.simd_eq(Simd::splat(0 as _)) | value.simd_eq(Simd::splat(-1 as _))).all()
             }
 
+            #[inline]
             fn eq(self, other: Self) -> bool { self == other }
 
             const TRUE: Self = -1;
@@ -83,7 +85,9 @@ impl_element! { isize }
 ///
 /// Masks represent boolean inclusion/exclusion on a per-lane basis.
 ///
-/// The layout of this type is unspecified.
+/// The layout of this type is unspecified, and may change between platforms
+/// and/or Rust versions, and code should not assume that it is equivalent to
+/// `[T; LANES]`.
 #[repr(transparent)]
 pub struct Mask<T, const LANES: usize>(mask_impl::Mask<T, LANES>)
 where
@@ -102,6 +106,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn clone(&self) -> Self {
         *self
     }
@@ -113,11 +118,13 @@ where
     LaneCount<LANES>: SupportedLaneCount,
 {
     /// Construct a mask by setting all lanes to the given value.
+    #[inline]
     pub fn splat(value: bool) -> Self {
         Self(mask_impl::Mask::splat(value))
     }
 
     /// Converts an array of bools to a SIMD mask.
+    #[inline]
     pub fn from_array(array: [bool; LANES]) -> Self {
         // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of
         //     true:    0b_0000_0001
@@ -134,6 +141,7 @@ where
     }
 
     /// Converts a SIMD mask to an array of bools.
+    #[inline]
     pub fn to_array(self) -> [bool; LANES] {
         // This follows mostly the same logic as from_array.
         // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of
@@ -261,6 +269,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn from(array: [bool; LANES]) -> Self {
         Self::from_array(array)
     }
@@ -271,6 +280,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn from(vector: Mask<T, LANES>) -> Self {
         vector.to_array()
     }
@@ -520,60 +530,6 @@ where
     }
 }
 
-/// A mask for SIMD vectors with eight elements of 8 bits.
-pub type mask8x8 = Mask<i8, 8>;
-
-/// A mask for SIMD vectors with 16 elements of 8 bits.
-pub type mask8x16 = Mask<i8, 16>;
-
-/// A mask for SIMD vectors with 32 elements of 8 bits.
-pub type mask8x32 = Mask<i8, 32>;
-
-/// A mask for SIMD vectors with 64 elements of 8 bits.
-pub type mask8x64 = Mask<i8, 64>;
-
-/// A mask for SIMD vectors with four elements of 16 bits.
-pub type mask16x4 = Mask<i16, 4>;
-
-/// A mask for SIMD vectors with eight elements of 16 bits.
-pub type mask16x8 = Mask<i16, 8>;
-
-/// A mask for SIMD vectors with 16 elements of 16 bits.
-pub type mask16x16 = Mask<i16, 16>;
-
-/// A mask for SIMD vectors with 32 elements of 16 bits.
-pub type mask16x32 = Mask<i16, 32>;
-
-/// A mask for SIMD vectors with two elements of 32 bits.
-pub type mask32x2 = Mask<i32, 2>;
-
-/// A mask for SIMD vectors with four elements of 32 bits.
-pub type mask32x4 = Mask<i32, 4>;
-
-/// A mask for SIMD vectors with eight elements of 32 bits.
-pub type mask32x8 = Mask<i32, 8>;
-
-/// A mask for SIMD vectors with 16 elements of 32 bits.
-pub type mask32x16 = Mask<i32, 16>;
-
-/// A mask for SIMD vectors with two elements of 64 bits.
-pub type mask64x2 = Mask<i64, 2>;
-
-/// A mask for SIMD vectors with four elements of 64 bits.
-pub type mask64x4 = Mask<i64, 4>;
-
-/// A mask for SIMD vectors with eight elements of 64 bits.
-pub type mask64x8 = Mask<i64, 8>;
-
-/// A mask for SIMD vectors with two elements of pointer width.
-pub type masksizex2 = Mask<isize, 2>;
-
-/// A mask for SIMD vectors with four elements of pointer width.
-pub type masksizex4 = Mask<isize, 4>;
-
-/// A mask for SIMD vectors with eight elements of pointer width.
-pub type masksizex8 = Mask<isize, 8>;
-
 macro_rules! impl_from {
     { $from:ty  => $($to:ty),* } => {
         $(
@@ -581,6 +537,7 @@ macro_rules! impl_from {
         where
             LaneCount<LANES>: SupportedLaneCount,
         {
+            #[inline]
             fn from(value: Mask<$from, LANES>) -> Self {
                 value.cast()
             }

library/portable-simd/crates/core_simd/src/masks/bitmask.rs

@@ -26,6 +26,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn clone(&self) -> Self {
         *self
     }
@@ -36,6 +37,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn eq(&self, other: &Self) -> bool {
         self.0.as_ref() == other.0.as_ref()
     }
@@ -46,6 +48,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
         self.0.as_ref().partial_cmp(other.0.as_ref())
     }
@@ -63,6 +66,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn cmp(&self, other: &Self) -> core::cmp::Ordering {
         self.0.as_ref().cmp(other.0.as_ref())
     }

library/portable-simd/crates/core_simd/src/masks/full_masks.rs

@@ -37,6 +37,7 @@ where
     T: MaskElement + PartialEq,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn eq(&self, other: &Self) -> bool {
         self.0.eq(&other.0)
     }
@@ -47,6 +48,7 @@ where
     T: MaskElement + PartialOrd,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
         self.0.partial_cmp(&other.0)
     }
@@ -64,6 +66,7 @@ where
     T: MaskElement + Ord,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn cmp(&self, other: &Self) -> core::cmp::Ordering {
         self.0.cmp(&other.0)
     }
@@ -262,6 +265,7 @@ where
     T: MaskElement,
     LaneCount<LANES>: SupportedLaneCount,
 {
+    #[inline]
     fn from(value: Mask<T, LANES>) -> Self {
         value.0
     }

library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs

@@ -48,10 +48,12 @@ macro_rules! impl_integer_intrinsic {
         impl<T: MaskElement> ToBitMask for Mask<T, $lanes> {
             type BitMask = $int;
 
+            #[inline]
             fn to_bitmask(self) -> $int {
                 self.0.to_bitmask_integer()
             }
 
+            #[inline]
             fn from_bitmask(bitmask: $int) -> Self {
                 Self(mask_impl::Mask::from_bitmask_integer(bitmask))
             }
@@ -83,10 +85,12 @@ where
 {
     const BYTES: usize = bitmask_len(LANES);
 
+    #[inline]
     fn to_bitmask_array(self) -> [u8; Self::BYTES] {
         self.0.to_bitmask_array()
     }
 
+    #[inline]
     fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self {
         Mask(mask_impl::Mask::from_bitmask_array(bitmask))
     }

library/portable-simd/crates/core_simd/src/mod.rs

@@ -6,6 +6,8 @@ pub(crate) mod intrinsics;
 #[cfg(feature = "generic_const_exprs")]
 mod to_bytes;
 
+mod alias;
+mod cast;
 mod elements;
 mod eq;
 mod fmt;
@@ -15,6 +17,7 @@ mod masks;
 mod ops;
 mod ord;
 mod select;
+mod swizzle_dyn;
 mod vector;
 mod vendor;
 
@@ -22,11 +25,14 @@ mod vendor;
 pub mod simd {
     pub(crate) use crate::core_simd::intrinsics;
 
+    pub use crate::core_simd::alias::*;
+    pub use crate::core_simd::cast::*;
     pub use crate::core_simd::elements::*;
     pub use crate::core_simd::eq::*;
     pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount};
     pub use crate::core_simd::masks::*;
     pub use crate::core_simd::ord::*;
     pub use crate::core_simd::swizzle::*;
+    pub use crate::core_simd::swizzle_dyn::*;
     pub use crate::core_simd::vector::*;
 }

library/portable-simd/crates/core_simd/src/ord.rs

@@ -1,4 +1,6 @@
-use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount};
+use crate::simd::{
+    intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdMutPtr, SimdPartialEq, SupportedLaneCount,
+};
 
 /// Parallel `PartialOrd`.
 pub trait SimdPartialOrd: SimdPartialEq {
@@ -211,3 +213,101 @@ macro_rules! impl_mask {
 }
 
 impl_mask! { i8, i16, i32, i64, isize }
+
+impl<T, const LANES: usize> SimdPartialOrd for Simd<*const T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    #[inline]
+    fn simd_lt(self, other: Self) -> Self::Mask {
+        self.addr().simd_lt(other.addr())
+    }
+
+    #[inline]
+    fn simd_le(self, other: Self) -> Self::Mask {
+        self.addr().simd_le(other.addr())
+    }
+
+    #[inline]
+    fn simd_gt(self, other: Self) -> Self::Mask {
+        self.addr().simd_gt(other.addr())
+    }
+
+    #[inline]
+    fn simd_ge(self, other: Self) -> Self::Mask {
+        self.addr().simd_ge(other.addr())
+    }
+}
+
+impl<T, const LANES: usize> SimdOrd for Simd<*const T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    #[inline]
+    fn simd_max(self, other: Self) -> Self {
+        self.simd_lt(other).select(other, self)
+    }
+
+    #[inline]
+    fn simd_min(self, other: Self) -> Self {
+        self.simd_gt(other).select(other, self)
+    }
+
+    #[inline]
+    fn simd_clamp(self, min: Self, max: Self) -> Self {
+        assert!(
+            min.simd_le(max).all(),
+            "each lane in `min` must be less than or equal to the corresponding lane in `max`",
+        );
+        self.simd_max(min).simd_min(max)
+    }
+}
+
+impl<T, const LANES: usize> SimdPartialOrd for Simd<*mut T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    #[inline]
+    fn simd_lt(self, other: Self) -> Self::Mask {
+        self.addr().simd_lt(other.addr())
+    }
+
+    #[inline]
+    fn simd_le(self, other: Self) -> Self::Mask {
+        self.addr().simd_le(other.addr())
+    }
+
+    #[inline]
+    fn simd_gt(self, other: Self) -> Self::Mask {
+        self.addr().simd_gt(other.addr())
+    }
+
+    #[inline]
+    fn simd_ge(self, other: Self) -> Self::Mask {
+        self.addr().simd_ge(other.addr())
+    }
+}
+
+impl<T, const LANES: usize> SimdOrd for Simd<*mut T, LANES>
+where
+    LaneCount<LANES>: SupportedLaneCount,
+{
+    #[inline]
+    fn simd_max(self, other: Self) -> Self {
+        self.simd_lt(other).select(other, self)
+    }
+
+    #[inline]
+    fn simd_min(self, other: Self) -> Self {
+        self.simd_gt(other).select(other, self)
+    }
+
+    #[inline]
+    fn simd_clamp(self, min: Self, max: Self) -> Self {
+        assert!(
+            min.simd_le(max).all(),
+            "each lane in `min` must be less than or equal to the corresponding lane in `max`",
+        );
+        self.simd_max(min).simd_min(max)
+    }
+}

library/portable-simd/crates/core_simd/src/swizzle.rs

@@ -265,16 +265,13 @@ where
 
     /// Interleave two vectors.
     ///
-    /// Produces two vectors with lanes taken alternately from `self` and `other`.
+    /// The resulting vectors contain lanes taken alternatively from `self` and `other`, first
+    /// filling the first result, and then the second.
     ///
-    /// The first result contains the first `LANES / 2` lanes from `self` and `other`,
+    /// The reverse of this operation is [`Simd::deinterleave`].
-    /// alternating, starting with the first lane of `self`.
-    ///
-    /// The second result contains the last `LANES / 2` lanes from `self` and `other`,
-    /// alternating, starting with the lane `LANES / 2` from the start of `self`.
     ///
     /// ```
-    /// #![feature(portable_simd)]
+    /// # #![feature(portable_simd)]
     /// # use core::simd::Simd;
     /// let a = Simd::from_array([0, 1, 2, 3]);
     /// let b = Simd::from_array([4, 5, 6, 7]);
@@ -285,29 +282,17 @@ where
     #[inline]
     #[must_use = "method returns a new vector and does not mutate the original inputs"]
     pub fn interleave(self, other: Self) -> (Self, Self) {
-        const fn lo<const LANES: usize>() -> [Which; LANES] {
+        const fn interleave<const LANES: usize>(high: bool) -> [Which; LANES] {
             let mut idx = [Which::First(0); LANES];
             let mut i = 0;
             while i < LANES {
-                let offset = i / 2;
+                // Treat the source as a concatenated vector
-                idx[i] = if i % 2 == 0 {
+                let dst_index = if high { i + LANES } else { i };
-                    Which::First(offset)
+                let src_index = dst_index / 2 + (dst_index % 2) * LANES;
+                idx[i] = if src_index < LANES {
+                    Which::First(src_index)
                 } else {
-                    Which::Second(offset)
+                    Which::Second(src_index % LANES)
-                };
-                i += 1;
-            }
-            idx
-        }
-        const fn hi<const LANES: usize>() -> [Which; LANES] {
-            let mut idx = [Which::First(0); LANES];
-            let mut i = 0;
-            while i < LANES {
-                let offset = (LANES + i) / 2;
-                idx[i] = if i % 2 == 0 {
-                    Which::First(offset)
-                } else {
-                    Which::Second(offset)
                 };
                 i += 1;
             }
@@ -318,11 +303,11 @@ where
         struct Hi;
 
         impl<const LANES: usize> Swizzle2<LANES, LANES> for Lo {
-            const INDEX: [Which; LANES] = lo::<LANES>();
+            const INDEX: [Which; LANES] = interleave::<LANES>(false);
         }
 
         impl<const LANES: usize> Swizzle2<LANES, LANES> for Hi {
-            const INDEX: [Which; LANES] = hi::<LANES>();
+            const INDEX: [Which; LANES] = interleave::<LANES>(true);
         }
 
         (Lo::swizzle2(self, other), Hi::swizzle2(self, other))
@@ -336,8 +321,10 @@ where
     /// The second result takes every other lane of `self` and then `other`, starting with
     /// the second lane.
     ///
+    /// The reverse of this operation is [`Simd::interleave`].
+    ///
     /// ```
-    /// #![feature(portable_simd)]
+    /// # #![feature(portable_simd)]
     /// # use core::simd::Simd;
     /// let a = Simd::from_array([0, 4, 1, 5]);
     /// let b = Simd::from_array([2, 6, 3, 7]);
@@ -348,22 +335,17 @@ where
     #[inline]
     #[must_use = "method returns a new vector and does not mutate the original inputs"]
     pub fn deinterleave(self, other: Self) -> (Self, Self) {
-        const fn even<const LANES: usize>() -> [Which; LANES] {
+        const fn deinterleave<const LANES: usize>(second: bool) -> [Which; LANES] {
             let mut idx = [Which::First(0); LANES];
             let mut i = 0;
-            while i < LANES / 2 {
+            while i < LANES {
-                idx[i] = Which::First(2 * i);
+                // Treat the source as a concatenated vector
-                idx[i + LANES / 2] = Which::Second(2 * i);
+                let src_index = i * 2 + second as usize;
-                i += 1;
+                idx[i] = if src_index < LANES {
-            }
+                    Which::First(src_index)
-            idx
+                } else {
-        }
+                    Which::Second(src_index % LANES)
-        const fn odd<const LANES: usize>() -> [Which; LANES] {
+                };
-            let mut idx = [Which::First(0); LANES];
-            let mut i = 0;
-            while i < LANES / 2 {
-                idx[i] = Which::First(2 * i + 1);
-                idx[i + LANES / 2] = Which::Second(2 * i + 1);
                 i += 1;
             }
             idx
@@ -373,11 +355,11 @@ where
         struct Odd;
 
         impl<const LANES: usize> Swizzle2<LANES, LANES> for Even {
-            const INDEX: [Which; LANES] = even::<LANES>();
+            const INDEX: [Which; LANES] = deinterleave::<LANES>(false);
         }
 
         impl<const LANES: usize> Swizzle2<LANES, LANES> for Odd {
-            const INDEX: [Which; LANES] = odd::<LANES>();
+            const INDEX: [Which; LANES] = deinterleave::<LANES>(true);
         }
 
         (Even::swizzle2(self, other), Odd::swizzle2(self, other))

library/portable-simd/crates/core_simd/src/swizzle_dyn.rs

@@ -0,0 +1,157 @@
+use crate::simd::{LaneCount, Simd, SupportedLaneCount};
+use core::mem;
+
+impl<const N: usize> Simd<u8, N>
+where
+    LaneCount<N>: SupportedLaneCount,
+{
+    /// Swizzle a vector of bytes according to the index vector.
+    /// Indices within range select the appropriate byte.
+    /// Indices "out of bounds" instead select 0.
+    ///
+    /// Note that the current implementation is selected during build-time
+    /// of the standard library, so `cargo build -Zbuild-std` may be necessary
+    /// to unlock better performance, especially for larger vectors.
+    /// A planned compiler improvement will enable using `#[target_feature]` instead.
+    #[inline]
+    pub fn swizzle_dyn(self, idxs: Simd<u8, N>) -> Self {
+        #![allow(unused_imports, unused_unsafe)]
+        #[cfg(target_arch = "aarch64")]
+        use core::arch::aarch64::{uint8x8_t, vqtbl1q_u8, vtbl1_u8};
+        #[cfg(all(target_arch = "arm", target_feature = "v7", target_feature = "neon"))]
+        use core::arch::arm::{uint8x8_t, vtbl1_u8};
+        #[cfg(target_arch = "wasm32")]
+        use core::arch::wasm32 as wasm;
+        #[cfg(target_arch = "x86")]
+        use core::arch::x86;
+        #[cfg(target_arch = "x86_64")]
+        use core::arch::x86_64 as x86;
+        // SAFETY: Intrinsics covered by cfg
+        unsafe {
+            match N {
+                #[cfg(target_feature = "neon")]
+                8 => transize(vtbl1_u8, self, idxs),
+                #[cfg(target_feature = "ssse3")]
+                16 => transize(x86::_mm_shuffle_epi8, self, idxs),
+                #[cfg(target_feature = "simd128")]
+                16 => transize(wasm::i8x16_swizzle, self, idxs),
+                #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
+                16 => transize(vqtbl1q_u8, self, idxs),
+                #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))]
+                32 => transize_raw(avx2_pshufb, self, idxs),
+                #[cfg(target_feature = "avx512vl,avx512vbmi")]
+                32 => transize(x86::_mm256_permutexvar_epi8, self, idxs),
+                // Notable absence: avx512bw shuffle
+                // If avx512bw is available, odds of avx512vbmi are good
+                // FIXME: initial AVX512VBMI variant didn't actually pass muster
+                // #[cfg(target_feature = "avx512vbmi")]
+                // 64 => transize(x86::_mm512_permutexvar_epi8, self, idxs),
+                _ => {
+                    let mut array = [0; N];
+                    for (i, k) in idxs.to_array().into_iter().enumerate() {
+                        if (k as usize) < N {
+                            array[i] = self[k as usize];
+                        };
+                    }
+                    array.into()
+                }
+            }
+        }
+    }
+}
+
+/// "vpshufb like it was meant to be" on AVX2
+///
+/// # Safety
+/// This requires AVX2 to work
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+#[target_feature(enable = "avx2")]
+#[allow(unused)]
+#[inline]
+#[allow(clippy::let_and_return)]
+unsafe fn avx2_pshufb(bytes: Simd<u8, 32>, idxs: Simd<u8, 32>) -> Simd<u8, 32> {
+    use crate::simd::SimdPartialOrd;
+    #[cfg(target_arch = "x86")]
+    use core::arch::x86;
+    #[cfg(target_arch = "x86_64")]
+    use core::arch::x86_64 as x86;
+    use x86::_mm256_permute2x128_si256 as avx2_cross_shuffle;
+    use x86::_mm256_shuffle_epi8 as avx2_half_pshufb;
+    let mid = Simd::splat(16u8);
+    let high = mid + mid;
+    // SAFETY: Caller promised AVX2
+    unsafe {
+        // This is ordering sensitive, and LLVM will order these how you put them.
+        // Most AVX2 impls use ~5 "ports", and only 1 or 2 are capable of permutes.
+        // But the "compose" step will lower to ops that can also use at least 1 other port.
+        // So this tries to break up permutes so composition flows through "open" ports.
+        // Comparative benches should be done on multiple AVX2 CPUs before reordering this
+
+        let hihi = avx2_cross_shuffle::<0x11>(bytes.into(), bytes.into());
+        let hi_shuf = Simd::from(avx2_half_pshufb(
+            hihi,        // duplicate the vector's top half
+            idxs.into(), // so that using only 4 bits of an index still picks bytes 16-31
+        ));
+        // A zero-fill during the compose step gives the "all-Neon-like" OOB-is-0 semantics
+        let compose = idxs.simd_lt(high).select(hi_shuf, Simd::splat(0));
+        let lolo = avx2_cross_shuffle::<0x00>(bytes.into(), bytes.into());
+        let lo_shuf = Simd::from(avx2_half_pshufb(lolo, idxs.into()));
+        // Repeat, then pick indices < 16, overwriting indices 0-15 from previous compose step
+        let compose = idxs.simd_lt(mid).select(lo_shuf, compose);
+        compose
+    }
+}
+
+/// This sets up a call to an architecture-specific function, and in doing so
+/// it persuades rustc that everything is the correct size. Which it is.
+/// This would not be needed if one could convince Rust that, by matching on N,
+/// N is that value, and thus it would be valid to substitute e.g. 16.
+///
+/// # Safety
+/// The correctness of this function hinges on the sizes agreeing in actuality.
+#[allow(dead_code)]
+#[inline(always)]
+unsafe fn transize<T, const N: usize>(
+    f: unsafe fn(T, T) -> T,
+    bytes: Simd<u8, N>,
+    idxs: Simd<u8, N>,
+) -> Simd<u8, N>
+where
+    LaneCount<N>: SupportedLaneCount,
+{
+    let idxs = zeroing_idxs(idxs);
+    // SAFETY: Same obligation to use this function as to use mem::transmute_copy.
+    unsafe { mem::transmute_copy(&f(mem::transmute_copy(&bytes), mem::transmute_copy(&idxs))) }
+}
+
+/// Make indices that yield 0 for this architecture
+#[inline(always)]
+fn zeroing_idxs<const N: usize>(idxs: Simd<u8, N>) -> Simd<u8, N>
+where
+    LaneCount<N>: SupportedLaneCount,
+{
+    // On x86, make sure the top bit is set.
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    let idxs = {
+        use crate::simd::SimdPartialOrd;
+        idxs.simd_lt(Simd::splat(N as u8))
+            .select(idxs, Simd::splat(u8::MAX))
+    };
+    // Simply do nothing on most architectures.
+    idxs
+}
+
+/// As transize but no implicit call to `zeroing_idxs`.
+#[allow(dead_code)]
+#[inline(always)]
+unsafe fn transize_raw<T, const N: usize>(
+    f: unsafe fn(T, T) -> T,
+    bytes: Simd<u8, N>,
+    idxs: Simd<u8, N>,
+) -> Simd<u8, N>
+where
+    LaneCount<N>: SupportedLaneCount,
+{
+    // SAFETY: Same obligation to use this function as to use mem::transmute_copy.
+    unsafe { mem::transmute_copy(&f(mem::transmute_copy(&bytes), mem::transmute_copy(&idxs))) }
+}

library/portable-simd/crates/core_simd/src/vector/float.rs

@@ -1,24 +0,0 @@
-#![allow(non_camel_case_types)]
-
-use crate::simd::Simd;
-
-/// A 64-bit SIMD vector with two elements of type `f32`.
-pub type f32x2 = Simd<f32, 2>;
-
-/// A 128-bit SIMD vector with four elements of type `f32`.
-pub type f32x4 = Simd<f32, 4>;
-
-/// A 256-bit SIMD vector with eight elements of type `f32`.
-pub type f32x8 = Simd<f32, 8>;
-
-/// A 512-bit SIMD vector with 16 elements of type `f32`.
-pub type f32x16 = Simd<f32, 16>;
-
-/// A 128-bit SIMD vector with two elements of type `f64`.
-pub type f64x2 = Simd<f64, 2>;
-
-/// A 256-bit SIMD vector with four elements of type `f64`.
-pub type f64x4 = Simd<f64, 4>;
-
-/// A 512-bit SIMD vector with eight elements of type `f64`.
-pub type f64x8 = Simd<f64, 8>;

library/portable-simd/crates/core_simd/src/vector/int.rs

@@ -1,63 +0,0 @@
-#![allow(non_camel_case_types)]
-
-use crate::simd::Simd;
-
-/// A SIMD vector with two elements of type `isize`.
-pub type isizex2 = Simd<isize, 2>;
-
-/// A SIMD vector with four elements of type `isize`.
-pub type isizex4 = Simd<isize, 4>;
-
-/// A SIMD vector with eight elements of type `isize`.
-pub type isizex8 = Simd<isize, 8>;
-
-/// A 32-bit SIMD vector with two elements of type `i16`.
-pub type i16x2 = Simd<i16, 2>;
-
-/// A 64-bit SIMD vector with four elements of type `i16`.
-pub type i16x4 = Simd<i16, 4>;
-
-/// A 128-bit SIMD vector with eight elements of type `i16`.
-pub type i16x8 = Simd<i16, 8>;
-
-/// A 256-bit SIMD vector with 16 elements of type `i16`.
-pub type i16x16 = Simd<i16, 16>;
-
-/// A 512-bit SIMD vector with 32 elements of type `i16`.
-pub type i16x32 = Simd<i16, 32>;
-
-/// A 64-bit SIMD vector with two elements of type `i32`.
-pub type i32x2 = Simd<i32, 2>;
-
-/// A 128-bit SIMD vector with four elements of type `i32`.
-pub type i32x4 = Simd<i32, 4>;
-
-/// A 256-bit SIMD vector with eight elements of type `i32`.
-pub type i32x8 = Simd<i32, 8>;
-
-/// A 512-bit SIMD vector with 16 elements of type `i32`.
-pub type i32x16 = Simd<i32, 16>;
-
-/// A 128-bit SIMD vector with two elements of type `i64`.
-pub type i64x2 = Simd<i64, 2>;
-
-/// A 256-bit SIMD vector with four elements of type `i64`.
-pub type i64x4 = Simd<i64, 4>;
-
-/// A 512-bit SIMD vector with eight elements of type `i64`.
-pub type i64x8 = Simd<i64, 8>;
-
-/// A 32-bit SIMD vector with four elements of type `i8`.
-pub type i8x4 = Simd<i8, 4>;
-
-/// A 64-bit SIMD vector with eight elements of type `i8`.
-pub type i8x8 = Simd<i8, 8>;
-
-/// A 128-bit SIMD vector with 16 elements of type `i8`.
-pub type i8x16 = Simd<i8, 16>;
-
-/// A 256-bit SIMD vector with 32 elements of type `i8`.
-pub type i8x32 = Simd<i8, 32>;
-
-/// A 512-bit SIMD vector with 64 elements of type `i8`.
-pub type i8x64 = Simd<i8, 64>;

library/portable-simd/crates/core_simd/src/vector/ptr.rs

@@ -1,51 +0,0 @@
-//! Private implementation details of public gather/scatter APIs.
-use crate::simd::intrinsics;
-use crate::simd::{LaneCount, Simd, SupportedLaneCount};
-
-/// A vector of *const T.
-#[derive(Debug, Copy, Clone)]
-#[repr(simd)]
-pub(crate) struct SimdConstPtr<T, const LANES: usize>([*const T; LANES]);
-
-impl<T, const LANES: usize> SimdConstPtr<T, LANES>
-where
-    LaneCount<LANES>: SupportedLaneCount,
-    T: Sized,
-{
-    #[inline]
-    #[must_use]
-    pub fn splat(ptr: *const T) -> Self {
-        Self([ptr; LANES])
-    }
-
-    #[inline]
-    #[must_use]
-    pub fn wrapping_add(self, addend: Simd<usize, LANES>) -> Self {
-        // Safety: this intrinsic doesn't have a precondition
-        unsafe { intrinsics::simd_arith_offset(self, addend) }
-    }
-}
-
-/// A vector of *mut T. Be very careful around potential aliasing.
-#[derive(Debug, Copy, Clone)]
-#[repr(simd)]
-pub(crate) struct SimdMutPtr<T, const LANES: usize>([*mut T; LANES]);
-
-impl<T, const LANES: usize> SimdMutPtr<T, LANES>
-where
-    LaneCount<LANES>: SupportedLaneCount,
-    T: Sized,
-{
-    #[inline]
-    #[must_use]
-    pub fn splat(ptr: *mut T) -> Self {
-        Self([ptr; LANES])
-    }
-
-    #[inline]
-    #[must_use]
-    pub fn wrapping_add(self, addend: Simd<usize, LANES>) -> Self {
-        // Safety: this intrinsic doesn't have a precondition
-        unsafe { intrinsics::simd_arith_offset(self, addend) }
-    }
-}