Auto merge of #139396 - Zalathar:rollup-lmoqvru, r=Zalathar

Rollup of 11 pull requests

Successful merges:

 - #136457 (Expose algebraic floating point intrinsics)
 - #137880 (Autodiff batching)
 - #137897 (fix pthread-based tls on apple targets)
 - #138024 (Allow optimizing out `panic_bounds_check` in Unicode checks.)
 - #138546 (Add integer to string formatting tests)
 - #138826 (StableMIR: Add `associated_items`.)
 - #138950 (replace extra_filename with strict version hash in metrics file names)
 - #139274 (Rustdoc: typecheck settings.js)
 - #139285 (use lower case to match other error messages)
 - #139341 (Apply `Recovery::Forbidden` when reparsing pasted macro fragments.)
 - #139389 (make `Arguments::as_statically_known_str` doc(hidden))

r? `@ghost`
`@rustbot` modify labels: rollup

compiler/rustc_ast/src/expand/autodiff_attrs.rs

@@ -77,6 +77,17 @@ pub struct AutoDiffAttrs {
     /// e.g. in the [JAX
     /// Documentation](https://jax.readthedocs.io/en/latest/_tutorials/advanced-autodiff.html#how-it-s-made-two-foundational-autodiff-functions).
     pub mode: DiffMode,
+    /// A user-provided, batching width. If not given, we will default to 1 (no batching).
+    /// Calling a differentiated, non-batched function through a loop 100 times is equivalent to:
+    /// - Calling the function 50 times with a batch size of 2
+    /// - Calling the function 25 times with a batch size of 4,
+    /// etc. A batched function takes more (or longer) arguments, and might be able to benefit from
+    /// cache locality, better re-usal of primal values, and other optimizations.
+    /// We will (before LLVM's vectorizer runs) just generate most LLVM-IR instructions `width`
+    /// times, so this massively increases code size. As such, values like 1024 are unlikely to
+    /// work. We should consider limiting this to u8 or u16, but will leave it at u32 for
+    /// experiments for now and focus on documenting the implications of a large width.
+    pub width: u32,
     pub ret_activity: DiffActivity,
     pub input_activity: Vec<DiffActivity>,
 }
@@ -222,6 +233,7 @@ impl AutoDiffAttrs {
     pub const fn error() -> Self {
         AutoDiffAttrs {
             mode: DiffMode::Error,
+            width: 0,
             ret_activity: DiffActivity::None,
             input_activity: Vec::new(),
         }
@@ -229,6 +241,7 @@ impl AutoDiffAttrs {
     pub fn source() -> Self {
         AutoDiffAttrs {
             mode: DiffMode::Source,
+            width: 0,
             ret_activity: DiffActivity::None,
             input_activity: Vec::new(),
         }

compiler/rustc_ast_lowering/src/item.rs

@@ -676,12 +676,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let ty =
                     self.lower_ty(ty, ImplTraitContext::Disallowed(ImplTraitPosition::StaticTy));
                 let safety = self.lower_safety(*safety, hir::Safety::Unsafe);
-
-                // njn: where for this?
                 if define_opaque.is_some() {
                     self.dcx().span_err(i.span, "foreign statics cannot define opaque types");
                 }
-
                 (ident, hir::ForeignItemKind::Static(ty, *mutability, safety))
             }
             ForeignItemKind::TyAlias(box TyAlias { ident, .. }) => {

compiler/rustc_builtin_macros/messages.ftl

@@ -79,6 +79,7 @@ builtin_macros_autodiff_ret_activity = invalid return activity {$act} in {$mode}
 builtin_macros_autodiff_ty_activity = {$act} can not be used for this type
 builtin_macros_autodiff_unknown_activity = did not recognize Activity: `{$act}`
 
+builtin_macros_autodiff_width = autodiff width must fit u32, but is {$width}
 builtin_macros_bad_derive_target = `derive` may only be applied to `struct`s, `enum`s and `union`s
     .label = not applicable here
     .label2 = not a `struct`, `enum` or `union`

compiler/rustc_builtin_macros/src/autodiff.rs

@@ -12,12 +12,12 @@ mod llvm_enzyme {
         valid_ty_for_activity,
     };
     use rustc_ast::ptr::P;
-    use rustc_ast::token::{Token, TokenKind};
+    use rustc_ast::token::{Lit, LitKind, Token, TokenKind};
     use rustc_ast::tokenstream::*;
     use rustc_ast::visit::AssocCtxt::*;
     use rustc_ast::{
-        self as ast, AssocItemKind, BindingMode, FnRetTy, FnSig, Generics, ItemKind, MetaItemInner,
+        self as ast, AssocItemKind, BindingMode, ExprKind, FnRetTy, FnSig, Generics, ItemKind,
-        PatKind, TyKind,
+        MetaItemInner, PatKind, QSelf, TyKind,
     };
     use rustc_expand::base::{Annotatable, ExtCtxt};
     use rustc_span::{Ident, Span, Symbol, kw, sym};
@@ -45,6 +45,16 @@ mod llvm_enzyme {
         }
     }
     fn first_ident(x: &MetaItemInner) -> rustc_span::Ident {
+        if let Some(l) = x.lit() {
+            match l.kind {
+                ast::LitKind::Int(val, _) => {
+                    // get an Ident from a lit
+                    return rustc_span::Ident::from_str(val.get().to_string().as_str());
+                }
+                _ => {}
+            }
+        }
+
         let segments = &x.meta_item().unwrap().path.segments;
         assert!(segments.len() == 1);
         segments[0].ident
@@ -54,6 +64,14 @@ mod llvm_enzyme {
         first_ident(x).name.to_string()
     }
 
+    fn width(x: &MetaItemInner) -> Option<u128> {
+        let lit = x.lit()?;
+        match lit.kind {
+            ast::LitKind::Int(x, _) => Some(x.get()),
+            _ => return None,
+        }
+    }
+
     pub(crate) fn from_ast(
         ecx: &mut ExtCtxt<'_>,
         meta_item: &ThinVec<MetaItemInner>,
@@ -65,9 +83,32 @@ mod llvm_enzyme {
             dcx.emit_err(errors::AutoDiffInvalidMode { span: meta_item[1].span(), mode });
             return AutoDiffAttrs::error();
         };
+
+        // Now we check, whether the user wants autodiff in batch/vector mode, or scalar mode.
+        // If he doesn't specify an integer (=width), we default to scalar mode, thus width=1.
+        let mut first_activity = 2;
+
+        let width = if let [_, _, x, ..] = &meta_item[..]
+            && let Some(x) = width(x)
+        {
+            first_activity = 3;
+            match x.try_into() {
+                Ok(x) => x,
+                Err(_) => {
+                    dcx.emit_err(errors::AutoDiffInvalidWidth {
+                        span: meta_item[2].span(),
+                        width: x,
+                    });
+                    return AutoDiffAttrs::error();
+                }
+            }
+        } else {
+            1
+        };
+
         let mut activities: Vec<DiffActivity> = vec![];
         let mut errors = false;
-        for x in &meta_item[2..] {
+        for x in &meta_item[first_activity..] {
             let activity_str = name(&x);
             let res = DiffActivity::from_str(&activity_str);
             match res {
@@ -98,7 +139,20 @@ mod llvm_enzyme {
             (&DiffActivity::None, activities.as_slice())
         };
 
-        AutoDiffAttrs { mode, ret_activity: *ret_activity, input_activity: input_activity.to_vec() }
+        AutoDiffAttrs {
+            mode,
+            width,
+            ret_activity: *ret_activity,
+            input_activity: input_activity.to_vec(),
+        }
+    }
+
+    fn meta_item_inner_to_ts(t: &MetaItemInner, ts: &mut Vec<TokenTree>) {
+        let comma: Token = Token::new(TokenKind::Comma, Span::default());
+        let val = first_ident(t);
+        let t = Token::from_ast_ident(val);
+        ts.push(TokenTree::Token(t, Spacing::Joint));
+        ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
     }
 
     /// We expand the autodiff macro to generate a new placeholder function which passes
@@ -195,27 +249,49 @@ mod llvm_enzyme {
 
         // create TokenStream from vec elemtents:
         // meta_item doesn't have a .tokens field
-        let comma: Token = Token::new(TokenKind::Comma, Span::default());
         let mut ts: Vec<TokenTree> = vec![];
         if meta_item_vec.len() < 2 {
             // At the bare minimum, we need a fnc name and a mode, even for a dummy function with no
             // input and output args.
             dcx.emit_err(errors::AutoDiffMissingConfig { span: item.span() });
             return vec![item];
-        } else {
-            for t in meta_item_vec.clone()[1..].iter() {
-                let val = first_ident(t);
-                let t = Token::from_ast_ident(val);
-                ts.push(TokenTree::Token(t, Spacing::Joint));
-                ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
-            }
         }
+
+        meta_item_inner_to_ts(&meta_item_vec[1], &mut ts);
+
+        // Now, if the user gave a width (vector aka batch-mode ad), then we copy it.
+        // If it is not given, we default to 1 (scalar mode).
+        let start_position;
+        let kind: LitKind = LitKind::Integer;
+        let symbol;
+        if meta_item_vec.len() >= 3
+            && let Some(width) = width(&meta_item_vec[2])
+        {
+            start_position = 3;
+            symbol = Symbol::intern(&width.to_string());
+        } else {
+            start_position = 2;
+            symbol = sym::integer(1);
+        }
+        let l: Lit = Lit { kind, symbol, suffix: None };
+        let t = Token::new(TokenKind::Literal(l), Span::default());
+        let comma = Token::new(TokenKind::Comma, Span::default());
+        ts.push(TokenTree::Token(t, Spacing::Joint));
+        ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
+
+        for t in meta_item_vec.clone()[start_position..].iter() {
+            meta_item_inner_to_ts(t, &mut ts);
+        }
+
         if !has_ret {
             // We don't want users to provide a return activity if the function doesn't return anything.
             // For simplicity, we just add a dummy token to the end of the list.
             let t = Token::new(TokenKind::Ident(sym::None, false.into()), Span::default());
             ts.push(TokenTree::Token(t, Spacing::Joint));
+            ts.push(TokenTree::Token(comma, Spacing::Alone));
         }
+        // We remove the last, trailing comma.
+        ts.pop();
         let ts: TokenStream = TokenStream::from_iter(ts);
 
         let x: AutoDiffAttrs = from_ast(ecx, &meta_item_vec, has_ret);
@@ -470,6 +546,8 @@ mod llvm_enzyme {
             return body;
         }
 
+        // Everything from here onwards just tries to fullfil the return type. Fun!
+
         // having an active-only return means we'll drop the original return type.
         // So that can be treated identical to not having one in the first place.
         let primal_ret = has_ret(&sig.decl.output) && !x.has_active_only_ret();
@@ -497,86 +575,65 @@ mod llvm_enzyme {
             return body;
         }
 
-        let mut exprs = ThinVec::<P<ast::Expr>>::new();
+        let mut exprs: P<ast::Expr> = primal_call.clone();
-        if primal_ret {
-            // We have both primal ret and active floats.
-            // primal ret is first, by construction.
-            exprs.push(primal_call);
-        }
-
-        // Now construct default placeholder for each active float.
-        // Is there something nicer than f32::default() and f64::default()?
         let d_ret_ty = match d_sig.decl.output {
             FnRetTy::Ty(ref ty) => ty.clone(),
             FnRetTy::Default(span) => {
                 panic!("Did not expect Default ret ty: {:?}", span);
             }
         };
-        let mut d_ret_ty = match d_ret_ty.kind.clone() {
-            TyKind::Tup(ref tys) => tys.clone(),
-            TyKind::Path(_, rustc_ast::Path { segments, .. }) => {
-                if let [segment] = &segments[..]
-                    && segment.args.is_none()
-                {
-                    let id = vec![segments[0].ident];
-                    let kind = TyKind::Path(None, ecx.path(span, id));
-                    let ty = P(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None });
-                    thin_vec![ty]
-                } else {
-                    panic!("Expected tuple or simple path return type");
-                }
-            }
-            _ => {
-                // We messed up construction of d_sig
-                panic!("Did not expect non-tuple ret ty: {:?}", d_ret_ty);
-            }
-        };
 
-        if x.mode.is_fwd() && x.ret_activity == DiffActivity::Dual {
+        if x.mode.is_fwd() {
-            assert!(d_ret_ty.len() == 2);
+            // Fwd mode is easy. If the return activity is Const, we support arbitrary types.
-            // both should be identical, by construction
+            // Otherwise, we only support a scalar, a pair of scalars, or an array of scalars.
-            let arg = d_ret_ty[0].kind.is_simple_path().unwrap();
+            // We checked that (on a best-effort base) in the preceding gen_enzyme_decl function.
-            let arg2 = d_ret_ty[1].kind.is_simple_path().unwrap();
+            // In all three cases, we can return `std::hint::black_box(<T>::default())`.
-            assert!(arg == arg2);
+            if x.ret_activity == DiffActivity::Const {
-            let sl: Vec<Symbol> = vec![arg, kw::Default];
+                // Here we call the primal function, since our dummy function has the same return
-            let tmp = ecx.def_site_path(&sl);
+                // type due to the Const return activity.
-            let default_call_expr = ecx.expr_path(ecx.path(span, tmp));
+                exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]);
-            let default_call_expr = ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]);
+            } else {
-            exprs.push(default_call_expr);
+                let q = QSelf { ty: d_ret_ty.clone(), path_span: span, position: 0 };
-        } else if x.mode.is_rev() {
+                let y =
-            if primal_ret {
+                    ExprKind::Path(Some(P(q)), ecx.path_ident(span, Ident::from_str("default")));
-                // We have extra handling above for the primal ret
+                let default_call_expr = ecx.expr(span, y);
-                d_ret_ty = d_ret_ty[1..].to_vec().into();
-            }
-
-            for arg in d_ret_ty.iter() {
-                let arg = arg.kind.is_simple_path().unwrap();
-                let sl: Vec<Symbol> = vec![arg, kw::Default];
-                let tmp = ecx.def_site_path(&sl);
-                let default_call_expr = ecx.expr_path(ecx.path(span, tmp));
                 let default_call_expr =
                     ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]);
-                exprs.push(default_call_expr);
+                exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![default_call_expr]);
             }
+        } else if x.mode.is_rev() {
+            if x.width == 1 {
+                // We either have `-> ArbitraryType` or `-> (ArbitraryType, repeated_float_scalars)`.
+                match d_ret_ty.kind {
+                    TyKind::Tup(ref args) => {
+                        // We have a tuple return type. We need to create a tuple of the same size
+                        // and fill it with default values.
+                        let mut exprs2 = thin_vec![exprs];
+                        for arg in args.iter().skip(1) {
+                            let arg = arg.kind.is_simple_path().unwrap();
+                            let sl: Vec<Symbol> = vec![arg, kw::Default];
+                            let tmp = ecx.def_site_path(&sl);
+                            let default_call_expr = ecx.expr_path(ecx.path(span, tmp));
+                            let default_call_expr =
+                                ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]);
+                            exprs2.push(default_call_expr);
+                        }
+                        exprs = ecx.expr_tuple(new_decl_span, exprs2);
+                    }
+                    _ => {
+                        // Interestingly, even the `-> ArbitraryType` case
+                        // ends up getting matched and handled correctly above,
+                        // so we don't have to handle any other case for now.
+                        panic!("Unsupported return type: {:?}", d_ret_ty);
+                    }
+                }
+            }
+            exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]);
+        } else {
+            unreachable!("Unsupported mode: {:?}", x.mode);
         }
 
-        let ret: P<ast::Expr>;
+        body.stmts.push(ecx.stmt_expr(exprs));
-        match &exprs[..] {
-            [] => {
-                assert!(!has_ret(&d_sig.decl.output));
-                // We don't have to match the return type.
-                return body;
-            }
-            [arg] => {
-                ret = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![arg.clone()]);
-            }
-            args => {
-                let ret_tuple: P<ast::Expr> = ecx.expr_tuple(span, args.into());
-                ret = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![ret_tuple]);
-            }
-        }
-        assert!(has_ret(&d_sig.decl.output));
-        body.stmts.push(ecx.stmt_expr(ret));
 
         body
     }
@@ -684,50 +741,55 @@ mod llvm_enzyme {
             match activity {
                 DiffActivity::Active => {
                     act_ret.push(arg.ty.clone());
+                    // if width =/= 1, then push [arg.ty; width] to act_ret
                 }
                 DiffActivity::ActiveOnly => {
                     // We will add the active scalar to the return type.
                     // This is handled later.
                 }
                 DiffActivity::Duplicated | DiffActivity::DuplicatedOnly => {
-                    let mut shadow_arg = arg.clone();
+                    for i in 0..x.width {
-                    // We += into the shadow in reverse mode.
+                        let mut shadow_arg = arg.clone();
-                    shadow_arg.ty = P(assure_mut_ref(&arg.ty));
+                        // We += into the shadow in reverse mode.
-                    let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
+                        shadow_arg.ty = P(assure_mut_ref(&arg.ty));
-                        ident.name
+                        let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
-                    } else {
+                            ident.name
-                        debug!("{:#?}", &shadow_arg.pat);
+                        } else {
-                        panic!("not an ident?");
+                            debug!("{:#?}", &shadow_arg.pat);
-                    };
+                            panic!("not an ident?");
-                    let name: String = format!("d{}", old_name);
+                        };
-                    new_inputs.push(name.clone());
+                        let name: String = format!("d{}_{}", old_name, i);
-                    let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
+                        new_inputs.push(name.clone());
-                    shadow_arg.pat = P(ast::Pat {
+                        let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
-                        id: ast::DUMMY_NODE_ID,
+                        shadow_arg.pat = P(ast::Pat {
-                        kind: PatKind::Ident(BindingMode::NONE, ident, None),
+                            id: ast::DUMMY_NODE_ID,
-                        span: shadow_arg.pat.span,
+                            kind: PatKind::Ident(BindingMode::NONE, ident, None),
-                        tokens: shadow_arg.pat.tokens.clone(),
+                            span: shadow_arg.pat.span,
-                    });
+                            tokens: shadow_arg.pat.tokens.clone(),
-                    d_inputs.push(shadow_arg);
+                        });
+                        d_inputs.push(shadow_arg.clone());
+                    }
                 }
                 DiffActivity::Dual | DiffActivity::DualOnly => {
-                    let mut shadow_arg = arg.clone();
+                    for i in 0..x.width {
-                    let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
+                        let mut shadow_arg = arg.clone();
-                        ident.name
+                        let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind {
-                    } else {
+                            ident.name
-                        debug!("{:#?}", &shadow_arg.pat);
+                        } else {
-                        panic!("not an ident?");
+                            debug!("{:#?}", &shadow_arg.pat);
-                    };
+                            panic!("not an ident?");
-                    let name: String = format!("b{}", old_name);
+                        };
-                    new_inputs.push(name.clone());
+                        let name: String = format!("b{}_{}", old_name, i);
-                    let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
+                        new_inputs.push(name.clone());
-                    shadow_arg.pat = P(ast::Pat {
+                        let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span);
-                        id: ast::DUMMY_NODE_ID,
+                        shadow_arg.pat = P(ast::Pat {
-                        kind: PatKind::Ident(BindingMode::NONE, ident, None),
+                            id: ast::DUMMY_NODE_ID,
-                        span: shadow_arg.pat.span,
+                            kind: PatKind::Ident(BindingMode::NONE, ident, None),
-                        tokens: shadow_arg.pat.tokens.clone(),
+                            span: shadow_arg.pat.span,
-                    });
+                            tokens: shadow_arg.pat.tokens.clone(),
-                    d_inputs.push(shadow_arg);
+                        });
+                        d_inputs.push(shadow_arg.clone());
+                    }
                 }
                 DiffActivity::Const => {
                     // Nothing to do here.
@@ -783,23 +845,48 @@ mod llvm_enzyme {
         d_decl.inputs = d_inputs.into();
 
         if x.mode.is_fwd() {
+            let ty = match d_decl.output {
+                FnRetTy::Ty(ref ty) => ty.clone(),
+                FnRetTy::Default(span) => {
+                    // We want to return std::hint::black_box(()).
+                    let kind = TyKind::Tup(ThinVec::new());
+                    let ty = P(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None });
+                    d_decl.output = FnRetTy::Ty(ty.clone());
+                    assert!(matches!(x.ret_activity, DiffActivity::None));
+                    // this won't be used below, so any type would be fine.
+                    ty
+                }
+            };
+
             if let DiffActivity::Dual = x.ret_activity {
-                let ty = match d_decl.output {
+                let kind = if x.width == 1 {
-                    FnRetTy::Ty(ref ty) => ty.clone(),
+                    // Dual can only be used for f32/f64 ret.
-                    FnRetTy::Default(span) => {
+                    // In that case we return now a tuple with two floats.
-                        panic!("Did not expect Default ret ty: {:?}", span);
+                    TyKind::Tup(thin_vec![ty.clone(), ty.clone()])
-                    }
+                } else {
+                    // We have to return [T; width+1], +1 for the primal return.
+                    let anon_const = rustc_ast::AnonConst {
+                        id: ast::DUMMY_NODE_ID,
+                        value: ecx.expr_usize(span, 1 + x.width as usize),
+                    };
+                    TyKind::Array(ty.clone(), anon_const)
                 };
-                // Dual can only be used for f32/f64 ret.
-                // In that case we return now a tuple with two floats.
-                let kind = TyKind::Tup(thin_vec![ty.clone(), ty.clone()]);
                 let ty = P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None });
                 d_decl.output = FnRetTy::Ty(ty);
             }
             if let DiffActivity::DualOnly = x.ret_activity {
                 // No need to change the return type,
-                // we will just return the shadow in place
+                // we will just return the shadow in place of the primal return.
-                // of the primal return.
+                // However, if we have a width > 1, then we don't return -> T, but -> [T; width]
+                if x.width > 1 {
+                    let anon_const = rustc_ast::AnonConst {
+                        id: ast::DUMMY_NODE_ID,
+                        value: ecx.expr_usize(span, x.width as usize),
+                    };
+                    let kind = TyKind::Array(ty.clone(), anon_const);
+                    let ty = P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None });
+                    d_decl.output = FnRetTy::Ty(ty);
+                }
             }
         }
 

compiler/rustc_builtin_macros/src/errors.rs

@@ -202,6 +202,14 @@ mod autodiff {
         pub(crate) mode: String,
     }
 
+    #[derive(Diagnostic)]
+    #[diag(builtin_macros_autodiff_width)]
+    pub(crate) struct AutoDiffInvalidWidth {
+        #[primary_span]
+        pub(crate) span: Span,
+        pub(crate) width: u128,
+    }
+
     #[derive(Diagnostic)]
     #[diag(builtin_macros_autodiff)]
     pub(crate) struct AutoDiffInvalidApplication {

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -610,6 +610,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff], module: &mut ModuleCodegen<
             }
             // We handle this below
             config::AutoDiff::PrintModAfter => {}
+            // We handle this below
+            config::AutoDiff::PrintModFinal => {}
             // This is required and already checked
             config::AutoDiff::Enable => {}
         }
@@ -657,14 +659,20 @@ pub(crate) fn run_pass_manager(
     }
 
     if cfg!(llvm_enzyme) && enable_ad {
+        // This is the post-autodiff IR, mainly used for testing and educational purposes.
+        if config.autodiff.contains(&config::AutoDiff::PrintModAfter) {
+            unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
+        }
+
         let opt_stage = llvm::OptStage::FatLTO;
         let stage = write::AutodiffStage::PostAD;
         unsafe {
             write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?;
         }
 
-        // This is the final IR, so people should be able to inspect the optimized autodiff output.
+        // This is the final IR, so people should be able to inspect the optimized autodiff output,
-        if config.autodiff.contains(&config::AutoDiff::PrintModAfter) {
+        // for manual inspection.
+        if config.autodiff.contains(&config::AutoDiff::PrintModFinal) {
             unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
         }
     }

compiler/rustc_codegen_llvm/src/builder/autodiff.rs

@@ -3,8 +3,10 @@ use std::ptr;
 use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode};
 use rustc_codegen_ssa::ModuleCodegen;
 use rustc_codegen_ssa::back::write::ModuleConfig;
-use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _;
+use rustc_codegen_ssa::common::TypeKind;
+use rustc_codegen_ssa::traits::BaseTypeCodegenMethods;
 use rustc_errors::FatalError;
+use rustc_middle::bug;
 use tracing::{debug, trace};
 
 use crate::back::write::llvm_err;
@@ -18,21 +20,42 @@ use crate::value::Value;
 use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm};
 
 fn get_params(fnc: &Value) -> Vec<&Value> {
+    let param_num = llvm::LLVMCountParams(fnc) as usize;
+    let mut fnc_args: Vec<&Value> = vec![];
+    fnc_args.reserve(param_num);
     unsafe {
-        let param_num = llvm::LLVMCountParams(fnc) as usize;
-        let mut fnc_args: Vec<&Value> = vec![];
-        fnc_args.reserve(param_num);
         llvm::LLVMGetParams(fnc, fnc_args.as_mut_ptr());
         fnc_args.set_len(param_num);
-        fnc_args
+    }
+    fnc_args
+}
+
+fn has_sret(fnc: &Value) -> bool {
+    let num_args = llvm::LLVMCountParams(fnc) as usize;
+    if num_args == 0 {
+        false
+    } else {
+        unsafe { llvm::LLVMRustHasAttributeAtIndex(fnc, 0, llvm::AttributeKind::StructRet) }
     }
 }
 
+// When we call the `__enzyme_autodiff` or `__enzyme_fwddiff` function, we need to pass all the
+// original inputs, as well as metadata and the additional shadow arguments.
+// This function matches the arguments from the outer function to the inner enzyme call.
+//
+// This function also considers that Rust level arguments not always match the llvm-ir level
+// arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on
+// llvm-ir level. The number of activities matches the number of Rust level arguments, so we
+// need to match those.
+// FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it
+// using iterators and peek()?
 fn match_args_from_caller_to_enzyme<'ll>(
     cx: &SimpleCx<'ll>,
+    width: u32,
     args: &mut Vec<&'ll llvm::Value>,
     inputs: &[DiffActivity],
     outer_args: &[&'ll llvm::Value],
+    has_sret: bool,
 ) {
     debug!("matching autodiff arguments");
     // We now handle the issue that Rust level arguments not always match the llvm-ir level
@@ -44,6 +67,14 @@ fn match_args_from_caller_to_enzyme<'ll>(
     let mut outer_pos: usize = 0;
     let mut activity_pos = 0;
 
+    if has_sret {
+        // Then the first outer arg is the sret pointer. Enzyme doesn't know about sret, so the
+        // inner function will still return something. We increase our outer_pos by one,
+        // and once we're done with all other args we will take the return of the inner call and
+        // update the sret pointer with it
+        outer_pos = 1;
+    }
+
     let enzyme_const = cx.create_metadata("enzyme_const".to_string()).unwrap();
     let enzyme_out = cx.create_metadata("enzyme_out".to_string()).unwrap();
     let enzyme_dup = cx.create_metadata("enzyme_dup".to_string()).unwrap();
@@ -92,23 +123,20 @@ fn match_args_from_caller_to_enzyme<'ll>(
                 // (..., metadata! enzyme_dup, ptr, ptr, int1, ...).
                 // FIXME(ZuseZ4): We will upstream a safety check later which asserts that
                 // int2 >= int1, which means the shadow vector is large enough to store the gradient.
-                assert!(unsafe {
+                assert_eq!(cx.type_kind(next_outer_ty), TypeKind::Integer);
-                    llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Integer
+
-                });
+                for i in 0..(width as usize) {
-                let next_outer_arg2 = outer_args[outer_pos + 2];
+                    let next_outer_arg2 = outer_args[outer_pos + 2 * (i + 1)];
-                let next_outer_ty2 = cx.val_ty(next_outer_arg2);
+                    let next_outer_ty2 = cx.val_ty(next_outer_arg2);
-                assert!(unsafe {
+                    assert_eq!(cx.type_kind(next_outer_ty2), TypeKind::Pointer);
-                    llvm::LLVMRustGetTypeKind(next_outer_ty2) == llvm::TypeKind::Pointer
+                    let next_outer_arg3 = outer_args[outer_pos + 2 * (i + 1) + 1];
-                });
+                    let next_outer_ty3 = cx.val_ty(next_outer_arg3);
-                let next_outer_arg3 = outer_args[outer_pos + 3];
+                    assert_eq!(cx.type_kind(next_outer_ty3), TypeKind::Integer);
-                let next_outer_ty3 = cx.val_ty(next_outer_arg3);
+                    args.push(next_outer_arg2);
-                assert!(unsafe {
+                }
-                    llvm::LLVMRustGetTypeKind(next_outer_ty3) == llvm::TypeKind::Integer
-                });
-                args.push(next_outer_arg2);
                 args.push(cx.get_metadata_value(enzyme_const));
                 args.push(next_outer_arg);
-                outer_pos += 4;
+                outer_pos += 2 + 2 * width as usize;
                 activity_pos += 2;
             } else {
                 // A duplicated pointer will have the following two outer_fn arguments:
@@ -116,15 +144,19 @@ fn match_args_from_caller_to_enzyme<'ll>(
                 // (..., metadata! enzyme_dup, ptr, ptr, ...).
                 if matches!(diff_activity, DiffActivity::Duplicated | DiffActivity::DuplicatedOnly)
                 {
-                    assert!(
+                    assert_eq!(cx.type_kind(next_outer_ty), TypeKind::Pointer);
-                        unsafe { llvm::LLVMRustGetTypeKind(next_outer_ty) }
-                            == llvm::TypeKind::Pointer
-                    );
                 }
                 // In the case of Dual we don't have assumptions, e.g. f32 would be valid.
                 args.push(next_outer_arg);
                 outer_pos += 2;
                 activity_pos += 1;
+
+                // Now, if width > 1, we need to account for that
+                for _ in 1..width {
+                    let next_outer_arg = outer_args[outer_pos];
+                    args.push(next_outer_arg);
+                    outer_pos += 1;
+                }
             }
         } else {
             // We do not differentiate with resprect to this argument.
@@ -135,6 +167,76 @@ fn match_args_from_caller_to_enzyme<'ll>(
     }
 }
 
+// On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input
+// arguments. We do however need to declare them with their correct return type.
+// We already figured the correct return type out in our frontend, when generating the outer_fn,
+// so we can now just go ahead and use that. This is not always trivial, e.g. because sret.
+// Beyond sret, this article describes our challenges nicely:
+// <https://yorickpeterse.com/articles/the-mess-that-is-handling-structure-arguments-and-returns-in-llvm/>
+// I.e. (i32, f32) will get merged into i64, but we don't handle that yet.
+fn compute_enzyme_fn_ty<'ll>(
+    cx: &SimpleCx<'ll>,
+    attrs: &AutoDiffAttrs,
+    fn_to_diff: &'ll Value,
+    outer_fn: &'ll Value,
+) -> &'ll llvm::Type {
+    let fn_ty = cx.get_type_of_global(outer_fn);
+    let mut ret_ty = cx.get_return_type(fn_ty);
+
+    let has_sret = has_sret(outer_fn);
+
+    if has_sret {
+        // Now we don't just forward the return type, so we have to figure it out based on the
+        // primal return type, in combination with the autodiff settings.
+        let fn_ty = cx.get_type_of_global(fn_to_diff);
+        let inner_ret_ty = cx.get_return_type(fn_ty);
+
+        let void_ty = unsafe { llvm::LLVMVoidTypeInContext(cx.llcx) };
+        if inner_ret_ty == void_ty {
+            // This indicates that even the inner function has an sret.
+            // Right now I only look for an sret in the outer function.
+            // This *probably* needs some extra handling, but I never ran
+            // into such a case. So I'll wait for user reports to have a test case.
+            bug!("sret in inner function");
+        }
+
+        if attrs.width == 1 {
+            todo!("Handle sret for scalar ad");
+        } else {
+            // First we check if we also have to deal with the primal return.
+            match attrs.mode {
+                DiffMode::Forward => match attrs.ret_activity {
+                    DiffActivity::Dual => {
+                        let arr_ty =
+                            unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64 + 1) };
+                        ret_ty = arr_ty;
+                    }
+                    DiffActivity::DualOnly => {
+                        let arr_ty =
+                            unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64) };
+                        ret_ty = arr_ty;
+                    }
+                    DiffActivity::Const => {
+                        todo!("Not sure, do we need to do something here?");
+                    }
+                    _ => {
+                        bug!("unreachable");
+                    }
+                },
+                DiffMode::Reverse => {
+                    todo!("Handle sret for reverse mode");
+                }
+                _ => {
+                    bug!("unreachable");
+                }
+            }
+        }
+    }
+
+    // LLVM can figure out the input types on it's own, so we take a shortcut here.
+    unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) }
+}
+
 /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another
 /// function with expected naming and calling conventions[^1] which will be
 /// discovered by the enzyme LLVM pass and its body populated with the differentiated
@@ -197,17 +299,9 @@ fn generate_enzyme_call<'ll>(
     // }
     // ```
     unsafe {
-        // On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input
+        let enzyme_ty = compute_enzyme_fn_ty(cx, &attrs, fn_to_diff, outer_fn);
-        // arguments. We do however need to declare them with their correct return type.
-        // We already figured the correct return type out in our frontend, when generating the outer_fn,
-        // so we can now just go ahead and use that. FIXME(ZuseZ4): This doesn't handle sret yet.
-        let fn_ty = llvm::LLVMGlobalGetValueType(outer_fn);
-        let ret_ty = llvm::LLVMGetReturnType(fn_ty);
 
-        // LLVM can figure out the input types on it's own, so we take a shortcut here.
+        // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and
-        let enzyme_ty = llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True);
-
-        //FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and
         // think a bit more about what should go here.
         let cc = llvm::LLVMGetFunctionCallConv(outer_fn);
         let ad_fn = declare_simple_fn(
@@ -240,14 +334,27 @@ fn generate_enzyme_call<'ll>(
         if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) {
             args.push(cx.get_metadata_value(enzyme_primal_ret));
         }
+        if attrs.width > 1 {
+            let enzyme_width = cx.create_metadata("enzyme_width".to_string()).unwrap();
+            args.push(cx.get_metadata_value(enzyme_width));
+            args.push(cx.get_const_i64(attrs.width as u64));
+        }
 
+        let has_sret = has_sret(outer_fn);
         let outer_args: Vec<&llvm::Value> = get_params(outer_fn);
-        match_args_from_caller_to_enzyme(&cx, &mut args, &attrs.input_activity, &outer_args);
+        match_args_from_caller_to_enzyme(
+            &cx,
+            attrs.width,
+            &mut args,
+            &attrs.input_activity,
+            &outer_args,
+            has_sret,
+        );
 
         let call = builder.call(enzyme_ty, ad_fn, &args, None);
 
         // This part is a bit iffy. LLVM requires that a call to an inlineable function has some
-        // metadata attachted to it, but we just created this code oota. Given that the
+        // metadata attached to it, but we just created this code oota. Given that the
         // differentiated function already has partly confusing metadata, and given that this
         // affects nothing but the auttodiff IR, we take a shortcut and just steal metadata from the
         // dummy code which we inserted at a higher level.
@@ -268,7 +375,22 @@ fn generate_enzyme_call<'ll>(
         // Now that we copied the metadata, get rid of dummy code.
         llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst);
 
-        if cx.val_ty(call) == cx.type_void() {
+        if cx.val_ty(call) == cx.type_void() || has_sret {
+            if has_sret {
+                // This is what we already have in our outer_fn (shortened):
+                // define void @_foo(ptr <..> sret([32 x i8]) initializes((0, 32)) %0, <...>) {
+                //   %7 = call [4 x double] (...) @__enzyme_fwddiff_foo(ptr @square, metadata !"enzyme_width", i64 4, <...>)
+                //   <Here we are, we want to add the following two lines>
+                //   store [4 x double] %7, ptr %0, align 8
+                //   ret void
+                // }
+
+                // now store the result of the enzyme call into the sret pointer.
+                let sret_ptr = outer_args[0];
+                let call_ty = cx.val_ty(call);
+                assert_eq!(cx.type_kind(call_ty), TypeKind::Array);
+                llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr);
+            }
             builder.ret_void();
         } else {
             builder.ret(call);
@@ -300,8 +422,7 @@ pub(crate) fn differentiate<'ll>(
     if !diff_items.is_empty()
         && !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable)
     {
-        let dcx = cgcx.create_dcx();
+        return Err(diag_handler.handle().emit_almost_fatal(AutoDiffWithoutEnable));
-        return Err(dcx.handle().emit_almost_fatal(AutoDiffWithoutEnable));
     }
 
     // Before dumping the module, we want all the TypeTrees to become part of the module.

compiler/rustc_codegen_llvm/src/consts.rs

@@ -430,7 +430,7 @@ impl<'ll> CodegenCx<'ll, '_> {
             let val_llty = self.val_ty(v);
 
             let g = self.get_static_inner(def_id, val_llty);
-            let llty = llvm::LLVMGlobalGetValueType(g);
+            let llty = self.get_type_of_global(g);
 
             let g = if val_llty == llty {
                 g

compiler/rustc_codegen_llvm/src/context.rs

@@ -8,6 +8,7 @@ use std::str;
 use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx};
 use rustc_codegen_ssa::back::versioned_llvm_target;
 use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh};
+use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::errors as ssa_errors;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN};
@@ -38,7 +39,7 @@ use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
 use crate::llvm::Metadata;
 use crate::type_::Type;
 use crate::value::Value;
-use crate::{attributes, coverageinfo, debuginfo, llvm, llvm_util};
+use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util};
 
 /// `TyCtxt` (and related cache datastructures) can't be move between threads.
 /// However, there are various cx related functions which we want to be available to the builder and
@@ -643,7 +644,18 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         llvm::set_section(g, c"llvm.metadata");
     }
 }
-
+impl<'ll> SimpleCx<'ll> {
+    pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type {
+        assert_eq!(self.type_kind(ty), TypeKind::Function);
+        unsafe { llvm::LLVMGetReturnType(ty) }
+    }
+    pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type {
+        unsafe { llvm::LLVMGlobalGetValueType(val) }
+    }
+    pub(crate) fn val_ty(&self, v: &'ll Value) -> &'ll Type {
+        common::val_ty(v)
+    }
+}
 impl<'ll> SimpleCx<'ll> {
     pub(crate) fn new(
         llmod: &'ll llvm::Module,
@@ -660,6 +672,13 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
         llvm::LLVMMetadataAsValue(self.llcx(), metadata)
     }
 
+    // FIXME(autodiff): We should split `ConstCodegenMethods` to pull the reusable parts
+    // onto a trait that is also implemented for GenericCx.
+    pub(crate) fn get_const_i64(&self, n: u64) -> &'ll Value {
+        let ty = unsafe { llvm::LLVMInt64TypeInContext(self.llcx()) };
+        unsafe { llvm::LLVMConstInt(ty, n, llvm::False) }
+    }
+
     pub(crate) fn get_function(&self, name: &str) -> Option<&'ll Value> {
         let name = SmallCStr::new(name);
         unsafe { llvm::LLVMGetNamedFunction((**self).borrow().llmod, name.as_ptr()) }

compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs

@@ -4,7 +4,7 @@
 use libc::{c_char, c_uint};
 
 use super::MetadataKindId;
-use super::ffi::{BasicBlock, Metadata, Module, Type, Value};
+use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value};
 use crate::llvm::Bool;
 
 #[link(name = "llvm-wrapper", kind = "static")]
@@ -17,6 +17,8 @@ unsafe extern "C" {
     pub(crate) fn LLVMRustEraseInstFromParent(V: &Value);
     pub(crate) fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value;
     pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool;
+    pub(crate) fn LLVMRustHasAttributeAtIndex(V: &Value, i: c_uint, Kind: AttributeKind) -> bool;
+    pub(crate) fn LLVMRustGetArrayNumElements(Ty: &Type) -> u64;
 }
 
 unsafe extern "C" {

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1180,7 +1180,7 @@ unsafe extern "C" {
 
     // Operations on parameters
     pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
-    pub(crate) fn LLVMCountParams(Fn: &Value) -> c_uint;
+    pub(crate) safe fn LLVMCountParams(Fn: &Value) -> c_uint;
     pub(crate) fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value;
 
     // Operations on basic blocks

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -2,7 +2,7 @@ use std::str::FromStr;
 
 use rustc_abi::ExternAbi;
 use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode};
-use rustc_ast::{MetaItem, MetaItemInner, attr};
+use rustc_ast::{LitKind, MetaItem, MetaItemInner, attr};
 use rustc_attr_parsing::ReprAttr::ReprAlign;
 use rustc_attr_parsing::{AttributeKind, InlineAttr, InstructionSetAttr, OptimizeAttr};
 use rustc_data_structures::fx::FxHashMap;
@@ -805,8 +805,8 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
         return Some(AutoDiffAttrs::source());
     }
 
-    let [mode, input_activities @ .., ret_activity] = &list[..] else {
+    let [mode, width_meta, input_activities @ .., ret_activity] = &list[..] else {
-        span_bug!(attr.span(), "rustc_autodiff attribute must contain mode and activities");
+        span_bug!(attr.span(), "rustc_autodiff attribute must contain mode, width and activities");
     };
     let mode = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = mode {
         p1.segments.first().unwrap().ident
@@ -823,6 +823,30 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
         }
     };
 
+    let width: u32 = match width_meta {
+        MetaItemInner::MetaItem(MetaItem { path: p1, .. }) => {
+            let w = p1.segments.first().unwrap().ident;
+            match w.as_str().parse() {
+                Ok(val) => val,
+                Err(_) => {
+                    span_bug!(w.span, "rustc_autodiff width should fit u32");
+                }
+            }
+        }
+        MetaItemInner::Lit(lit) => {
+            if let LitKind::Int(val, _) = lit.kind {
+                match val.get().try_into() {
+                    Ok(val) => val,
+                    Err(_) => {
+                        span_bug!(lit.span, "rustc_autodiff width should fit u32");
+                    }
+                }
+            } else {
+                span_bug!(lit.span, "rustc_autodiff width should be an integer");
+            }
+        }
+    };
+
     // First read the ret symbol from the attribute
     let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = ret_activity {
         p1.segments.first().unwrap().ident
@@ -860,7 +884,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
         }
     }
 
-    Some(AutoDiffAttrs { mode, ret_activity, input_activity: arg_activities })
+    Some(AutoDiffAttrs { mode, width, ret_activity, input_activity: arg_activities })
 }
 
 pub(crate) fn provide(providers: &mut Providers) {

compiler/rustc_driver_impl/src/lib.rs

@@ -64,6 +64,7 @@ use rustc_session::lint::{Lint, LintId};
 use rustc_session::output::{CRATE_TYPES, collect_crate_types, invalid_output_for_target};
 use rustc_session::{EarlyDiagCtxt, Session, config, filesearch};
 use rustc_span::FileName;
+use rustc_span::def_id::LOCAL_CRATE;
 use rustc_target::json::ToJson;
 use rustc_target::spec::{Target, TargetTuple};
 use time::OffsetDateTime;
@@ -392,14 +393,10 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
 }
 
 fn dump_feature_usage_metrics(tcxt: TyCtxt<'_>, metrics_dir: &Path) {
-    let output_filenames = tcxt.output_filenames(());
+    let hash = tcxt.crate_hash(LOCAL_CRATE);
-    let mut metrics_file_name = std::ffi::OsString::from("unstable_feature_usage_metrics-");
+    let crate_name = tcxt.crate_name(LOCAL_CRATE);
-    let mut metrics_path = output_filenames.with_directory_and_extension(metrics_dir, "json");
+    let metrics_file_name = format!("unstable_feature_usage_metrics-{crate_name}-{hash}.json");
-    let metrics_file_stem =
+    let metrics_path = metrics_dir.join(metrics_file_name);
-        metrics_path.file_name().expect("there should be a valid default output filename");
-    metrics_file_name.push(metrics_file_stem);
-    metrics_path.pop();
-    metrics_path.push(metrics_file_name);
     if let Err(error) = tcxt.features().dump_feature_usage_metrics(metrics_path) {
         // FIXME(yaahc): once metrics can be enabled by default we will want "failure to emit
         // default metrics" to only produce a warning when metrics are enabled by default and emit

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -384,6 +384,12 @@ static inline void AddAttributes(T *t, unsigned Index, LLVMAttributeRef *Attrs,
   t->setAttributes(PALNew);
 }
 
+extern "C" bool LLVMRustHasAttributeAtIndex(LLVMValueRef Fn, unsigned Index,
+                                            LLVMRustAttributeKind RustAttr) {
+  Function *F = unwrap<Function>(Fn);
+  return F->hasParamAttribute(Index, fromRust(RustAttr));
+}
+
 extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index,
                                               LLVMAttributeRef *Attrs,
                                               size_t AttrsLen) {
@@ -636,6 +642,10 @@ static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) {
   }
 }
 
+extern "C" uint64_t LLVMRustGetArrayNumElements(LLVMTypeRef Ty) {
+  return unwrap(Ty)->getArrayNumElements();
+}
+
 extern "C" LLVMValueRef
 LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen,
                   char *Constraints, size_t ConstraintsLen,

compiler/rustc_middle/src/ty/context.rs

@@ -1807,10 +1807,15 @@ impl<'tcx> TyCtxt<'tcx> {
         // - needs_metadata: for putting into crate metadata.
         // - instrument_coverage: for putting into coverage data (see
         //   `hash_mir_source`).
+        // - metrics_dir: metrics use the strict version hash in the filenames
+        //   for dumped metrics files to prevent overwriting distinct metrics
+        //   for similar source builds (may change in the future, this is part
+        //   of the proof of concept impl for the metrics initiative project goal)
         cfg!(debug_assertions)
             || self.sess.opts.incremental.is_some()
             || self.needs_metadata()
             || self.sess.instrument_coverage()
+            || self.sess.opts.unstable_opts.metrics_dir.is_some()
     }
 
     #[inline]

compiler/rustc_parse/src/parser/mod.rs

@@ -512,6 +512,14 @@ impl<'a> Parser<'a> {
         self
     }
 
+    #[inline]
+    fn with_recovery<T>(&mut self, recovery: Recovery, f: impl FnOnce(&mut Self) -> T) -> T {
+        let old = mem::replace(&mut self.recovery, recovery);
+        let res = f(self);
+        self.recovery = old;
+        res
+    }
+
     /// Whether the parser is allowed to recover from broken code.
     ///
     /// If this returns false, recovering broken code into valid code (especially if this recovery does lookahead)
@@ -770,7 +778,14 @@ impl<'a> Parser<'a> {
             && match_mv_kind(mv_kind)
         {
             self.bump();
-            let res = f(self).expect("failed to reparse {mv_kind:?}");
+
+            // Recovery is disabled when parsing macro arguments, so it must
+            // also be disabled when reparsing pasted macro arguments,
+            // otherwise we get inconsistent results (e.g. #137874).
+            let res = self.with_recovery(Recovery::Forbidden, |this| {
+                f(this).expect("failed to reparse {mv_kind:?}")
+            });
+
             if let token::CloseDelim(delim) = self.token.kind
                 && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
                 && match_mv_kind(mv_kind)

compiler/rustc_session/src/config.rs

@@ -237,10 +237,12 @@ pub enum AutoDiff {
     PrintPerf,
     /// Print intermediate IR generation steps
     PrintSteps,
-    /// Print the whole module, before running opts.
+    /// Print the module, before running autodiff.
     PrintModBefore,
-    /// Print the module after Enzyme differentiated everything.
+    /// Print the module after running autodiff.
     PrintModAfter,
+    /// Print the module after running autodiff and optimizations.
+    PrintModFinal,
 
     /// Enzyme's loose type debug helper (can cause incorrect gradients!!)
     /// Usable in cases where Enzyme errors with `can not deduce type of X`.
@@ -1425,10 +1427,12 @@ pub fn build_target_config(
             }
             target
         }
-        Err(e) => early_dcx.early_fatal(format!(
+        Err(e) => {
-            "Error loading target specification: {e}. \
+            let mut err =
-                     Run `rustc --print target-list` for a list of built-in targets"
+                early_dcx.early_struct_fatal(format!("error loading target specification: {e}"));
-        )),
+            err.help("run `rustc --print target-list` for a list of built-in targets");
+            err.emit();
+        }
     }
 }
 

compiler/rustc_session/src/options.rs

@@ -711,7 +711,7 @@ mod desc {
     pub(crate) const parse_list: &str = "a space-separated list of strings";
     pub(crate) const parse_list_with_polarity: &str =
         "a comma-separated list of strings, with elements beginning with + or -";
-    pub(crate) const parse_autodiff: &str = "a comma separated list of settings: `Enable`, `PrintSteps`, `PrintTA`, `PrintAA`, `PrintPerf`, `PrintModBefore`, `PrintModAfter`, `LooseTypes`, `Inline`";
+    pub(crate) const parse_autodiff: &str = "a comma separated list of settings: `Enable`, `PrintSteps`, `PrintTA`, `PrintAA`, `PrintPerf`, `PrintModBefore`, `PrintModAfter`, `PrintModFinal`, `LooseTypes`, `Inline`";
     pub(crate) const parse_comma_list: &str = "a comma-separated list of strings";
     pub(crate) const parse_opt_comma_list: &str = parse_comma_list;
     pub(crate) const parse_number: &str = "a number";
@@ -1359,6 +1359,7 @@ pub mod parse {
                 "PrintSteps" => AutoDiff::PrintSteps,
                 "PrintModBefore" => AutoDiff::PrintModBefore,
                 "PrintModAfter" => AutoDiff::PrintModAfter,
+                "PrintModFinal" => AutoDiff::PrintModFinal,
                 "LooseTypes" => AutoDiff::LooseTypes,
                 "Inline" => AutoDiff::Inline,
                 _ => {
@@ -2093,6 +2094,7 @@ options! {
         `=PrintSteps`
         `=PrintModBefore`
         `=PrintModAfter`
+        `=PrintModFinal`
         `=LooseTypes`
         `=Inline`
         Multiple options can be combined with commas."),

compiler/rustc_smir/src/rustc_internal/mod.rs

@@ -147,6 +147,14 @@ impl<'tcx> Tables<'tcx> {
         stable_mir::ty::CoroutineWitnessDef(self.create_def_id(did))
     }
 
+    pub fn assoc_def(&mut self, did: DefId) -> stable_mir::ty::AssocDef {
+        stable_mir::ty::AssocDef(self.create_def_id(did))
+    }
+
+    pub fn opaque_def(&mut self, did: DefId) -> stable_mir::ty::OpaqueDef {
+        stable_mir::ty::OpaqueDef(self.create_def_id(did))
+    }
+
     pub fn prov(&mut self, aid: AllocId) -> stable_mir::ty::Prov {
         stable_mir::ty::Prov(self.create_alloc_id(aid))
     }

compiler/rustc_smir/src/rustc_smir/context.rs

@@ -822,6 +822,21 @@ impl<'tcx> Context for TablesWrapper<'tcx> {
         let ty = un_op.internal(&mut *tables, tcx).ty(tcx, arg_internal);
         ty.stable(&mut *tables)
     }
+
+    fn associated_items(&self, def_id: stable_mir::DefId) -> stable_mir::AssocItems {
+        let mut tables = self.0.borrow_mut();
+        let tcx = tables.tcx;
+        let def_id = tables[def_id];
+        let assoc_items = if tcx.is_trait_alias(def_id) {
+            Vec::new()
+        } else {
+            tcx.associated_item_def_ids(def_id)
+                .iter()
+                .map(|did| tcx.associated_item(*did).stable(&mut *tables))
+                .collect()
+        };
+        assoc_items
+    }
 }
 
 pub(crate) struct TablesWrapper<'tcx>(pub RefCell<Tables<'tcx>>);

compiler/rustc_smir/src/rustc_smir/convert/ty.rs

@@ -890,3 +890,63 @@ impl<'tcx> Stable<'tcx> for rustc_session::cstore::ForeignModule {
         }
     }
 }
+
+impl<'tcx> Stable<'tcx> for ty::AssocKind {
+    type T = stable_mir::ty::AssocKind;
+
+    fn stable(&self, _tables: &mut Tables<'_>) -> Self::T {
+        use stable_mir::ty::AssocKind;
+        match self {
+            ty::AssocKind::Const => AssocKind::Const,
+            ty::AssocKind::Fn => AssocKind::Fn,
+            ty::AssocKind::Type => AssocKind::Type,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AssocItemContainer {
+    type T = stable_mir::ty::AssocItemContainer;
+
+    fn stable(&self, _tables: &mut Tables<'_>) -> Self::T {
+        use stable_mir::ty::AssocItemContainer;
+        match self {
+            ty::AssocItemContainer::Trait => AssocItemContainer::Trait,
+            ty::AssocItemContainer::Impl => AssocItemContainer::Impl,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AssocItem {
+    type T = stable_mir::ty::AssocItem;
+
+    fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
+        stable_mir::ty::AssocItem {
+            def_id: tables.assoc_def(self.def_id),
+            name: self.name.to_string(),
+            kind: self.kind.stable(tables),
+            container: self.container.stable(tables),
+            trait_item_def_id: self.trait_item_def_id.map(|did| tables.assoc_def(did)),
+            fn_has_self_parameter: self.fn_has_self_parameter,
+            opt_rpitit_info: self.opt_rpitit_info.map(|rpitit| rpitit.stable(tables)),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ImplTraitInTraitData {
+    type T = stable_mir::ty::ImplTraitInTraitData;
+
+    fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
+        use stable_mir::ty::ImplTraitInTraitData;
+        match self {
+            ty::ImplTraitInTraitData::Trait { fn_def_id, opaque_def_id } => {
+                ImplTraitInTraitData::Trait {
+                    fn_def_id: tables.fn_def(*fn_def_id),
+                    opaque_def_id: tables.opaque_def(*opaque_def_id),
+                }
+            }
+            ty::ImplTraitInTraitData::Impl { fn_def_id } => {
+                ImplTraitInTraitData::Impl { fn_def_id: tables.fn_def(*fn_def_id) }
+            }
+        }
+    }
+}

compiler/rustc_target/src/spec/mod.rs

@@ -3516,7 +3516,7 @@ impl Target {
                     Err("the `i586-pc-windows-msvc` target has been removed. Use the `i686-pc-windows-msvc` target instead.\n\
                         Windows 10 (the minimum required OS version) requires a CPU baseline of at least i686 so you can safely switch".into())
                 } else {
-                    Err(format!("Could not find specification for target {target_tuple:?}"))
+                    Err(format!("could not find specification for target {target_tuple:?}"))
                 }
             }
             TargetTuple::TargetJson { ref contents, .. } => {

compiler/stable_mir/src/compiler_interface.rs

@@ -18,8 +18,8 @@ use crate::ty::{
     TraitDef, Ty, TyConst, TyConstId, TyKind, UintTy, VariantDef,
 };
 use crate::{
-    Crate, CrateItem, CrateItems, CrateNum, DefId, Error, Filename, ImplTraitDecls, ItemKind,
+    AssocItems, Crate, CrateItem, CrateItems, CrateNum, DefId, Error, Filename, ImplTraitDecls,
-    Symbol, TraitDecls, mir,
+    ItemKind, Symbol, TraitDecls, mir,
 };
 
 /// This trait defines the interface between stable_mir and the Rust compiler.
@@ -251,6 +251,9 @@ pub trait Context {
 
     /// Get the resulting type of unary operation.
     fn unop_ty(&self, un_op: UnOp, arg: Ty) -> Ty;
+
+    /// Get all associated items of a definition.
+    fn associated_items(&self, def_id: DefId) -> AssocItems;
 }
 
 // A thread local variable that stores a pointer to the tables mapping between TyCtxt

compiler/stable_mir/src/crate_def.rs

@@ -4,7 +4,7 @@
 use serde::Serialize;
 
 use crate::ty::{GenericArgs, Span, Ty};
-use crate::{Crate, Symbol, with};
+use crate::{AssocItems, Crate, Symbol, with};
 
 /// A unique identification number for each item accessible for the current compilation unit.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize)]
@@ -103,6 +103,14 @@ pub trait CrateDefType: CrateDef {
     }
 }
 
+/// A trait for retrieving all items from a definition within a crate.
+pub trait CrateDefItems: CrateDef {
+    /// Retrieve all associated items from a definition.
+    fn associated_items(&self) -> AssocItems {
+        with(|cx| cx.associated_items(self.def_id()))
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct Attribute {
     value: String,
@@ -158,3 +166,9 @@ macro_rules! crate_def_with_ty {
         impl CrateDefType for $name {}
     };
 }
+
+macro_rules! impl_crate_def_items {
+    ( $name:ident $(;)? ) => {
+        impl CrateDefItems for $name {}
+    };
+}

compiler/stable_mir/src/lib.rs

@@ -23,11 +23,11 @@ use std::{fmt, io};
 use serde::Serialize;
 
 use crate::compiler_interface::with;
-pub use crate::crate_def::{CrateDef, CrateDefType, DefId};
+pub use crate::crate_def::{CrateDef, CrateDefItems, CrateDefType, DefId};
 pub use crate::error::*;
 use crate::mir::mono::StaticDef;
 use crate::mir::{Body, Mutability};
-use crate::ty::{FnDef, ForeignModuleDef, ImplDef, IndexedVal, Span, TraitDef, Ty};
+use crate::ty::{AssocItem, FnDef, ForeignModuleDef, ImplDef, IndexedVal, Span, TraitDef, Ty};
 
 pub mod abi;
 #[macro_use]
@@ -71,6 +71,9 @@ pub type TraitDecls = Vec<TraitDef>;
 /// A list of impl trait decls.
 pub type ImplTraitDecls = Vec<ImplDef>;
 
+/// A list of associated items.
+pub type AssocItems = Vec<AssocItem>;
+
 /// Holds information about a crate.
 #[derive(Clone, PartialEq, Eq, Debug, Serialize)]
 pub struct Crate {

compiler/stable_mir/src/mir/pretty.rs

@@ -9,7 +9,7 @@ use super::{AggregateKind, AssertMessage, BinOp, BorrowKind, FakeBorrowKind, Ter
 use crate::mir::{
     Operand, Place, RawPtrKind, Rvalue, StatementKind, UnwindAction, VarDebugInfoContents,
 };
-use crate::ty::{AdtKind, IndexedVal, MirConst, Ty, TyConst};
+use crate::ty::{AdtKind, AssocKind, IndexedVal, MirConst, Ty, TyConst};
 use crate::{Body, CrateDef, Mutability, with};
 
 impl Display for Ty {
@@ -18,6 +18,16 @@ impl Display for Ty {
     }
 }
 
+impl Display for AssocKind {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            AssocKind::Fn => write!(f, "method"),
+            AssocKind::Const => write!(f, "associated const"),
+            AssocKind::Type => write!(f, "associated type"),
+        }
+    }
+}
+
 impl Debug for Place {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         with(|ctx| write!(f, "{}", ctx.place_pretty(self)))

compiler/stable_mir/src/ty.rs

@@ -6,7 +6,7 @@ use serde::Serialize;
 use super::mir::{Body, Mutability, Safety};
 use super::{DefId, Error, Symbol, with};
 use crate::abi::{FnAbi, Layout};
-use crate::crate_def::{CrateDef, CrateDefType};
+use crate::crate_def::{CrateDef, CrateDefItems, CrateDefType};
 use crate::mir::alloc::{AllocId, read_target_int, read_target_uint};
 use crate::mir::mono::StaticDef;
 use crate::target::MachineInfo;
@@ -910,6 +910,10 @@ crate_def! {
     pub TraitDef;
 }
 
+impl_crate_def_items! {
+    TraitDef;
+}
+
 impl TraitDef {
     pub fn declaration(trait_def: &TraitDef) -> TraitDecl {
         with(|cx| cx.trait_decl(trait_def))
@@ -932,6 +936,10 @@ crate_def! {
     pub ImplDef;
 }
 
+impl_crate_def_items! {
+    ImplDef;
+}
+
 impl ImplDef {
     /// Retrieve information about this implementation.
     pub fn trait_impl(&self) -> ImplTrait {
@@ -1555,3 +1563,60 @@ index_impl!(Span);
 pub struct VariantIdx(usize);
 
 index_impl!(VariantIdx);
+
+crate_def! {
+    /// Hold infomation about an Opaque definition, particularly useful in `RPITIT`.
+    #[derive(Serialize)]
+    pub OpaqueDef;
+}
+
+crate_def! {
+    #[derive(Serialize)]
+    pub AssocDef;
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub struct AssocItem {
+    pub def_id: AssocDef,
+    pub name: Symbol,
+    pub kind: AssocKind,
+    pub container: AssocItemContainer,
+
+    /// If this is an item in an impl of a trait then this is the `DefId` of
+    /// the associated item on the trait that this implements.
+    pub trait_item_def_id: Option<AssocDef>,
+
+    /// Whether this is a method with an explicit self
+    /// as its first parameter, allowing method calls.
+    pub fn_has_self_parameter: bool,
+
+    /// `Some` if the associated item (an associated type) comes from the
+    /// return-position `impl Trait` in trait desugaring. The `ImplTraitInTraitData`
+    /// provides additional information about its source.
+    pub opt_rpitit_info: Option<ImplTraitInTraitData>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub enum AssocKind {
+    Const,
+    Fn,
+    Type,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub enum AssocItemContainer {
+    Trait,
+    Impl,
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Serialize)]
+pub enum ImplTraitInTraitData {
+    Trait { fn_def_id: FnDef, opaque_def_id: OpaqueDef },
+    Impl { fn_def_id: FnDef },
+}
+
+impl AssocItem {
+    pub fn is_impl_trait_in_trait(&self) -> bool {
+        self.opt_rpitit_info.is_some()
+    }
+}

library/alloctests/tests/lib.rs

@@ -63,6 +63,7 @@ mod fmt;
 mod heap;
 mod linked_list;
 mod misc_tests;
+mod num;
 mod rc;
 mod slice;
 mod sort;

library/alloctests/tests/num.rs

@@ -0,0 +1,69 @@
+use std::fmt::{Debug, Display};
+use std::str::FromStr;
+
+fn assert_nb<Int: ToString + FromStr + Debug + Display + Eq>(value: Int) {
+    let s = value.to_string();
+    let s2 = format!("s: {}.", value);
+
+    assert_eq!(format!("s: {s}."), s2);
+    let Ok(ret) = Int::from_str(&s) else {
+        panic!("failed to convert into to string");
+    };
+    assert_eq!(ret, value);
+}
+
+macro_rules! uint_to_s {
+    ($($fn_name:ident, $int:ident,)+) => {
+        $(
+            #[test]
+            fn $fn_name() {
+                assert_nb::<$int>($int::MIN);
+                assert_nb::<$int>($int::MAX);
+                assert_nb::<$int>(1);
+                assert_nb::<$int>($int::MIN / 2);
+                assert_nb::<$int>($int::MAX / 2);
+            }
+        )+
+    }
+}
+macro_rules! int_to_s {
+    ($($fn_name:ident, $int:ident,)+) => {
+        $(
+            #[test]
+            fn $fn_name() {
+                assert_nb::<$int>($int::MIN);
+                assert_nb::<$int>($int::MAX);
+                assert_nb::<$int>(1);
+                assert_nb::<$int>(0);
+                assert_nb::<$int>(-1);
+                assert_nb::<$int>($int::MIN / 2);
+                assert_nb::<$int>($int::MAX / 2);
+            }
+        )+
+    }
+}
+
+int_to_s!(
+    test_i8_to_string,
+    i8,
+    test_i16_to_string,
+    i16,
+    test_i32_to_string,
+    i32,
+    test_i64_to_string,
+    i64,
+    test_i128_to_string,
+    i128,
+);
+uint_to_s!(
+    test_u8_to_string,
+    u8,
+    test_u16_to_string,
+    u16,
+    test_u32_to_string,
+    u32,
+    test_u64_to_string,
+    u64,
+    test_u128_to_string,
+    u128,
+);

library/core/src/fmt/mod.rs

@@ -743,6 +743,7 @@ impl<'a> Arguments<'a> {
     #[unstable(feature = "fmt_internals", reason = "internal to standard library", issue = "none")]
     #[must_use]
     #[inline]
+    #[doc(hidden)]
     pub fn as_statically_known_str(&self) -> Option<&'static str> {
         let s = self.as_str();
         if core::intrinsics::is_val_statically_known(s.is_some()) { s } else { None }

library/core/src/intrinsics/mod.rs

@@ -2475,35 +2475,35 @@ pub unsafe fn float_to_int_unchecked<Float: Copy, Int: Copy>(value: Float) -> In
 
 /// Float addition that allows optimizations based on algebraic rules.
 ///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_add`], [`f32::algebraic_add`], [`f64::algebraic_add`] and [`f128::algebraic_add`].
 #[rustc_nounwind]
 #[rustc_intrinsic]
 pub fn fadd_algebraic<T: Copy>(a: T, b: T) -> T;
 
 /// Float subtraction that allows optimizations based on algebraic rules.
 ///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_sub`], [`f32::algebraic_sub`], [`f64::algebraic_sub`] and [`f128::algebraic_sub`].
 #[rustc_nounwind]
 #[rustc_intrinsic]
 pub fn fsub_algebraic<T: Copy>(a: T, b: T) -> T;
 
 /// Float multiplication that allows optimizations based on algebraic rules.
 ///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_mul`], [`f32::algebraic_mul`], [`f64::algebraic_mul`] and [`f128::algebraic_mul`].
 #[rustc_nounwind]
 #[rustc_intrinsic]
 pub fn fmul_algebraic<T: Copy>(a: T, b: T) -> T;
 
 /// Float division that allows optimizations based on algebraic rules.
 ///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_div`], [`f32::algebraic_div`], [`f64::algebraic_div`] and [`f128::algebraic_div`].
 #[rustc_nounwind]
 #[rustc_intrinsic]
 pub fn fdiv_algebraic<T: Copy>(a: T, b: T) -> T;
 
 /// Float remainder that allows optimizations based on algebraic rules.
 ///
-/// This intrinsic does not have a stable counterpart.
+/// Stabilized as [`f16::algebraic_rem`], [`f32::algebraic_rem`], [`f64::algebraic_rem`] and [`f128::algebraic_rem`].
 #[rustc_nounwind]
 #[rustc_intrinsic]
 pub fn frem_algebraic<T: Copy>(a: T, b: T) -> T;

library/core/src/num/f128.rs

@@ -1362,4 +1362,54 @@ impl f128 {
         // SAFETY: this is actually a safe intrinsic
         unsafe { intrinsics::copysignf128(self, sign) }
     }
+
+    /// Float addition that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_add(self, rhs: f128) -> f128 {
+        intrinsics::fadd_algebraic(self, rhs)
+    }
+
+    /// Float subtraction that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_sub(self, rhs: f128) -> f128 {
+        intrinsics::fsub_algebraic(self, rhs)
+    }
+
+    /// Float multiplication that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_mul(self, rhs: f128) -> f128 {
+        intrinsics::fmul_algebraic(self, rhs)
+    }
+
+    /// Float division that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_div(self, rhs: f128) -> f128 {
+        intrinsics::fdiv_algebraic(self, rhs)
+    }
+
+    /// Float remainder that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_rem(self, rhs: f128) -> f128 {
+        intrinsics::frem_algebraic(self, rhs)
+    }
 }

library/core/src/num/f16.rs

@@ -1338,4 +1338,54 @@ impl f16 {
         // SAFETY: this is actually a safe intrinsic
         unsafe { intrinsics::copysignf16(self, sign) }
     }
+
+    /// Float addition that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_add(self, rhs: f16) -> f16 {
+        intrinsics::fadd_algebraic(self, rhs)
+    }
+
+    /// Float subtraction that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_sub(self, rhs: f16) -> f16 {
+        intrinsics::fsub_algebraic(self, rhs)
+    }
+
+    /// Float multiplication that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_mul(self, rhs: f16) -> f16 {
+        intrinsics::fmul_algebraic(self, rhs)
+    }
+
+    /// Float division that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_div(self, rhs: f16) -> f16 {
+        intrinsics::fdiv_algebraic(self, rhs)
+    }
+
+    /// Float remainder that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_rem(self, rhs: f16) -> f16 {
+        intrinsics::frem_algebraic(self, rhs)
+    }
 }

library/core/src/num/f32.rs

@@ -1504,4 +1504,54 @@ impl f32 {
         // SAFETY: this is actually a safe intrinsic
         unsafe { intrinsics::copysignf32(self, sign) }
     }
+
+    /// Float addition that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_add(self, rhs: f32) -> f32 {
+        intrinsics::fadd_algebraic(self, rhs)
+    }
+
+    /// Float subtraction that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_sub(self, rhs: f32) -> f32 {
+        intrinsics::fsub_algebraic(self, rhs)
+    }
+
+    /// Float multiplication that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_mul(self, rhs: f32) -> f32 {
+        intrinsics::fmul_algebraic(self, rhs)
+    }
+
+    /// Float division that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_div(self, rhs: f32) -> f32 {
+        intrinsics::fdiv_algebraic(self, rhs)
+    }
+
+    /// Float remainder that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_rem(self, rhs: f32) -> f32 {
+        intrinsics::frem_algebraic(self, rhs)
+    }
 }

library/core/src/num/f64.rs

@@ -1503,4 +1503,54 @@ impl f64 {
         // SAFETY: this is actually a safe intrinsic
         unsafe { intrinsics::copysignf64(self, sign) }
     }
+
+    /// Float addition that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_add(self, rhs: f64) -> f64 {
+        intrinsics::fadd_algebraic(self, rhs)
+    }
+
+    /// Float subtraction that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_sub(self, rhs: f64) -> f64 {
+        intrinsics::fsub_algebraic(self, rhs)
+    }
+
+    /// Float multiplication that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_mul(self, rhs: f64) -> f64 {
+        intrinsics::fmul_algebraic(self, rhs)
+    }
+
+    /// Float division that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_div(self, rhs: f64) -> f64 {
+        intrinsics::fdiv_algebraic(self, rhs)
+    }
+
+    /// Float remainder that allows optimizations based on algebraic rules.
+    ///
+    /// See [algebraic operators](primitive@f32#algebraic-operators) for more info.
+    #[must_use = "method returns a new number and does not mutate the original value"]
+    #[unstable(feature = "float_algebraic", issue = "136469")]
+    #[inline]
+    pub fn algebraic_rem(self, rhs: f64) -> f64 {
+        intrinsics::frem_algebraic(self, rhs)
+    }
 }

library/core/src/primitive_docs.rs

@@ -1313,6 +1313,51 @@ mod prim_f16 {}
 /// | `wasm32`, `wasm64` | If all input NaNs are quiet with all-zero payload: None.<br> Otherwise: all possible payloads. |
 ///
 /// For targets not in this table, all payloads are possible.
+///
+/// # Algebraic operators
+///
+/// Algebraic operators of the form `a.algebraic_*(b)` allow the compiler to optimize
+/// floating point operations using all the usual algebraic properties of real numbers --
+/// despite the fact that those properties do *not* hold on floating point numbers.
+/// This can give a great performance boost since it may unlock vectorization.
+///
+/// The exact set of optimizations is unspecified but typically allows combining operations,
+/// rearranging series of operations based on mathematical properties, converting between division
+/// and reciprocal multiplication, and disregarding the sign of zero. This means that the results of
+/// elementary operations may have undefined precision, and "non-mathematical" values
+/// such as NaN, +/-Inf, or -0.0 may behave in unexpected ways, but these operations
+/// will never cause undefined behavior.
+///
+/// Because of the unpredictable nature of compiler optimizations, the same inputs may produce
+/// different results even within a single program run. **Unsafe code must not rely on any property
+/// of the return value for soundness.** However, implementations will generally do their best to
+/// pick a reasonable tradeoff between performance and accuracy of the result.
+///
+/// For example:
+///
+/// ```
+/// # #![feature(float_algebraic)]
+/// # #![allow(unused_assignments)]
+/// # let mut x: f32 = 0.0;
+/// # let a: f32 = 1.0;
+/// # let b: f32 = 2.0;
+/// # let c: f32 = 3.0;
+/// # let d: f32 = 4.0;
+/// x = a.algebraic_add(b).algebraic_add(c).algebraic_add(d);
+/// ```
+///
+/// May be rewritten as:
+///
+/// ```
+/// # #![allow(unused_assignments)]
+/// # let mut x: f32 = 0.0;
+/// # let a: f32 = 1.0;
+/// # let b: f32 = 2.0;
+/// # let c: f32 = 3.0;
+/// # let d: f32 = 4.0;
+/// x = a + b + c + d; // As written
+/// x = (a + c) + (b + d); // Reordered to shorten critical path and enable vectorization
+/// ```
 
 #[stable(feature = "rust1", since = "1.0.0")]
 mod prim_f32 {}

library/core/src/unicode/unicode_data.rs

@@ -47,45 +47,78 @@ const fn bitset_search<
     (word & (1 << (needle % 64) as u64)) != 0
 }
 
-fn decode_prefix_sum(short_offset_run_header: u32) -> u32 {
+#[repr(transparent)]
-    short_offset_run_header & ((1 << 21) - 1)
+struct ShortOffsetRunHeader(u32);
-}
+
-
+impl ShortOffsetRunHeader {
-fn decode_length(short_offset_run_header: u32) -> usize {
+    const fn new(start_index: usize, prefix_sum: u32) -> Self {
-    (short_offset_run_header >> 21) as usize
+        assert!(start_index < (1 << 11));
+        assert!(prefix_sum < (1 << 21));
+
+        Self((start_index as u32) << 21 | prefix_sum)
+    }
+
+    #[inline]
+    const fn start_index(&self) -> usize {
+        (self.0 >> 21) as usize
+    }
+
+    #[inline]
+    const fn prefix_sum(&self) -> u32 {
+        self.0 & ((1 << 21) - 1)
+    }
 }
 
+/// # Safety
+///
+/// - The last element of `short_offset_runs` must be greater than `std::char::MAX`.
+/// - The start indices of all elements in `short_offset_runs` must be less than `OFFSETS`.
 #[inline(always)]
-fn skip_search<const SOR: usize, const OFFSETS: usize>(
+unsafe fn skip_search<const SOR: usize, const OFFSETS: usize>(
-    needle: u32,
+    needle: char,
-    short_offset_runs: &[u32; SOR],
+    short_offset_runs: &[ShortOffsetRunHeader; SOR],
     offsets: &[u8; OFFSETS],
 ) -> bool {
-    // Note that this *cannot* be past the end of the array, as the last
+    let needle = needle as u32;
-    // element is greater than std::char::MAX (the largest possible needle).
+
-    //
-    // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct
-    // location cannot be past it, so Err(idx) != length either.
-    //
-    // This means that we can avoid bounds checking for the accesses below, too.
     let last_idx =
-        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) {
+        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| (header.0 << 11)) {
             Ok(idx) => idx + 1,
             Err(idx) => idx,
         };
+    // SAFETY: `last_idx` *cannot* be past the end of the array, as the last
+    // element is greater than `std::char::MAX` (the largest possible needle)
+    // as guaranteed by the caller.
+    //
+    // So, we cannot have found it (i.e. `Ok(idx) => idx + 1 != length`) and the
+    // correct location cannot be past it, so `Err(idx) => idx != length` either.
+    //
+    // This means that we can avoid bounds checking for the accesses below, too.
+    //
+    // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+    // in `hint::assert_unchecked` may not be optimized out.
+    unsafe { crate::intrinsics::assume(last_idx < SOR) };
 
-    let mut offset_idx = decode_length(short_offset_runs[last_idx]);
+    let mut offset_idx = short_offset_runs[last_idx].start_index();
     let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
-        decode_length(*next) - offset_idx
+        (*next).start_index() - offset_idx
     } else {
         offsets.len() - offset_idx
     };
+
     let prev =
-        last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0);
+        last_idx.checked_sub(1).map(|prev| short_offset_runs[prev].prefix_sum()).unwrap_or(0);
 
     let total = needle - prev;
     let mut prefix_sum = 0;
     for _ in 0..(length - 1) {
+        // SAFETY: It is guaranteed that `length <= OFFSETS - offset_idx`,
+        // so it follows that `length - 1 + offset_idx < OFFSETS`, therefore
+        // `offset_idx < OFFSETS` is always true in this loop.
+        //
+        // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+        // in `hint::assert_unchecked` may not be optimized out.
+        unsafe { crate::intrinsics::assume(offset_idx < OFFSETS) };
         let offset = offsets[offset_idx];
         prefix_sum += offset as u32;
         if prefix_sum > total {
@@ -100,15 +133,36 @@ pub const UNICODE_VERSION: (u8, u8, u8) = (16, 0, 0);
 
 #[rustfmt::skip]
 pub mod alphabetic {
-    static SHORT_OFFSET_RUNS: [u32; 53] = [
+    use super::ShortOffsetRunHeader;
-        706, 33559113, 876615277, 956309270, 1166025910, 1314925568, 1319120901, 1398813696,
+
-        1449151936, 1451271309, 1455465997, 1463867300, 1652619520, 1663105646, 1665203518,
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 53] = [
-        1711342208, 1797326647, 1895898848, 2560697242, 2583768976, 2594255920, 2600551419,
+        ShortOffsetRunHeader::new(0, 706), ShortOffsetRunHeader::new(16, 4681),
-        2608940615, 2613141760, 2615240704, 2619435577, 2621533504, 2652997624, 2688650454,
+        ShortOffsetRunHeader::new(418, 5741), ShortOffsetRunHeader::new(456, 7958),
-        2692853744, 2699145507, 2713826044, 2734799872, 2736903168, 2757875366, 2835472128,
+        ShortOffsetRunHeader::new(556, 9398), ShortOffsetRunHeader::new(627, 11264),
-        2883707536, 2934039760, 2942429152, 2955013632, 2988568880, 3126984704, 3139610336,
+        ShortOffsetRunHeader::new(629, 12293), ShortOffsetRunHeader::new(667, 13312),
-        3141711674, 3145911970, 3154308065, 3158503006, 3162699776, 3164797470, 3166896128,
+        ShortOffsetRunHeader::new(691, 19904), ShortOffsetRunHeader::new(692, 42125),
-        3168998219, 3171099568, 3176407984,
+        ShortOffsetRunHeader::new(694, 42509), ShortOffsetRunHeader::new(698, 55204),
+        ShortOffsetRunHeader::new(788, 63744), ShortOffsetRunHeader::new(793, 64110),
+        ShortOffsetRunHeader::new(794, 64830), ShortOffsetRunHeader::new(816, 66176),
+        ShortOffsetRunHeader::new(857, 67383), ShortOffsetRunHeader::new(904, 73440),
+        ShortOffsetRunHeader::new(1221, 74650), ShortOffsetRunHeader::new(1232, 77712),
+        ShortOffsetRunHeader::new(1237, 78896), ShortOffsetRunHeader::new(1240, 82939),
+        ShortOffsetRunHeader::new(1244, 83527), ShortOffsetRunHeader::new(1246, 90368),
+        ShortOffsetRunHeader::new(1247, 92160), ShortOffsetRunHeader::new(1249, 92729),
+        ShortOffsetRunHeader::new(1250, 93504), ShortOffsetRunHeader::new(1265, 100344),
+        ShortOffsetRunHeader::new(1282, 101590), ShortOffsetRunHeader::new(1284, 110576),
+        ShortOffsetRunHeader::new(1287, 110883), ShortOffsetRunHeader::new(1294, 111356),
+        ShortOffsetRunHeader::new(1304, 113664), ShortOffsetRunHeader::new(1305, 119808),
+        ShortOffsetRunHeader::new(1315, 120486), ShortOffsetRunHeader::new(1352, 122624),
+        ShortOffsetRunHeader::new(1375, 123536), ShortOffsetRunHeader::new(1399, 124112),
+        ShortOffsetRunHeader::new(1403, 124896), ShortOffsetRunHeader::new(1409, 126464),
+        ShortOffsetRunHeader::new(1425, 127280), ShortOffsetRunHeader::new(1491, 131072),
+        ShortOffsetRunHeader::new(1497, 173792), ShortOffsetRunHeader::new(1498, 177978),
+        ShortOffsetRunHeader::new(1500, 183970), ShortOffsetRunHeader::new(1504, 191457),
+        ShortOffsetRunHeader::new(1506, 192094), ShortOffsetRunHeader::new(1508, 194560),
+        ShortOffsetRunHeader::new(1509, 195102), ShortOffsetRunHeader::new(1510, 196608),
+        ShortOffsetRunHeader::new(1511, 201547), ShortOffsetRunHeader::new(1512, 205744),
+        ShortOffsetRunHeader::new(1514, 1319856),
     ];
     static OFFSETS: [u8; 1515] = [
         65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 0, 4, 12, 14, 5, 7, 1, 1, 1, 86, 1, 29,
@@ -169,22 +223,44 @@ pub mod alphabetic {
         0, 0, 0, 0, 5, 0, 0,
     ];
     pub fn lookup(c: char) -> bool {
-        super::skip_search(
+        const {
-            c as u32,
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
-            &SHORT_OFFSET_RUNS,
+            let mut i = 0;
-            &OFFSETS,
+            while i < SHORT_OFFSET_RUNS.len() {
-        )
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
     }
 }
 
 #[rustfmt::skip]
 pub mod case_ignorable {
-    static SHORT_OFFSET_RUNS: [u32; 37] = [
+    use super::ShortOffsetRunHeader;
-        688, 44045149, 572528402, 576724925, 807414908, 878718981, 903913493, 929080568, 933275148,
+
-        937491230, 1138818560, 1147208189, 1210124160, 1222707713, 1235291428, 1260457643,
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 37] = [
-        1277237295, 1537284411, 1545673776, 1604394739, 1667314736, 1692492062, 1700883184,
+        ShortOffsetRunHeader::new(0, 688), ShortOffsetRunHeader::new(21, 4957),
-        1709272384, 1721855823, 1730260976, 1747041437, 1759629056, 1768018279, 1776409088,
+        ShortOffsetRunHeader::new(273, 5906), ShortOffsetRunHeader::new(275, 8125),
-        1797382144, 1822548654, 1856103659, 1864493264, 1872884731, 1882062849, 1887371760,
+        ShortOffsetRunHeader::new(385, 11388), ShortOffsetRunHeader::new(419, 12293),
+        ShortOffsetRunHeader::new(431, 40981), ShortOffsetRunHeader::new(443, 42232),
+        ShortOffsetRunHeader::new(445, 42508), ShortOffsetRunHeader::new(447, 64286),
+        ShortOffsetRunHeader::new(543, 65024), ShortOffsetRunHeader::new(547, 66045),
+        ShortOffsetRunHeader::new(577, 67456), ShortOffsetRunHeader::new(583, 68097),
+        ShortOffsetRunHeader::new(589, 68900), ShortOffsetRunHeader::new(601, 69291),
+        ShortOffsetRunHeader::new(609, 71727), ShortOffsetRunHeader::new(733, 71995),
+        ShortOffsetRunHeader::new(737, 72752), ShortOffsetRunHeader::new(765, 73459),
+        ShortOffsetRunHeader::new(795, 78896), ShortOffsetRunHeader::new(807, 90398),
+        ShortOffsetRunHeader::new(811, 92912), ShortOffsetRunHeader::new(815, 93504),
+        ShortOffsetRunHeader::new(821, 94031), ShortOffsetRunHeader::new(825, 110576),
+        ShortOffsetRunHeader::new(833, 113821), ShortOffsetRunHeader::new(839, 118528),
+        ShortOffsetRunHeader::new(843, 119143), ShortOffsetRunHeader::new(847, 121344),
+        ShortOffsetRunHeader::new(857, 122880), ShortOffsetRunHeader::new(869, 123566),
+        ShortOffsetRunHeader::new(885, 124139), ShortOffsetRunHeader::new(889, 125136),
+        ShortOffsetRunHeader::new(893, 127995), ShortOffsetRunHeader::new(897, 917505),
+        ShortOffsetRunHeader::new(899, 2032112),
     ];
     static OFFSETS: [u8; 905] = [
         39, 1, 6, 1, 11, 1, 35, 1, 1, 1, 71, 1, 4, 1, 1, 1, 4, 1, 2, 2, 0, 192, 4, 2, 4, 1, 9, 2,
@@ -222,20 +298,36 @@ pub mod case_ignorable {
         1, 61, 4, 0, 5, 254, 2, 0, 7, 109, 8, 0, 5, 0, 1, 30, 96, 128, 240, 0,
     ];
     pub fn lookup(c: char) -> bool {
-        super::skip_search(
+        const {
-            c as u32,
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
-            &SHORT_OFFSET_RUNS,
+            let mut i = 0;
-            &OFFSETS,
+            while i < SHORT_OFFSET_RUNS.len() {
-        )
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
     }
 }
 
 #[rustfmt::skip]
 pub mod cased {
-    static SHORT_OFFSET_RUNS: [u32; 22] = [
+    use super::ShortOffsetRunHeader;
-        4256, 115348384, 136322176, 144711446, 163587254, 320875520, 325101120, 350268208,
+
-        392231680, 404815649, 413205504, 421595008, 467733632, 484513952, 501313088, 505533440,
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 22] = [
-        509728422, 587325184, 635559984, 648145152, 652341552, 657650058,
+        ShortOffsetRunHeader::new(0, 4256), ShortOffsetRunHeader::new(55, 5024),
+        ShortOffsetRunHeader::new(65, 7296), ShortOffsetRunHeader::new(69, 7958),
+        ShortOffsetRunHeader::new(78, 9398), ShortOffsetRunHeader::new(153, 11264),
+        ShortOffsetRunHeader::new(155, 42560), ShortOffsetRunHeader::new(167, 43824),
+        ShortOffsetRunHeader::new(187, 64256), ShortOffsetRunHeader::new(193, 65313),
+        ShortOffsetRunHeader::new(197, 66560), ShortOffsetRunHeader::new(201, 67456),
+        ShortOffsetRunHeader::new(223, 68736), ShortOffsetRunHeader::new(231, 71840),
+        ShortOffsetRunHeader::new(239, 93760), ShortOffsetRunHeader::new(241, 119808),
+        ShortOffsetRunHeader::new(243, 120486), ShortOffsetRunHeader::new(280, 122624),
+        ShortOffsetRunHeader::new(303, 122928), ShortOffsetRunHeader::new(309, 125184),
+        ShortOffsetRunHeader::new(311, 127280), ShortOffsetRunHeader::new(313, 1241482),
     ];
     static OFFSETS: [u8; 319] = [
         65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 195, 1, 4, 4, 208, 1, 36, 7, 2, 30, 5,
@@ -252,39 +344,67 @@ pub mod cased {
         8, 0, 10, 1, 20, 6, 6, 0, 62, 0, 68, 0, 26, 6, 26, 6, 26, 0,
     ];
     pub fn lookup(c: char) -> bool {
-        super::skip_search(
+        const {
-            c as u32,
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
-            &SHORT_OFFSET_RUNS,
+            let mut i = 0;
-            &OFFSETS,
+            while i < SHORT_OFFSET_RUNS.len() {
-        )
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
     }
 }
 
 #[rustfmt::skip]
 pub mod cc {
-    static SHORT_OFFSET_RUNS: [u32; 1] = [
+    use super::ShortOffsetRunHeader;
-        1114272,
+
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 1] = [
+        ShortOffsetRunHeader::new(0, 1114272),
     ];
     static OFFSETS: [u8; 5] = [
         0, 32, 95, 33, 0,
     ];
     pub fn lookup(c: char) -> bool {
-        super::skip_search(
+        const {
-            c as u32,
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
-            &SHORT_OFFSET_RUNS,
+            let mut i = 0;
-            &OFFSETS,
+            while i < SHORT_OFFSET_RUNS.len() {
-        )
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
     }
 }
 
 #[rustfmt::skip]
 pub mod grapheme_extend {
-    static SHORT_OFFSET_RUNS: [u32; 34] = [
+    use super::ShortOffsetRunHeader;
-        768, 2098307, 6292881, 10490717, 522196754, 526393356, 723528943, 731918378, 744531567,
+
-        752920578, 769719070, 908131840, 912326558, 920715773, 924912129, 937495844, 962662059,
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 34] = [
-        971053103, 1256266800, 1323376371, 1386296384, 1407279390, 1415670512, 1424060239,
+        ShortOffsetRunHeader::new(0, 768), ShortOffsetRunHeader::new(1, 1155),
-        1432468637, 1449250560, 1453445477, 1461836288, 1487003648, 1512170158, 1541530860,
+        ShortOffsetRunHeader::new(3, 1425), ShortOffsetRunHeader::new(5, 4957),
-        1549920464, 1559101472, 1568604656,
+        ShortOffsetRunHeader::new(249, 5906), ShortOffsetRunHeader::new(251, 8204),
+        ShortOffsetRunHeader::new(345, 11503), ShortOffsetRunHeader::new(349, 12330),
+        ShortOffsetRunHeader::new(355, 42607), ShortOffsetRunHeader::new(359, 43010),
+        ShortOffsetRunHeader::new(367, 64286), ShortOffsetRunHeader::new(433, 65024),
+        ShortOffsetRunHeader::new(435, 65438), ShortOffsetRunHeader::new(439, 66045),
+        ShortOffsetRunHeader::new(441, 68097), ShortOffsetRunHeader::new(447, 68900),
+        ShortOffsetRunHeader::new(459, 69291), ShortOffsetRunHeader::new(463, 71727),
+        ShortOffsetRunHeader::new(599, 72752), ShortOffsetRunHeader::new(631, 73459),
+        ShortOffsetRunHeader::new(661, 78912), ShortOffsetRunHeader::new(671, 90398),
+        ShortOffsetRunHeader::new(675, 92912), ShortOffsetRunHeader::new(679, 94031),
+        ShortOffsetRunHeader::new(683, 113821), ShortOffsetRunHeader::new(691, 118528),
+        ShortOffsetRunHeader::new(693, 119141), ShortOffsetRunHeader::new(697, 121344),
+        ShortOffsetRunHeader::new(709, 122880), ShortOffsetRunHeader::new(721, 123566),
+        ShortOffsetRunHeader::new(735, 124140), ShortOffsetRunHeader::new(739, 125136),
+        ShortOffsetRunHeader::new(743, 917536), ShortOffsetRunHeader::new(747, 2032112),
     ];
     static OFFSETS: [u8; 751] = [
         0, 112, 0, 7, 0, 45, 1, 1, 1, 2, 1, 2, 1, 1, 72, 11, 48, 21, 16, 1, 101, 7, 2, 6, 2, 2, 1,
@@ -319,12 +439,20 @@ pub mod grapheme_extend {
     pub fn lookup(c: char) -> bool {
         (c as u32) >= 0x300 && lookup_slow(c)
     }
+
+    #[inline(never)]
     fn lookup_slow(c: char) -> bool {
-        super::skip_search(
+        const {
-            c as u32,
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
-            &SHORT_OFFSET_RUNS,
+            let mut i = 0;
-            &OFFSETS,
+            while i < SHORT_OFFSET_RUNS.len() {
-        )
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
     }
 }
 
@@ -436,13 +564,30 @@ pub mod lowercase {
 
 #[rustfmt::skip]
 pub mod n {
-    static SHORT_OFFSET_RUNS: [u32; 42] = [
+    use super::ShortOffsetRunHeader;
-        1632, 18876774, 31461440, 102765417, 111154926, 115349830, 132128880, 165684320, 186656630,
+
-        195046653, 199241735, 203436434, 216049184, 241215536, 249605104, 274792208, 278987015,
+    static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; 42] = [
-        283181793, 295766104, 320933114, 383848032, 396432464, 438376016, 446765280, 463543280,
+        ShortOffsetRunHeader::new(0, 1632), ShortOffsetRunHeader::new(9, 2406),
-        471932752, 488711168, 497115440, 501312096, 505507184, 522284672, 526503152, 530698944,
+        ShortOffsetRunHeader::new(15, 4160), ShortOffsetRunHeader::new(49, 4969),
-        534894542, 547479872, 551674608, 555869424, 560064711, 568454257, 576844032, 597818352,
+        ShortOffsetRunHeader::new(53, 5870), ShortOffsetRunHeader::new(55, 6470),
-        603126778,
+        ShortOffsetRunHeader::new(63, 8304), ShortOffsetRunHeader::new(79, 9312),
+        ShortOffsetRunHeader::new(89, 10102), ShortOffsetRunHeader::new(93, 11517),
+        ShortOffsetRunHeader::new(95, 12295), ShortOffsetRunHeader::new(97, 12690),
+        ShortOffsetRunHeader::new(103, 42528), ShortOffsetRunHeader::new(115, 43056),
+        ShortOffsetRunHeader::new(119, 44016), ShortOffsetRunHeader::new(131, 65296),
+        ShortOffsetRunHeader::new(133, 65799), ShortOffsetRunHeader::new(135, 66273),
+        ShortOffsetRunHeader::new(141, 67672), ShortOffsetRunHeader::new(153, 68858),
+        ShortOffsetRunHeader::new(183, 69216), ShortOffsetRunHeader::new(189, 70736),
+        ShortOffsetRunHeader::new(209, 71248), ShortOffsetRunHeader::new(213, 71904),
+        ShortOffsetRunHeader::new(221, 72688), ShortOffsetRunHeader::new(225, 73552),
+        ShortOffsetRunHeader::new(233, 74752), ShortOffsetRunHeader::new(237, 90416),
+        ShortOffsetRunHeader::new(239, 92768), ShortOffsetRunHeader::new(241, 93552),
+        ShortOffsetRunHeader::new(249, 93824), ShortOffsetRunHeader::new(251, 118000),
+        ShortOffsetRunHeader::new(253, 119488), ShortOffsetRunHeader::new(255, 120782),
+        ShortOffsetRunHeader::new(261, 123200), ShortOffsetRunHeader::new(263, 123632),
+        ShortOffsetRunHeader::new(265, 124144), ShortOffsetRunHeader::new(267, 125127),
+        ShortOffsetRunHeader::new(271, 126065), ShortOffsetRunHeader::new(275, 127232),
+        ShortOffsetRunHeader::new(285, 130032), ShortOffsetRunHeader::new(287, 1244154),
     ];
     static OFFSETS: [u8; 289] = [
         48, 10, 120, 2, 5, 1, 2, 3, 0, 10, 134, 10, 198, 10, 0, 10, 118, 10, 4, 6, 108, 10, 118,
@@ -459,11 +604,17 @@ pub mod n {
         10, 247, 10, 0, 9, 128, 10, 0, 59, 1, 3, 1, 4, 76, 45, 1, 15, 0, 13, 0, 10, 0,
     ];
     pub fn lookup(c: char) -> bool {
-        super::skip_search(
+        const {
-            c as u32,
+            assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
-            &SHORT_OFFSET_RUNS,
+            let mut i = 0;
-            &OFFSETS,
+            while i < SHORT_OFFSET_RUNS.len() {
-        )
+                assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                i += 1;
+            }
+        }
+        // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+        // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+        unsafe { super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }
     }
 }
 

library/std/src/lib.rs

@@ -341,6 +341,7 @@
 #![feature(exact_size_is_empty)]
 #![feature(exclusive_wrapper)]
 #![feature(extend_one)]
+#![feature(float_algebraic)]
 #![feature(float_gamma)]
 #![feature(float_minimum_maximum)]
 #![feature(fmt_internals)]

library/std/src/sys/thread_local/mod.rs

@@ -138,6 +138,7 @@ pub(crate) mod key {
                 not(target_family = "wasm"),
                 target_family = "unix",
             ),
+            all(not(target_thread_local), target_vendor = "apple"),
             target_os = "teeos",
             all(target_os = "wasi", target_env = "p1", target_feature = "atomics"),
         ))] {

library/std/tests/floats/f128.rs

@@ -984,6 +984,25 @@ fn test_total_cmp() {
     assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
 }
 
+#[test]
+fn test_algebraic() {
+    let a: f128 = 123.0;
+    let b: f128 = 456.0;
+
+    // Check that individual operations match their primitive counterparts.
+    //
+    // This is a check of current implementations and does NOT imply any form of
+    // guarantee about future behavior. The compiler reserves the right to make
+    // these operations inexact matches in the future.
+    let eps = if cfg!(miri) { 1e-6 } else { 0.0 };
+
+    assert_approx_eq!(a.algebraic_add(b), a + b, eps);
+    assert_approx_eq!(a.algebraic_sub(b), a - b, eps);
+    assert_approx_eq!(a.algebraic_mul(b), a * b, eps);
+    assert_approx_eq!(a.algebraic_div(b), a / b, eps);
+    assert_approx_eq!(a.algebraic_rem(b), a % b, eps);
+}
+
 #[test]
 fn test_from() {
     assert_eq!(f128::from(false), 0.0);

library/std/tests/floats/f16.rs

@@ -954,6 +954,27 @@ fn test_total_cmp() {
     assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
 }
 
+#[test]
+fn test_algebraic() {
+    let a: f16 = 123.0;
+    let b: f16 = 456.0;
+
+    // Check that individual operations match their primitive counterparts.
+    //
+    // This is a check of current implementations and does NOT imply any form of
+    // guarantee about future behavior. The compiler reserves the right to make
+    // these operations inexact matches in the future.
+    let eps_add = if cfg!(miri) { 1e1 } else { 0.0 };
+    let eps_mul = if cfg!(miri) { 1e3 } else { 0.0 };
+    let eps_div = if cfg!(miri) { 1e0 } else { 0.0 };
+
+    assert_approx_eq!(a.algebraic_add(b), a + b, eps_add);
+    assert_approx_eq!(a.algebraic_sub(b), a - b, eps_add);
+    assert_approx_eq!(a.algebraic_mul(b), a * b, eps_mul);
+    assert_approx_eq!(a.algebraic_div(b), a / b, eps_div);
+    assert_approx_eq!(a.algebraic_rem(b), a % b, eps_div);
+}
+
 #[test]
 fn test_from() {
     assert_eq!(f16::from(false), 0.0);

library/std/tests/floats/f32.rs

@@ -915,3 +915,24 @@ fn test_total_cmp() {
     assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f32::INFINITY));
     assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
 }
+
+#[test]
+fn test_algebraic() {
+    let a: f32 = 123.0;
+    let b: f32 = 456.0;
+
+    // Check that individual operations match their primitive counterparts.
+    //
+    // This is a check of current implementations and does NOT imply any form of
+    // guarantee about future behavior. The compiler reserves the right to make
+    // these operations inexact matches in the future.
+    let eps_add = if cfg!(miri) { 1e-3 } else { 0.0 };
+    let eps_mul = if cfg!(miri) { 1e-1 } else { 0.0 };
+    let eps_div = if cfg!(miri) { 1e-4 } else { 0.0 };
+
+    assert_approx_eq!(a.algebraic_add(b), a + b, eps_add);
+    assert_approx_eq!(a.algebraic_sub(b), a - b, eps_add);
+    assert_approx_eq!(a.algebraic_mul(b), a * b, eps_mul);
+    assert_approx_eq!(a.algebraic_div(b), a / b, eps_div);
+    assert_approx_eq!(a.algebraic_rem(b), a % b, eps_div);
+}

library/std/tests/floats/f64.rs

@@ -894,3 +894,22 @@ fn test_total_cmp() {
     assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&f64::INFINITY));
     assert_eq!(Ordering::Less, (-s_nan()).total_cmp(&s_nan()));
 }
+
+#[test]
+fn test_algebraic() {
+    let a: f64 = 123.0;
+    let b: f64 = 456.0;
+
+    // Check that individual operations match their primitive counterparts.
+    //
+    // This is a check of current implementations and does NOT imply any form of
+    // guarantee about future behavior. The compiler reserves the right to make
+    // these operations inexact matches in the future.
+    let eps = if cfg!(miri) { 1e-6 } else { 0.0 };
+
+    assert_approx_eq!(a.algebraic_add(b), a + b, eps);
+    assert_approx_eq!(a.algebraic_sub(b), a - b, eps);
+    assert_approx_eq!(a.algebraic_mul(b), a * b, eps);
+    assert_approx_eq!(a.algebraic_div(b), a / b, eps);
+    assert_approx_eq!(a.algebraic_rem(b), a % b, eps);
+}

library/std/tests/floats/lib.rs

@@ -1,4 +1,4 @@
-#![feature(f16, f128, float_gamma, float_minimum_maximum)]
+#![feature(f16, f128, float_algebraic, float_gamma, float_minimum_maximum)]
 
 use std::fmt;
 use std::ops::{Add, Div, Mul, Rem, Sub};
@@ -10,7 +10,7 @@ macro_rules! assert_approx_eq {
         let (a, b) = (&$a, &$b);
         let diff = (*a - *b).abs();
         assert!(
-            diff < $lim,
+            diff <= $lim,
             "{a:?} is not approximately equal to {b:?} (threshold {lim:?}, difference {diff:?})",
             lim = $lim
         );

library/std/tests/thread_local/lib.rs

@@ -1,3 +1,5 @@
+#![feature(cfg_target_thread_local)]
+
 #[cfg(not(any(target_os = "emscripten", target_os = "wasi")))]
 mod tests;
 

library/std/tests/thread_local/tests.rs

@@ -1,7 +1,7 @@
 use std::cell::{Cell, UnsafeCell};
 use std::sync::atomic::{AtomicU8, Ordering};
 use std::sync::{Arc, Condvar, Mutex};
-use std::thread::{self, Builder, LocalKey};
+use std::thread::{self, LocalKey};
 use std::thread_local;
 
 #[derive(Clone, Default)]
@@ -345,8 +345,27 @@ fn join_orders_after_tls_destructors() {
 }
 
 // Test that thread::current is still available in TLS destructors.
+//
+// The test won't currently work without target_thread_local, aka with slow tls.
+// The runtime tries very hard to drop last the TLS variable that keeps the information about the
+// current thread, by using several tricks like deffering the drop to a later round of TLS destruction.
+// However, this only seems to work with fast tls.
+//
+// With slow TLS, it seems that multiple libc implementations will just set the value to null the first
+// time they encounter it, regardless of it having a destructor or not. This means that trying to
+// retrieve it later in a drop impl of another TLS variable will not work.
+//
+// ** Apple libc: https://github.com/apple-oss-distributions/libpthread/blob/c032e0b076700a0a47db75528a282b8d3a06531a/src/pthread_tsd.c#L293
+// Sets the variable to null if it has a destructor and the value is not null. However, all variables
+// created with pthread_key_create are marked as having a destructor, even if the fn ptr called with
+// it is null.
+// ** glibc: https://github.com/bminor/glibc/blob/e5893e6349541d871e8a25120bca014551d13ff5/nptl/nptl_deallocate_tsd.c#L59
+// ** musl: https://github.com/kraj/musl/blob/1880359b54ff7dd9f5016002bfdae4b136007dde/src/thread/pthread_key_create.c#L87
+#[cfg(target_thread_local)]
 #[test]
 fn thread_current_in_dtor() {
+    use std::thread::Builder;
+
     // Go through one round of TLS destruction first.
     struct Defer;
     impl Drop for Defer {

src/librustdoc/html/static/js/main.js

@@ -1101,7 +1101,6 @@ function preLoadCss(cssUrl) {
         });
     }());
 
-    // @ts-expect-error
     window.rustdoc_add_line_numbers_to_examples = () => {
         // @ts-expect-error
         function generateLine(nb) {
@@ -1123,7 +1122,6 @@ function preLoadCss(cssUrl) {
         });
     };
 
-    // @ts-expect-error
     window.rustdoc_remove_line_numbers_from_examples = () => {
         onEachLazy(
             document.querySelectorAll(".rustdoc:not(.src) :not(.scraped-example) > .example-wrap"),
@@ -1132,7 +1130,6 @@ function preLoadCss(cssUrl) {
     };
 
     if (getSettingValue("line-numbers") === "true") {
-        // @ts-expect-error
         window.rustdoc_add_line_numbers_to_examples();
     }
 
@@ -1596,7 +1593,7 @@ function preLoadCss(cssUrl) {
     /**
      * Hide popover menus, clickable tooltips, and the sidebar (if applicable).
      *
-     * Pass "true" to reset focus for tooltip popovers.
+     * Pass `true` to reset focus for tooltip popovers.
      */
     window.hideAllModals = switchFocus => {
         hideSidebar();

src/librustdoc/html/static/js/rustdoc.d.ts

@@ -30,6 +30,8 @@ declare global {
         currentCrate: string|null;
         /**
          * Hide popovers, tooltips, or the mobile sidebar.
+         *
+         * Pass `true` to reset focus for tooltip popovers.
          */
         hideAllModals: function(boolean),
         /**
@@ -78,6 +80,8 @@ declare global {
         pending_implementors?: rustdoc.Implementors,
         register_type_impls?: function(rustdoc.TypeImpls): void,
         pending_type_impls?: rustdoc.TypeImpls,
+        rustdoc_add_line_numbers_to_examples?: function(),
+        rustdoc_remove_line_numbers_from_examples?: function(),
     }
     interface HTMLElement {
         /** Used by the popover tooltip code. */
@@ -477,4 +481,14 @@ declare namespace rustdoc {
      * is a tuple of (filename, subdirs, filenames).
      */
     type Dir = [string, rustdoc.Dir[], string[]]
+
+    /**
+     * Indivitual setting object, used in `settings.js`
+     */
+    interface Setting {
+        js_name: string,
+        name: string,
+        options?: string[],
+        default: string | boolean,
+    }
 }

src/librustdoc/html/static/js/settings.js

@@ -1,22 +1,39 @@
 // Local js definitions:
 /* global getSettingValue, updateLocalStorage, updateTheme */
 /* global addClass, removeClass, onEach, onEachLazy */
-/* global MAIN_ID, getVar, getSettingsButton, getHelpButton */
+/* global MAIN_ID, getVar, getSettingsButton, getHelpButton, nonnull */
-
-// Eventually fix this.
-// @ts-nocheck
 
 "use strict";
 
 (function() {
     const isSettingsPage = window.location.pathname.endsWith("/settings.html");
 
+    /**
+     * @param {Element} elem
+     * @param {EventTarget|null} target
+     */
+    function elemContainsTarget(elem, target) {
+        if (target instanceof Node) {
+            return elem.contains(target);
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * @overload {"theme"|"preferred-dark-theme"|"preferred-light-theme"}
+     * @param {string} settingName
+     * @param {string} value
+     * @returns
+     * @param {string} settingName
+     * @param {string|boolean} value
+     */
     function changeSetting(settingName, value) {
         if (settingName === "theme") {
             const useSystem = value === "system preference" ? "true" : "false";
             updateLocalStorage("use-system-theme", useSystem);
         }
-        updateLocalStorage(settingName, value);
+        updateLocalStorage(settingName, "" + value);
 
         switch (settingName) {
             case "theme":
@@ -27,9 +44,15 @@
                 break;
             case "line-numbers":
                 if (value === true) {
-                    window.rustdoc_add_line_numbers_to_examples();
+                    const f = window.rustdoc_add_line_numbers_to_examples;
+                    if (f !== undefined) {
+                        f();
+                    }
                 } else {
-                    window.rustdoc_remove_line_numbers_from_examples();
+                    const f = window.rustdoc_remove_line_numbers_from_examples;
+                    if (f !== undefined) {
+                        f();
+                    }
                 }
                 break;
             case "hide-sidebar":
@@ -89,6 +112,9 @@
         }
     }
 
+    /**
+     * @param {HTMLElement} settingsElement
+     */
     function setEvents(settingsElement) {
         updateLightAndDark();
         onEachLazy(settingsElement.querySelectorAll("input[type=\"checkbox\"]"), toggle => {
@@ -101,23 +127,27 @@
                 changeSetting(toggle.id, toggle.checked);
             };
         });
-        onEachLazy(settingsElement.querySelectorAll("input[type=\"radio\"]"), elem => {
+        onEachLazy(
-            const settingId = elem.name;
+            settingsElement.querySelectorAll("input[type=\"radio\"]"),
-            let settingValue = getSettingValue(settingId);
+            /** @param {HTMLInputElement} elem */
-            if (settingId === "theme") {
+            elem => {
-                const useSystem = getSettingValue("use-system-theme");
+                const settingId = elem.name;
-                if (useSystem === "true" || settingValue === null) {
+                let settingValue = getSettingValue(settingId);
-                    // "light" is the default theme
+                if (settingId === "theme") {
-                    settingValue = useSystem === "false" ? "light" : "system preference";
+                    const useSystem = getSettingValue("use-system-theme");
+                    if (useSystem === "true" || settingValue === null) {
+                        // "light" is the default theme
+                        settingValue = useSystem === "false" ? "light" : "system preference";
+                    }
                 }
-            }
+                if (settingValue !== null && settingValue !== "null") {
-            if (settingValue !== null && settingValue !== "null") {
+                    elem.checked = settingValue === elem.value;
-                elem.checked = settingValue === elem.value;
+                }
-            }
+                elem.addEventListener("change", () => {
-            elem.addEventListener("change", ev => {
+                    changeSetting(elem.name, elem.value);
-                changeSetting(ev.target.name, ev.target.value);
+                });
-            });
+            },
-        });
+        );
     }
 
     /**
@@ -125,7 +155,7 @@
      * as argument which describes each setting and how to render it. It returns a string
      * representing the raw HTML.
      *
-     * @param {Array<Object>} settings
+     * @param {Array<rustdoc.Setting>} settings
      *
      * @return {string}
      */
@@ -133,11 +163,6 @@
         let output = "";
 
         for (const setting of settings) {
-            if (setting === "hr") {
-                output += "<hr>";
-                continue;
-            }
-
             const js_data_name = setting["js_name"];
             const setting_name = setting["name"];
 
@@ -182,7 +207,9 @@
      * @return {HTMLElement}
      */
     function buildSettingsPage() {
-        const theme_names = getVar("themes").split(",").filter(t => t);
+        const theme_list = getVar("themes");
+        const theme_names = (theme_list === null ? "" : theme_list)
+              .split(",").filter(t => t);
         theme_names.push("light", "dark", "ayu");
 
         const settings = [
@@ -272,10 +299,16 @@
         el.innerHTML = innerHTML;
 
         if (isSettingsPage) {
-            document.getElementById(MAIN_ID).appendChild(el);
+            const mainElem = document.getElementById(MAIN_ID);
+            if (mainElem !== null) {
+                mainElem.appendChild(el);
+            }
         } else {
             el.setAttribute("tabindex", "-1");
-            getSettingsButton().appendChild(el);
+            const settingsBtn = getSettingsButton();
+            if (settingsBtn !== null) {
+                settingsBtn.appendChild(el);
+            }
         }
         return el;
     }
@@ -293,34 +326,44 @@
         });
     }
 
+    /**
+     * @param {FocusEvent} event
+     */
     function settingsBlurHandler(event) {
-        if (!getHelpButton().contains(document.activeElement) &&
+        const helpBtn = getHelpButton();
-            !getHelpButton().contains(event.relatedTarget) &&
+        const settingsBtn = getSettingsButton();
-            !getSettingsButton().contains(document.activeElement) &&
+        const helpUnfocused = helpBtn === null ||
-            !getSettingsButton().contains(event.relatedTarget)
+              (!helpBtn.contains(document.activeElement) &&
-        ) {
+               !elemContainsTarget(helpBtn, event.relatedTarget));
+        const settingsUnfocused = settingsBtn === null ||
+              (!settingsBtn.contains(document.activeElement) &&
+               !elemContainsTarget(settingsBtn, event.relatedTarget));
+        if (helpUnfocused && settingsUnfocused) {
             window.hidePopoverMenus();
         }
     }
 
     if (!isSettingsPage) {
         // We replace the existing "onclick" callback.
-        const settingsButton = getSettingsButton();
+        // These elements must exist, as (outside of the settings page)
-        const settingsMenu = document.getElementById("settings");
+        // `settings.js` is only loaded after the settings button is clicked.
+        const settingsButton = nonnull(getSettingsButton());
+        const settingsMenu = nonnull(document.getElementById("settings"));
         settingsButton.onclick = event => {
-            if (settingsMenu.contains(event.target)) {
+            if (elemContainsTarget(settingsMenu, event.target)) {
                 return;
             }
             event.preventDefault();
             const shouldDisplaySettings = settingsMenu.style.display === "none";
 
-            window.hideAllModals();
+            window.hideAllModals(false);
             if (shouldDisplaySettings) {
                 displaySettings();
             }
         };
         settingsButton.onblur = settingsBlurHandler;
-        settingsButton.querySelector("a").onblur = settingsBlurHandler;
+        // the settings button should always have a link in it
+        nonnull(settingsButton.querySelector("a")).onblur = settingsBlurHandler;
         onEachLazy(settingsMenu.querySelectorAll("input"), el => {
             el.onblur = settingsBlurHandler;
         });

src/tools/miri/src/intrinsics/mod.rs

@@ -411,9 +411,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 };
                 let res = this.binary_op(op, &a, &b)?;
                 // `binary_op` already called `generate_nan` if needed.
-                // Apply a relative error of 16ULP to simulate non-deterministic precision loss
+                // Apply a relative error of 4ULP to simulate non-deterministic precision loss
                 // due to optimizations.
-                let res = apply_random_float_error_to_imm(this, res, 4 /* log2(16) */)?;
+                let res = apply_random_float_error_to_imm(this, res, 2 /* log2(4) */)?;
                 this.write_immediate(*res, dest)?;
             }
 
@@ -464,9 +464,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 if !float_finite(&res)? {
                     throw_ub_format!("`{intrinsic_name}` intrinsic produced non-finite value as result");
                 }
-                // Apply a relative error of 16ULP to simulate non-deterministic precision loss
+                // Apply a relative error of 4ULP to simulate non-deterministic precision loss
                 // due to optimizations.
-                let res = apply_random_float_error_to_imm(this, res, 4 /* log2(16) */)?;
+                let res = apply_random_float_error_to_imm(this, res, 2 /* log2(4) */)?;
                 this.write_immediate(*res, dest)?;
             }
 

src/tools/unicode-table-generator/src/range_search.rs

@@ -45,45 +45,78 @@ const fn bitset_search<
     (word & (1 << (needle % 64) as u64)) != 0
 }
 
-fn decode_prefix_sum(short_offset_run_header: u32) -> u32 {
+#[repr(transparent)]
-    short_offset_run_header & ((1 << 21) - 1)
+struct ShortOffsetRunHeader(u32);
-}
+
-
+impl ShortOffsetRunHeader {
-fn decode_length(short_offset_run_header: u32) -> usize {
+    const fn new(start_index: usize, prefix_sum: u32) -> Self {
-    (short_offset_run_header >> 21) as usize
+        assert!(start_index < (1 << 11));
+        assert!(prefix_sum < (1 << 21));
+
+        Self((start_index as u32) << 21 | prefix_sum)
+    }
+
+    #[inline]
+    const fn start_index(&self) -> usize {
+        (self.0 >> 21) as usize
+    }
+
+    #[inline]
+    const fn prefix_sum(&self) -> u32 {
+        self.0 & ((1 << 21) - 1)
+    }
 }
 
+/// # Safety
+///
+/// - The last element of `short_offset_runs` must be greater than `std::char::MAX`.
+/// - The start indices of all elements in `short_offset_runs` must be less than `OFFSETS`.
 #[inline(always)]
-fn skip_search<const SOR: usize, const OFFSETS: usize>(
+unsafe fn skip_search<const SOR: usize, const OFFSETS: usize>(
-    needle: u32,
+    needle: char,
-    short_offset_runs: &[u32; SOR],
+    short_offset_runs: &[ShortOffsetRunHeader; SOR],
     offsets: &[u8; OFFSETS],
 ) -> bool {
-    // Note that this *cannot* be past the end of the array, as the last
+    let needle = needle as u32;
-    // element is greater than std::char::MAX (the largest possible needle).
+
-    //
-    // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct
-    // location cannot be past it, so Err(idx) != length either.
-    //
-    // This means that we can avoid bounds checking for the accesses below, too.
     let last_idx =
-        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) {
+        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| (header.0 << 11)) {
             Ok(idx) => idx + 1,
             Err(idx) => idx,
         };
+    // SAFETY: `last_idx` *cannot* be past the end of the array, as the last
+    // element is greater than `std::char::MAX` (the largest possible needle)
+    // as guaranteed by the caller.
+    //
+    // So, we cannot have found it (i.e. `Ok(idx) => idx + 1 != length`) and the
+    // correct location cannot be past it, so `Err(idx) => idx != length` either.
+    //
+    // This means that we can avoid bounds checking for the accesses below, too.
+    //
+    // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+    // in `hint::assert_unchecked` may not be optimized out.
+    unsafe { crate::intrinsics::assume(last_idx < SOR) };
 
-    let mut offset_idx = decode_length(short_offset_runs[last_idx]);
+    let mut offset_idx = short_offset_runs[last_idx].start_index();
     let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
-        decode_length(*next) - offset_idx
+        (*next).start_index() - offset_idx
     } else {
         offsets.len() - offset_idx
     };
+
     let prev =
-        last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0);
+        last_idx.checked_sub(1).map(|prev| short_offset_runs[prev].prefix_sum()).unwrap_or(0);
 
     let total = needle - prev;
     let mut prefix_sum = 0;
     for _ in 0..(length - 1) {
+        // SAFETY: It is guaranteed that `length <= OFFSETS - offset_idx`,
+        // so it follows that `length - 1 + offset_idx < OFFSETS`, therefore
+        // `offset_idx < OFFSETS` is always true in this loop.
+        //
+        // We need to use `intrinsics::assume` since the `panic_nounwind` contained
+        // in `hint::assert_unchecked` may not be optimized out.
+        unsafe { crate::intrinsics::assume(offset_idx < OFFSETS) };
         let offset = offsets[offset_idx];
         prefix_sum += offset as u32;
         if prefix_sum > total {

src/tools/unicode-table-generator/src/skiplist.rs

@@ -1,26 +1,23 @@
-use std::fmt::Write as _;
+use std::fmt::{self, Write as _};
 use std::ops::Range;
 
 use crate::fmt_list;
 use crate::raw_emitter::RawEmitter;
 
 /// This will get packed into a single u32 before inserting into the data set.
-#[derive(Debug, PartialEq)]
+#[derive(PartialEq)]
 struct ShortOffsetRunHeader {
-    /// Note, we only allow for 21 bits here.
-    prefix_sum: u32,
-
     /// Note, we actually only allow for 11 bits here. This should be enough --
     /// our largest sets are around ~1400 offsets long.
-    start_idx: u16,
+    start_index: u16,
+
+    /// Note, we only allow for 21 bits here.
+    prefix_sum: u32,
 }
 
-impl ShortOffsetRunHeader {
+impl fmt::Debug for ShortOffsetRunHeader {
-    fn pack(&self) -> u32 {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        assert!(self.start_idx < (1 << 11));
+        write!(f, "ShortOffsetRunHeader::new({}, {})", self.start_index, self.prefix_sum)
-        assert!(self.prefix_sum < (1 << 21));
-
-        (self.start_idx as u32) << 21 | self.prefix_sum
     }
 }
 
@@ -53,7 +50,7 @@ impl RawEmitter {
                     coded_offsets.push(offset);
                 } else {
                     short_offset_runs.push(ShortOffsetRunHeader {
-                        start_idx: start.try_into().unwrap(),
+                        start_index: start.try_into().unwrap(),
                         prefix_sum,
                     });
                     // This is just needed to maintain indices even/odd
@@ -71,11 +68,12 @@ impl RawEmitter {
             assert!(inserted);
         }
 
+        writeln!(&mut self.file, "use super::ShortOffsetRunHeader;\n").unwrap();
         writeln!(
             &mut self.file,
-            "static SHORT_OFFSET_RUNS: [u32; {}] = [{}];",
+            "static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; {}] = [{}];",
             short_offset_runs.len(),
-            fmt_list(short_offset_runs.iter().map(|v| v.pack()))
+            fmt_list(short_offset_runs.iter())
         )
         .unwrap();
         self.bytes_used += 4 * short_offset_runs.len();
@@ -104,15 +102,43 @@ impl RawEmitter {
             writeln!(&mut self.file, "    (c as u32) >= {first_code_point:#04x} && lookup_slow(c)")
                 .unwrap();
             writeln!(&mut self.file, "}}").unwrap();
+            writeln!(&mut self.file).unwrap();
+            writeln!(&mut self.file, "#[inline(never)]").unwrap();
             writeln!(&mut self.file, "fn lookup_slow(c: char) -> bool {{").unwrap();
         } else {
             writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
         }
-        writeln!(&mut self.file, "    super::skip_search(",).unwrap();
+        writeln!(&mut self.file, "    const {{").unwrap();
-        writeln!(&mut self.file, "        c as u32,").unwrap();
+        writeln!(
-        writeln!(&mut self.file, "        &SHORT_OFFSET_RUNS,").unwrap();
+            &mut self.file,
-        writeln!(&mut self.file, "        &OFFSETS,").unwrap();
+            "        assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);",
-        writeln!(&mut self.file, "    )").unwrap();
+        )
+        .unwrap();
+        writeln!(&mut self.file, "        let mut i = 0;").unwrap();
+        writeln!(&mut self.file, "        while i < SHORT_OFFSET_RUNS.len() {{").unwrap();
+        writeln!(
+            &mut self.file,
+            "            assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());",
+        )
+        .unwrap();
+        writeln!(&mut self.file, "            i += 1;").unwrap();
+        writeln!(&mut self.file, "        }}").unwrap();
+        writeln!(&mut self.file, "    }}").unwrap();
+        writeln!(
+            &mut self.file,
+            "    // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`",
+        )
+        .unwrap();
+        writeln!(
+            &mut self.file,
+            "    // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.",
+        )
+        .unwrap();
+        writeln!(
+            &mut self.file,
+            "    unsafe {{ super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }}"
+        )
+        .unwrap();
         writeln!(&mut self.file, "}}").unwrap();
     }
 }

tests/codegen/autodiff.rs

@@ -11,7 +11,7 @@ fn square(x: &f64) -> f64 {
     x * x
 }
 
-// CHECK:define internal fastcc double @diffesquare(double %x.0.val, ptr nocapture align 8 %"x'"
+// CHECK:define internal fastcc double @diffesquare(double %x.0.val, ptr nocapture nonnull align 8 %"x'"
 // CHECK-NEXT:invertstart:
 // CHECK-NEXT:  %_0 = fmul double %x.0.val, %x.0.val
 // CHECK-NEXT:  %0 = fadd fast double %x.0.val, %x.0.val
@@ -22,7 +22,7 @@ fn square(x: &f64) -> f64 {
 // CHECK-NEXT:}
 
 fn main() {
-    let x = 3.0;
+    let x = std::hint::black_box(3.0);
     let output = square(&x);
     assert_eq!(9.0, output);
 

tests/codegen/autodiffv.rs

@@ -0,0 +1,116 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+//
+// In Enzyme, we test against a large range of LLVM versions (5+) and don't have overly many
+// breakages. One benefit is that we match the IR generated by Enzyme only after running it
+// through LLVM's O3 pipeline, which will remove most of the noise.
+// However, our integration test could also be affected by changes in how rustc lowers MIR into
+// LLVM-IR, which could cause additional noise and thus breakages. If that's the case, we should
+// reduce this test to only match the first lines and the ret instructions.
+
+#![feature(autodiff)]
+
+use std::autodiff::autodiff;
+
+#[autodiff(d_square3, Forward, Dual, DualOnly)]
+#[autodiff(d_square2, Forward, 4, Dual, DualOnly)]
+#[autodiff(d_square1, Forward, 4, Dual, Dual)]
+#[no_mangle]
+fn square(x: &f32) -> f32 {
+    x * x
+}
+
+// d_sqaure2
+// CHECK: define internal fastcc [4 x float] @fwddiffe4square(float %x.0.val, [4 x ptr] %"x'")
+// CHECK-NEXT: start:
+// CHECK-NEXT:   %0 = extractvalue [4 x ptr] %"x'", 0
+// CHECK-NEXT:   %"_2'ipl" = load float, ptr %0, align 4
+// CHECK-NEXT:   %1 = extractvalue [4 x ptr] %"x'", 1
+// CHECK-NEXT:   %"_2'ipl1" = load float, ptr %1, align 4
+// CHECK-NEXT:   %2 = extractvalue [4 x ptr] %"x'", 2
+// CHECK-NEXT:   %"_2'ipl2" = load float, ptr %2, align 4
+// CHECK-NEXT:   %3 = extractvalue [4 x ptr] %"x'", 3
+// CHECK-NEXT:   %"_2'ipl3" = load float, ptr %3, align 4
+// CHECK-NEXT:   %4 = insertelement <4 x float> poison, float %"_2'ipl", i64 0
+// CHECK-NEXT:   %5 = insertelement <4 x float> %4, float %"_2'ipl1", i64 1
+// CHECK-NEXT:   %6 = insertelement <4 x float> %5, float %"_2'ipl2", i64 2
+// CHECK-NEXT:   %7 = insertelement <4 x float> %6, float %"_2'ipl3", i64 3
+// CHECK-NEXT:   %8 = fadd fast <4 x float> %7, %7
+// CHECK-NEXT:   %9 = insertelement <4 x float> poison, float %x.0.val, i64 0
+// CHECK-NEXT:   %10 = shufflevector <4 x float> %9, <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:   %11 = fmul fast <4 x float> %8, %10
+// CHECK-NEXT:   %12 = extractelement <4 x float> %11, i64 0
+// CHECK-NEXT:   %13 = insertvalue [4 x float] undef, float %12, 0
+// CHECK-NEXT:   %14 = extractelement <4 x float> %11, i64 1
+// CHECK-NEXT:   %15 = insertvalue [4 x float] %13, float %14, 1
+// CHECK-NEXT:   %16 = extractelement <4 x float> %11, i64 2
+// CHECK-NEXT:   %17 = insertvalue [4 x float] %15, float %16, 2
+// CHECK-NEXT:   %18 = extractelement <4 x float> %11, i64 3
+// CHECK-NEXT:   %19 = insertvalue [4 x float] %17, float %18, 3
+// CHECK-NEXT:   ret [4 x float] %19
+// CHECK-NEXT: }
+
+// d_square3, the extra float is the original return value (x * x)
+// CHECK: define internal fastcc { float, [4 x float] } @fwddiffe4square.1(float %x.0.val, [4 x ptr] %"x'")
+// CHECK-NEXT: start:
+// CHECK-NEXT:   %0 = extractvalue [4 x ptr] %"x'", 0
+// CHECK-NEXT:   %"_2'ipl" = load float, ptr %0, align 4
+// CHECK-NEXT:   %1 = extractvalue [4 x ptr] %"x'", 1
+// CHECK-NEXT:   %"_2'ipl1" = load float, ptr %1, align 4
+// CHECK-NEXT:   %2 = extractvalue [4 x ptr] %"x'", 2
+// CHECK-NEXT:   %"_2'ipl2" = load float, ptr %2, align 4
+// CHECK-NEXT:   %3 = extractvalue [4 x ptr] %"x'", 3
+// CHECK-NEXT:   %"_2'ipl3" = load float, ptr %3, align 4
+// CHECK-NEXT:   %_0 = fmul float %x.0.val, %x.0.val
+// CHECK-NEXT:   %4 = insertelement <4 x float> poison, float %"_2'ipl", i64 0
+// CHECK-NEXT:   %5 = insertelement <4 x float> %4, float %"_2'ipl1", i64 1
+// CHECK-NEXT:   %6 = insertelement <4 x float> %5, float %"_2'ipl2", i64 2
+// CHECK-NEXT:   %7 = insertelement <4 x float> %6, float %"_2'ipl3", i64 3
+// CHECK-NEXT:   %8 = fadd fast <4 x float> %7, %7
+// CHECK-NEXT:   %9 = insertelement <4 x float> poison, float %x.0.val, i64 0
+// CHECK-NEXT:   %10 = shufflevector <4 x float> %9, <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:   %11 = fmul fast <4 x float> %8, %10
+// CHECK-NEXT:   %12 = extractelement <4 x float> %11, i64 0
+// CHECK-NEXT:   %13 = insertvalue [4 x float] undef, float %12, 0
+// CHECK-NEXT:   %14 = extractelement <4 x float> %11, i64 1
+// CHECK-NEXT:   %15 = insertvalue [4 x float] %13, float %14, 1
+// CHECK-NEXT:   %16 = extractelement <4 x float> %11, i64 2
+// CHECK-NEXT:   %17 = insertvalue [4 x float] %15, float %16, 2
+// CHECK-NEXT:   %18 = extractelement <4 x float> %11, i64 3
+// CHECK-NEXT:   %19 = insertvalue [4 x float] %17, float %18, 3
+// CHECK-NEXT:   %20 = insertvalue { float, [4 x float] } undef, float %_0, 0
+// CHECK-NEXT:   %21 = insertvalue { float, [4 x float] } %20, [4 x float] %19, 1
+// CHECK-NEXT:   ret { float, [4 x float] } %21
+// CHECK-NEXT: }
+
+fn main() {
+    let x = std::hint::black_box(3.0);
+    let output = square(&x);
+    dbg!(&output);
+    assert_eq!(9.0, output);
+    dbg!(square(&x));
+
+    let mut df_dx1 = 1.0;
+    let mut df_dx2 = 2.0;
+    let mut df_dx3 = 3.0;
+    let mut df_dx4 = 0.0;
+    let [o1, o2, o3, o4] = d_square2(&x, &mut df_dx1, &mut df_dx2, &mut df_dx3, &mut df_dx4);
+    dbg!(o1, o2, o3, o4);
+    let [output2, o1, o2, o3, o4] =
+        d_square1(&x, &mut df_dx1, &mut df_dx2, &mut df_dx3, &mut df_dx4);
+    dbg!(o1, o2, o3, o4);
+    assert_eq!(output, output2);
+    assert!((6.0 - o1).abs() < 1e-10);
+    assert!((12.0 - o2).abs() < 1e-10);
+    assert!((18.0 - o3).abs() < 1e-10);
+    assert!((0.0 - o4).abs() < 1e-10);
+    assert_eq!(1.0, df_dx1);
+    assert_eq!(2.0, df_dx2);
+    assert_eq!(3.0, df_dx3);
+    assert_eq!(0.0, df_dx4);
+    assert_eq!(d_square3(&x, &mut df_dx1), 2.0 * o1);
+    assert_eq!(d_square3(&x, &mut df_dx2), 2.0 * o2);
+    assert_eq!(d_square3(&x, &mut df_dx3), 2.0 * o3);
+    assert_eq!(d_square3(&x, &mut df_dx4), 2.0 * o4);
+}

tests/codegen/char-escape-debug-no-bounds-check.rs

@@ -0,0 +1,14 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+
+use std::char::EscapeDebug;
+
+// Make sure no bounds checks are emitted when escaping a character.
+
+// CHECK-LABEL: @char_escape_debug_no_bounds_check
+#[no_mangle]
+pub fn char_escape_debug_no_bounds_check(c: char) -> EscapeDebug {
+    // CHECK-NOT: panic
+    // CHECK-NOT: panic_bounds_check
+    c.escape_debug()
+}

tests/codegen/float/algebraic.rs

@@ -0,0 +1,149 @@
+// Verify that algebraic intrinsics generate the correct LLVM calls
+
+// Ensure operations get inlined
+//@ compile-flags: -Copt-level=1
+
+#![crate_type = "lib"]
+#![feature(f16)]
+#![feature(f128)]
+#![feature(float_algebraic)]
+
+// CHECK-LABEL: @f16_algebraic_add
+#[no_mangle]
+pub fn f16_algebraic_add(a: f16, b: f16) -> f16 {
+    // CHECK: fadd reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+    a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_sub
+#[no_mangle]
+pub fn f16_algebraic_sub(a: f16, b: f16) -> f16 {
+    // CHECK: fsub reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+    a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_mul
+#[no_mangle]
+pub fn f16_algebraic_mul(a: f16, b: f16) -> f16 {
+    // CHECK: fmul reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+    a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_div
+#[no_mangle]
+pub fn f16_algebraic_div(a: f16, b: f16) -> f16 {
+    // CHECK: fdiv reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+    a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f16_algebraic_rem
+#[no_mangle]
+pub fn f16_algebraic_rem(a: f16, b: f16) -> f16 {
+    // CHECK: frem reassoc nsz arcp contract half %{{.+}}, %{{.+}}
+    a.algebraic_rem(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_add
+#[no_mangle]
+pub fn f32_algebraic_add(a: f32, b: f32) -> f32 {
+    // CHECK: fadd reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+    a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_sub
+#[no_mangle]
+pub fn f32_algebraic_sub(a: f32, b: f32) -> f32 {
+    // CHECK: fsub reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+    a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_mul
+#[no_mangle]
+pub fn f32_algebraic_mul(a: f32, b: f32) -> f32 {
+    // CHECK: fmul reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+    a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_div
+#[no_mangle]
+pub fn f32_algebraic_div(a: f32, b: f32) -> f32 {
+    // CHECK: fdiv reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+    a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f32_algebraic_rem
+#[no_mangle]
+pub fn f32_algebraic_rem(a: f32, b: f32) -> f32 {
+    // CHECK: frem reassoc nsz arcp contract float %{{.+}}, %{{.+}}
+    a.algebraic_rem(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_add
+#[no_mangle]
+pub fn f64_algebraic_add(a: f64, b: f64) -> f64 {
+    // CHECK: fadd reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+    a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_sub
+#[no_mangle]
+pub fn f64_algebraic_sub(a: f64, b: f64) -> f64 {
+    // CHECK: fsub reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+    a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_mul
+#[no_mangle]
+pub fn f64_algebraic_mul(a: f64, b: f64) -> f64 {
+    // CHECK: fmul reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+    a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_div
+#[no_mangle]
+pub fn f64_algebraic_div(a: f64, b: f64) -> f64 {
+    // CHECK: fdiv reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+    a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f64_algebraic_rem
+#[no_mangle]
+pub fn f64_algebraic_rem(a: f64, b: f64) -> f64 {
+    // CHECK: frem reassoc nsz arcp contract double %{{.+}}, %{{.+}}
+    a.algebraic_rem(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_add
+#[no_mangle]
+pub fn f128_algebraic_add(a: f128, b: f128) -> f128 {
+    // CHECK: fadd reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+    a.algebraic_add(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_sub
+#[no_mangle]
+pub fn f128_algebraic_sub(a: f128, b: f128) -> f128 {
+    // CHECK: fsub reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+    a.algebraic_sub(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_mul
+#[no_mangle]
+pub fn f128_algebraic_mul(a: f128, b: f128) -> f128 {
+    // CHECK: fmul reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+    a.algebraic_mul(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_div
+#[no_mangle]
+pub fn f128_algebraic_div(a: f128, b: f128) -> f128 {
+    // CHECK: fdiv reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+    a.algebraic_div(b)
+}
+
+// CHECK-LABEL: @f128_algebraic_rem
+#[no_mangle]
+pub fn f128_algebraic_rem(a: f128, b: f128) -> f128 {
+    // CHECK: frem reassoc nsz arcp contract fp128 %{{.+}}, %{{.+}}
+    a.algebraic_rem(b)
+}

tests/codegen/float_math.rs

@@ -3,7 +3,10 @@
 #![crate_type = "lib"]
 #![feature(core_intrinsics)]
 
-use std::intrinsics::{fadd_fast, fdiv_fast, fmul_fast, frem_fast, fsub_fast};
+use std::intrinsics::{
+    fadd_algebraic, fadd_fast, fdiv_algebraic, fdiv_fast, fmul_algebraic, fmul_fast,
+    frem_algebraic, frem_fast, fsub_algebraic, fsub_fast,
+};
 
 // CHECK-LABEL: @add
 #[no_mangle]
@@ -13,30 +16,72 @@ pub fn add(x: f32, y: f32) -> f32 {
     x + y
 }
 
-// CHECK-LABEL: @addition
+// CHECK-LABEL: @test_fadd_algebraic
 #[no_mangle]
-pub fn addition(x: f32, y: f32) -> f32 {
+pub fn test_fadd_algebraic(x: f32, y: f32) -> f32 {
-    // CHECK: fadd fast float
+    // CHECK: fadd reassoc nsz arcp contract float %x, %y
+    fadd_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fsub_algebraic
+#[no_mangle]
+pub fn test_fsub_algebraic(x: f32, y: f32) -> f32 {
+    // CHECK: fsub reassoc nsz arcp contract float %x, %y
+    fsub_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fmul_algebraic
+#[no_mangle]
+pub fn test_fmul_algebraic(x: f32, y: f32) -> f32 {
+    // CHECK: fmul reassoc nsz arcp contract float %x, %y
+    fmul_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fdiv_algebraic
+#[no_mangle]
+pub fn test_fdiv_algebraic(x: f32, y: f32) -> f32 {
+    // CHECK: fdiv reassoc nsz arcp contract float %x, %y
+    fdiv_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_frem_algebraic
+#[no_mangle]
+pub fn test_frem_algebraic(x: f32, y: f32) -> f32 {
+    // CHECK: frem reassoc nsz arcp contract float %x, %y
+    frem_algebraic(x, y)
+}
+
+// CHECK-LABEL: @test_fadd_fast
+#[no_mangle]
+pub fn test_fadd_fast(x: f32, y: f32) -> f32 {
+    // CHECK: fadd fast float %x, %y
     unsafe { fadd_fast(x, y) }
 }
 
-// CHECK-LABEL: @subtraction
+// CHECK-LABEL: @test_fsub_fast
 #[no_mangle]
-pub fn subtraction(x: f32, y: f32) -> f32 {
+pub fn test_fsub_fast(x: f32, y: f32) -> f32 {
-    // CHECK: fsub fast float
+    // CHECK: fsub fast float %x, %y
     unsafe { fsub_fast(x, y) }
 }
 
-// CHECK-LABEL: @multiplication
+// CHECK-LABEL: @test_fmul_fast
 #[no_mangle]
-pub fn multiplication(x: f32, y: f32) -> f32 {
+pub fn test_fmul_fast(x: f32, y: f32) -> f32 {
-    // CHECK: fmul fast float
+    // CHECK: fmul fast float %x, %y
     unsafe { fmul_fast(x, y) }
 }
 
-// CHECK-LABEL: @division
+// CHECK-LABEL: @test_fdiv_fast
 #[no_mangle]
-pub fn division(x: f32, y: f32) -> f32 {
+pub fn test_fdiv_fast(x: f32, y: f32) -> f32 {
-    // CHECK: fdiv fast float
+    // CHECK: fdiv fast float %x, %y
     unsafe { fdiv_fast(x, y) }
 }
+
+// CHECK-LABEL: @test_frem_fast
+#[no_mangle]
+pub fn test_frem_fast(x: f32, y: f32) -> f32 {
+    // CHECK: frem fast float %x, %y
+    unsafe { frem_fast(x, y) }
+}

tests/crashes/137874.rs

@@ -1,4 +0,0 @@
-//@ known-bug: #137874
-fn a() {
-    match b { deref !(0c) };
-}

tests/pretty/autodiff_forward.pp

@@ -25,27 +25,31 @@ pub fn f1(x: &[f64], y: f64) -> f64 {
 
     // We want to be sure that the same function can be differentiated in different ways
 
+
+    // Make sure, that we add the None for the default return.
+
+
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Forward, Dual, Const, Dual,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Dual)]
 #[inline(never)]
-pub fn df1(x: &[f64], bx: &[f64], y: f64) -> (f64, f64) {
+pub fn df1(x: &[f64], bx_0: &[f64], y: f64) -> (f64, f64) {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f1(x, y));
-    ::core::hint::black_box((bx,));
+    ::core::hint::black_box((bx_0,));
-    ::core::hint::black_box((f1(x, y), f64::default()))
+    ::core::hint::black_box(<(f64, f64)>::default())
 }
 #[rustc_autodiff]
 #[inline(never)]
 pub fn f2(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Forward, Dual, Const, Const,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Const)]
 #[inline(never)]
-pub fn df2(x: &[f64], bx: &[f64], y: f64) -> f64 {
+pub fn df2(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f2(x, y));
-    ::core::hint::black_box((bx,));
+    ::core::hint::black_box((bx_0,));
     ::core::hint::black_box(f2(x, y))
 }
 #[rustc_autodiff]
@@ -53,20 +57,20 @@ pub fn df2(x: &[f64], bx: &[f64], y: f64) -> f64 {
 pub fn f3(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Forward, Dual, Const, Const,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Const)]
 #[inline(never)]
-pub fn df3(x: &[f64], bx: &[f64], y: f64) -> f64 {
+pub fn df3(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f3(x, y));
-    ::core::hint::black_box((bx,));
+    ::core::hint::black_box((bx_0,));
     ::core::hint::black_box(f3(x, y))
 }
 #[rustc_autodiff]
 #[inline(never)]
 pub fn f4() {}
-#[rustc_autodiff(Forward, None)]
+#[rustc_autodiff(Forward, 1, None)]
 #[inline(never)]
-pub fn df4() {
+pub fn df4() -> () {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f4());
     ::core::hint::black_box(());
@@ -76,28 +80,82 @@ pub fn df4() {
 pub fn f5(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Forward, Const, Dual, Const,)]
+#[rustc_autodiff(Forward, 1, Const, Dual, Const)]
 #[inline(never)]
-pub fn df5_y(x: &[f64], y: f64, by: f64) -> f64 {
+pub fn df5_y(x: &[f64], y: f64, by_0: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((by,));
+    ::core::hint::black_box((by_0,));
     ::core::hint::black_box(f5(x, y))
 }
-#[rustc_autodiff(Forward, Dual, Const, Const,)]
+#[rustc_autodiff(Forward, 1, Dual, Const, Const)]
 #[inline(never)]
-pub fn df5_x(x: &[f64], bx: &[f64], y: f64) -> f64 {
+pub fn df5_x(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((bx,));
+    ::core::hint::black_box((bx_0,));
     ::core::hint::black_box(f5(x, y))
 }
-#[rustc_autodiff(Reverse, Duplicated, Const, Active,)]
+#[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
 #[inline(never)]
-pub fn df5_rev(x: &[f64], dx: &mut [f64], y: f64, dret: f64) -> f64 {
+pub fn df5_rev(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((dx, dret));
+    ::core::hint::black_box((dx_0, dret));
     ::core::hint::black_box(f5(x, y))
 }
+struct DoesNotImplDefault;
+#[rustc_autodiff]
+#[inline(never)]
+pub fn f6() -> DoesNotImplDefault {
+    ::core::panicking::panic("not implemented")
+}
+#[rustc_autodiff(Forward, 1, Const)]
+#[inline(never)]
+pub fn df6() -> DoesNotImplDefault {
+    unsafe { asm!("NOP", options(pure, nomem)); };
+    ::core::hint::black_box(f6());
+    ::core::hint::black_box(());
+    ::core::hint::black_box(f6())
+}
+#[rustc_autodiff]
+#[inline(never)]
+pub fn f7(x: f32) -> () {}
+#[rustc_autodiff(Forward, 1, Const, None)]
+#[inline(never)]
+pub fn df7(x: f32) -> () {
+    unsafe { asm!("NOP", options(pure, nomem)); };
+    ::core::hint::black_box(f7(x));
+    ::core::hint::black_box(());
+}
+#[no_mangle]
+#[rustc_autodiff]
+#[inline(never)]
+fn f8(x: &f32) -> f32 { ::core::panicking::panic("not implemented") }
+#[rustc_autodiff(Forward, 4, Dual, Dual)]
+#[inline(never)]
+fn f8_3(x: &f32, bx_0: &f32, bx_1: &f32, bx_2: &f32, bx_3: &f32)
+    -> [f32; 5usize] {
+    unsafe { asm!("NOP", options(pure, nomem)); };
+    ::core::hint::black_box(f8(x));
+    ::core::hint::black_box((bx_0, bx_1, bx_2, bx_3));
+    ::core::hint::black_box(<[f32; 5usize]>::default())
+}
+#[rustc_autodiff(Forward, 4, Dual, DualOnly)]
+#[inline(never)]
+fn f8_2(x: &f32, bx_0: &f32, bx_1: &f32, bx_2: &f32, bx_3: &f32)
+    -> [f32; 4usize] {
+    unsafe { asm!("NOP", options(pure, nomem)); };
+    ::core::hint::black_box(f8(x));
+    ::core::hint::black_box((bx_0, bx_1, bx_2, bx_3));
+    ::core::hint::black_box(<[f32; 4usize]>::default())
+}
+#[rustc_autodiff(Forward, 1, Dual, DualOnly)]
+#[inline(never)]
+fn f8_1(x: &f32, bx_0: &f32) -> f32 {
+    unsafe { asm!("NOP", options(pure, nomem)); };
+    ::core::hint::black_box(f8(x));
+    ::core::hint::black_box((bx_0,));
+    ::core::hint::black_box(<f32>::default())
+}
 fn main() {}

tests/pretty/autodiff_forward.rs

@@ -36,4 +36,22 @@ pub fn f5(x: &[f64], y: f64) -> f64 {
     unimplemented!()
 }
 
+struct DoesNotImplDefault;
+#[autodiff(df6, Forward, Const)]
+pub fn f6() -> DoesNotImplDefault {
+    unimplemented!()
+}
+
+// Make sure, that we add the None for the default return.
+#[autodiff(df7, Forward, Const)]
+pub fn f7(x: f32) -> () {}
+
+#[autodiff(f8_1, Forward, Dual, DualOnly)]
+#[autodiff(f8_2, Forward, 4, Dual, DualOnly)]
+#[autodiff(f8_3, Forward, 4, Dual, Dual)]
+#[no_mangle]
+fn f8(x: &f32) -> f32 {
+    unimplemented!()
+}
+
 fn main() {}

tests/pretty/autodiff_reverse.pp

@@ -28,18 +28,18 @@ pub fn f1(x: &[f64], y: f64) -> f64 {
 
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Reverse, Duplicated, Const, Active,)]
+#[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
 #[inline(never)]
-pub fn df1(x: &[f64], dx: &mut [f64], y: f64, dret: f64) -> f64 {
+pub fn df1(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f1(x, y));
-    ::core::hint::black_box((dx, dret));
+    ::core::hint::black_box((dx_0, dret));
     ::core::hint::black_box(f1(x, y))
 }
 #[rustc_autodiff]
 #[inline(never)]
 pub fn f2() {}
-#[rustc_autodiff(Reverse, None)]
+#[rustc_autodiff(Reverse, 1, None)]
 #[inline(never)]
 pub fn df2() {
     unsafe { asm!("NOP", options(pure, nomem)); };
@@ -51,12 +51,12 @@ pub fn df2() {
 pub fn f3(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Reverse, Duplicated, Const, Active,)]
+#[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
 #[inline(never)]
-pub fn df3(x: &[f64], dx: &mut [f64], y: f64, dret: f64) -> f64 {
+pub fn df3(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f3(x, y));
-    ::core::hint::black_box((dx, dret));
+    ::core::hint::black_box((dx_0, dret));
     ::core::hint::black_box(f3(x, y))
 }
 enum Foo { Reverse, }
@@ -64,7 +64,7 @@ use Foo::Reverse;
 #[rustc_autodiff]
 #[inline(never)]
 pub fn f4(x: f32) { ::core::panicking::panic("not implemented") }
-#[rustc_autodiff(Reverse, Const, None)]
+#[rustc_autodiff(Reverse, 1, Const, None)]
 #[inline(never)]
 pub fn df4(x: f32) {
     unsafe { asm!("NOP", options(pure, nomem)); };
@@ -76,11 +76,11 @@ pub fn df4(x: f32) {
 pub fn f5(x: *const f32, y: &f32) {
     ::core::panicking::panic("not implemented")
 }
-#[rustc_autodiff(Reverse, DuplicatedOnly, Duplicated, None)]
+#[rustc_autodiff(Reverse, 1, DuplicatedOnly, Duplicated, None)]
 #[inline(never)]
-pub unsafe fn df5(x: *const f32, dx: *mut f32, y: &f32, dy: &mut f32) {
+pub unsafe fn df5(x: *const f32, dx_0: *mut f32, y: &f32, dy_0: &mut f32) {
     unsafe { asm!("NOP", options(pure, nomem)); };
     ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((dx, dy));
+    ::core::hint::black_box((dx_0, dy_0));
 }
 fn main() {}

tests/run-make/apple-slow-tls/rmake.rs

@@ -0,0 +1,37 @@
+//! Test if compilation with has-thread-local=false works, and if the output
+//! has indeed no fast TLS variables.
+
+//@ only-apple
+
+use run_make_support::serde_json::{self, Value};
+use run_make_support::{cargo, llvm_nm, rfs, rustc};
+
+fn main() {
+    let output =
+        rustc().print("target-spec-json").args(["-Z", "unstable-options"]).run().stdout_utf8();
+
+    let mut target_json: Value = serde_json::from_str(&output).unwrap();
+    let has_thread_local = &mut target_json["has-thread-local"];
+    assert!(matches!(has_thread_local, Value::Bool(true)), "{:?}", has_thread_local);
+    *has_thread_local = Value::Bool(false);
+
+    let out_path = "t.json";
+    rfs::write(out_path, serde_json::to_string(&target_json).unwrap());
+
+    cargo()
+        .args([
+            "b",
+            "--manifest-path",
+            "tls_test/Cargo.toml",
+            "--target",
+            "t.json",
+            "-Zbuild-std=std,core,panic_abort",
+        ])
+        .run();
+
+    // If a binary has any fast TLS variables, it should also contain the symbols
+    // __tlv_bootstrap and __tlv_atexit. We don't want them.
+    let output = llvm_nm().arg("tls_test/target/t/debug/tls_test").run().stdout_utf8();
+    assert!(!output.contains("_tlv_bootstrap"));
+    assert!(!output.contains("_tlv_atexit"));
+}

tests/run-make/apple-slow-tls/tls_test/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "tls_test"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]

tests/run-make/apple-slow-tls/tls_test/src/main.rs

@@ -0,0 +1,10 @@
+use std::cell::RefCell;
+
+fn main() {
+    thread_local! {
+        static S: RefCell<String> = RefCell::default();
+    }
+
+    S.with(|x| *x.borrow_mut() = "pika pika".to_string());
+    S.with(|x| println!("{}", x.borrow()));
+}

tests/run-make/target-specs/rmake.rs

@@ -14,7 +14,7 @@ fn main() {
         .input("foo.rs")
         .target("my-invalid-platform.json")
         .run_fail()
-        .assert_stderr_contains("Error loading target specification");
+        .assert_stderr_contains("error loading target specification");
     rustc()
         .input("foo.rs")
         .target("my-incomplete-platform.json")

tests/ui-fulldeps/stable-mir/check_assoc_items.rs

@@ -0,0 +1,145 @@
+//@ run-pass
+//! Test that users are able to retrieve all associated items from a definition.
+//! definition.
+
+//@ ignore-stage1
+//@ ignore-cross-compile
+//@ ignore-remote
+//@ edition: 2021
+
+#![feature(rustc_private)]
+#![feature(assert_matches)]
+
+extern crate rustc_middle;
+#[macro_use]
+extern crate rustc_smir;
+extern crate rustc_driver;
+extern crate rustc_interface;
+extern crate stable_mir;
+
+use rustc_smir::rustc_internal;
+use std::io::Write;
+use std::collections::HashSet;
+use stable_mir::CrateDef;
+use stable_mir::*;
+use stable_mir::ty::*;
+use std::ops::ControlFlow;
+
+const CRATE_NAME: &str = "crate_assoc_items";
+
+/// This function uses the Stable MIR APIs to get information about the test crate.
+fn test_assoc_items() -> ControlFlow<()> {
+    let local_crate = stable_mir::local_crate();
+    check_items(
+        &local_crate.fn_defs(),
+        &[
+            "AStruct::new",
+            "<AStruct as ATrait>::assoc_fn_no_self",
+            "<AStruct as ATrait>::assoc_fn_has_self",
+            "ATrait::rpitit",
+            "ATrait::assoc_fn_has_self",
+            "ATrait::assoc_fn_no_self",
+            "<AStruct as ATrait>::rpitit",
+        ],
+    );
+
+    let local_impls = local_crate.trait_impls();
+    let local_traits = local_crate.trait_decls();
+
+    let trait_assoc_item_defs: Vec<AssocDef> = local_traits[0].associated_items()
+        .iter().map(|assoc_item| assoc_item.def_id).collect();
+    check_items(
+        &trait_assoc_item_defs,
+        &[
+            "ATrait::{synthetic#0}",
+            "ATrait::rpitit",
+            "ATrait::Assoc",
+            "ATrait::assoc_fn_no_self",
+            "ATrait::assoc_fn_has_self",
+        ]
+    );
+
+    let impl_assoc_item_defs: Vec<AssocDef> = local_impls[0].associated_items()
+        .iter().map(|assoc_item| assoc_item.def_id).collect();
+    check_items(
+        &impl_assoc_item_defs,
+        &[
+            "<AStruct as ATrait>::{synthetic#0}",
+            "<AStruct as ATrait>::rpitit",
+            "<AStruct as ATrait>::Assoc",
+            "<AStruct as ATrait>::assoc_fn_no_self",
+            "<AStruct as ATrait>::assoc_fn_has_self",
+        ]
+    );
+
+    ControlFlow::Continue(())
+}
+
+/// Check if the list of definitions matches the expected list.
+/// Note that order doesn't matter.
+fn check_items<T: CrateDef>(items: &[T], expected: &[&str]) {
+    let expected: HashSet<_> = expected.iter().map(|s| s.to_string()).collect();
+    let item_names: HashSet<_> = items.iter().map(|item| item.name()).collect();
+    assert_eq!(item_names, expected);
+}
+
+fn main() {
+    let path = "assoc_items.rs";
+    generate_input(&path).unwrap();
+    let args = vec![
+        "rustc".to_string(),
+        "--crate-type=lib".to_string(),
+        "--crate-name".to_string(),
+        CRATE_NAME.to_string(),
+        path.to_string(),
+    ];
+    run!(args, test_assoc_items).unwrap();
+}
+
+fn generate_input(path: &str) -> std::io::Result<()> {
+    let mut file = std::fs::File::create(path)?;
+    write!(
+        file,
+        r#"
+        #![allow(dead_code, unused_variables)]
+        struct AStruct;
+
+        impl AStruct {{
+            const ASSOC_CONST: &str = "Nina";
+
+            fn new() -> Self {{
+                AStruct{{}}
+            }}
+        }}
+
+        trait ATrait {{
+            type Assoc;
+
+            fn assoc_fn_no_self() {{
+            }}
+
+            fn assoc_fn_has_self(&self) {{
+            }}
+
+            fn rpitit(&self) -> impl std::fmt::Debug {{
+                "ciallo"
+            }}
+        }}
+
+        impl ATrait for AStruct {{
+            type Assoc = u32;
+
+            fn assoc_fn_no_self() {{
+            }}
+
+            fn assoc_fn_has_self(&self) {{
+            }}
+
+            fn rpitit(&self) -> impl std::fmt::Debug {{
+                "ciallo~"
+            }}
+        }}
+    "#
+    )?;
+    Ok(())
+}

tests/ui/associated-consts/issue-93835.rs

@@ -3,11 +3,10 @@
 fn e() {
     type_ascribe!(p, a<p:p<e=6>>);
     //~^ ERROR cannot find type `a` in this scope
-    //~| ERROR path separator must be a double colon
     //~| ERROR cannot find value
     //~| ERROR associated const equality
+    //~| ERROR cannot find trait `p` in this scope
     //~| ERROR associated const equality
-    //~| ERROR failed to resolve: use of unresolved module or unlinked crate `p`
 }
 
 fn main() {}

tests/ui/associated-consts/issue-93835.stderr

@@ -1,15 +1,3 @@
-error: path separator must be a double colon
-  --> $DIR/issue-93835.rs:4:25
-   |
-LL |     type_ascribe!(p, a<p:p<e=6>>);
-   |                         ^
-   |
-   = note: if you meant to annotate an expression with a type, the type ascription syntax has been removed, see issue #101728 <https://github.com/rust-lang/rust/issues/101728>
-help: use a double colon instead
-   |
-LL |     type_ascribe!(p, a<p::p<e=6>>);
-   |                          +
-
 error[E0425]: cannot find value `p` in this scope
   --> $DIR/issue-93835.rs:4:19
    |
@@ -22,6 +10,12 @@ error[E0412]: cannot find type `a` in this scope
 LL |     type_ascribe!(p, a<p:p<e=6>>);
    |                      ^ not found in this scope
 
+error[E0405]: cannot find trait `p` in this scope
+  --> $DIR/issue-93835.rs:4:26
+   |
+LL |     type_ascribe!(p, a<p:p<e=6>>);
+   |                          ^ not found in this scope
+
 error[E0658]: associated const equality is incomplete
   --> $DIR/issue-93835.rs:4:28
    |
@@ -43,15 +37,7 @@ LL |     type_ascribe!(p, a<p:p<e=6>>);
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0433]: failed to resolve: use of unresolved module or unlinked crate `p`
+error: aborting due to 5 previous errors
-  --> $DIR/issue-93835.rs:4:24
-   |
-LL |     type_ascribe!(p, a<p:p<e=6>>);
-   |                        ^ use of unresolved module or unlinked crate `p`
-   |
-   = help: you might be missing a crate named `p`
 
-error: aborting due to 6 previous errors
+Some errors have detailed explanations: E0405, E0412, E0425, E0658.
-
+For more information about an error, try `rustc --explain E0405`.
-Some errors have detailed explanations: E0412, E0425, E0433, E0658.
-For more information about an error, try `rustc --explain E0412`.

tests/ui/autodiff/autodiff_illegal.rs

@@ -177,4 +177,11 @@ fn f21(x: f32) -> f32 {
     unimplemented!()
 }
 
+struct DoesNotImplDefault;
+#[autodiff(df22, Forward, Dual)]
+pub fn f22() -> DoesNotImplDefault {
+    //~^^ ERROR the function or associated item `default` exists for tuple `(DoesNotImplDefault, DoesNotImplDefault)`, but its trait bounds were not satisfied
+    unimplemented!()
+}
+
 fn main() {}

tests/ui/autodiff/autodiff_illegal.stderr

@@ -19,32 +19,24 @@ error: expected 1 activities, but found 2
    |
 LL | #[autodiff(df3, Reverse, Duplicated, Const)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: expected 1 activities, but found 0
   --> $DIR/autodiff_illegal.rs:27:1
    |
 LL | #[autodiff(df4, Reverse)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: Dual can not be used in Reverse Mode
   --> $DIR/autodiff_illegal.rs:34:1
    |
 LL | #[autodiff(df5, Reverse, Dual)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: Duplicated can not be used in Forward Mode
   --> $DIR/autodiff_illegal.rs:41:1
    |
 LL | #[autodiff(df6, Forward, Duplicated)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: Duplicated can not be used for this type
   --> $DIR/autodiff_illegal.rs:42:14
@@ -107,7 +99,6 @@ LL | #[autodiff(fn_exists, Reverse, Active)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `fn_exists` redefined here
    |
    = note: `fn_exists` must be defined only once in the value namespace of this module
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: autodiff requires at least a name and mode
   --> $DIR/autodiff_illegal.rs:95:1
@@ -135,42 +126,49 @@ error: invalid return activity Active in Forward Mode
    |
 LL | #[autodiff(df19, Forward, Dual, Active)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: invalid return activity Dual in Reverse Mode
   --> $DIR/autodiff_illegal.rs:167:1
    |
 LL | #[autodiff(df20, Reverse, Active, Dual)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: invalid return activity Duplicated in Reverse Mode
   --> $DIR/autodiff_illegal.rs:174:1
    |
 LL | #[autodiff(df21, Reverse, Active, Duplicated)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error[E0433]: failed to resolve: use of undeclared type `MyFloat`
   --> $DIR/autodiff_illegal.rs:130:1
    |
 LL | #[autodiff(df15, Reverse, Active, Active)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ use of undeclared type `MyFloat`
-   |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error[E0433]: failed to resolve: use of undeclared type `F64Trans`
   --> $DIR/autodiff_illegal.rs:154:1
    |
 LL | #[autodiff(df18, Reverse, Active, Active)]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ use of undeclared type `F64Trans`
+
+error[E0599]: the function or associated item `default` exists for tuple `(DoesNotImplDefault, DoesNotImplDefault)`, but its trait bounds were not satisfied
+  --> $DIR/autodiff_illegal.rs:181:1
+   |
+LL | struct DoesNotImplDefault;
+   | ------------------------- doesn't satisfy `DoesNotImplDefault: Default`
+LL | #[autodiff(df22, Forward, Dual)]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function or associated item cannot be called on `(DoesNotImplDefault, DoesNotImplDefault)` due to unsatisfied trait bounds
+   |
+   = note: the following trait bounds were not satisfied:
+           `DoesNotImplDefault: Default`
+           which is required by `(DoesNotImplDefault, DoesNotImplDefault): Default`
+help: consider annotating `DoesNotImplDefault` with `#[derive(Default)]`
+   |
+LL + #[derive(Default)]
+LL | struct DoesNotImplDefault;
    |
-   = note: this error originates in the attribute macro `autodiff` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: aborting due to 22 previous errors
+error: aborting due to 23 previous errors
 
-Some errors have detailed explanations: E0428, E0433, E0658.
+Some errors have detailed explanations: E0428, E0433, E0599, E0658.
 For more information about an error, try `rustc --explain E0428`.

tests/ui/errors/wrong-target-spec.rs

@@ -7,4 +7,4 @@
 
 fn main() {}
 
-//~? ERROR Error loading target specification: Could not find specification for target "x86_64_unknown-linux-musl"
+//~? ERROR error loading target specification: could not find specification for target "x86_64_unknown-linux-musl"

tests/ui/errors/wrong-target-spec.stderr

@@ -1,2 +1,4 @@
-error: Error loading target specification: Could not find specification for target "x86_64_unknown-linux-musl". Run `rustc --print target-list` for a list of built-in targets
+error: error loading target specification: could not find specification for target "x86_64_unknown-linux-musl"
+  |
+  = help: run `rustc --print target-list` for a list of built-in targets
 

tests/ui/macros/failed-to-reparse-issue-137874.rs

@@ -0,0 +1,12 @@
+// This originally crashed because `Recovery::Forbidden` wasn't being applied
+// when fragments pasted by declarative macros were reparsed.
+
+macro_rules! m {
+    ($p:pat) => {
+        if let $p = 0 {}
+    }
+}
+
+fn main() {
+    m!(0X0); //~ ERROR invalid base prefix for number literal
+}

tests/ui/macros/failed-to-reparse-issue-137874.stderr

@@ -0,0 +1,10 @@
+error: invalid base prefix for number literal
+  --> $DIR/failed-to-reparse-issue-137874.rs:11:8
+   |
+LL |     m!(0X0);
+   |        ^^^ help: try making the prefix lowercase (notice the capitalization): `0x0`
+   |
+   = note: base prefixes (`0xff`, `0b1010`, `0o755`) are lowercase
+
+error: aborting due to 1 previous error
+