Auto merge of #91125 - eskarn:llvm-passes-plugin-support, r=nagisa

Allow loading LLVM plugins with both legacy and new pass manager

Opening a draft PR to get feedback and start discussion on this feature. There is already a codegen option `passes` which allow giving a list of LLVM pass names, however we currently can't use a LLVM pass plugin (as described here : https://llvm.org/docs/WritingAnLLVMPass.html), the only available passes are the LLVM built-in ones.

The proposed modification would be to add another codegen option `pass-plugins`, which can be set with a list of paths to shared library files. These libraries are loaded using the LLVM function `PassPlugin::Load`, which calls the expected symbol `lvmGetPassPluginInfo`, and register the pipeline parsing and optimization callbacks.

An example usage with a single plugin and 3 passes would look like this in the `.cargo/config`:

```toml
rustflags = [
    "-C", "pass-plugins=/tmp/libLLVMPassPlugin",
    "-C", "passes=pass1 pass2 pass3",
]
```
This would give the same functionality as the opt LLVM tool directly integrated in rust build system.

Additionally, we can also not specify the `passes` option, and use a plugin which inserts passes in the optimization pipeline, as one could do using clang.

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -3,7 +3,7 @@ use crate::back::write::{
 };
 use crate::llvm::archive_ro::ArchiveRO;
 use crate::llvm::{self, build_string, False, True};
-use crate::{LlvmCodegenBackend, ModuleLlvm};
+use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm};
 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared};
 use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{
@@ -596,7 +596,10 @@ pub(crate) fn run_pass_manager(
     //      tools/lto/LTOCodeGenerator.cpp
     debug!("running the pass manager");
     unsafe {
-        if write::should_use_new_llvm_pass_manager(cgcx, config) {
+        if llvm_util::should_use_new_llvm_pass_manager(
+            &config.new_llvm_pass_manager,
+            &cgcx.target_arch,
+        ) {
             let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
             let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
             write::optimize_with_new_llvm_pass_manager(

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -416,21 +416,6 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
         .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
 }
 
-pub(crate) fn should_use_new_llvm_pass_manager(
-    cgcx: &CodegenContext<LlvmCodegenBackend>,
-    config: &ModuleConfig,
-) -> bool {
-    // The new pass manager is enabled by default for LLVM >= 13.
-    // This matches Clang, which also enables it since Clang 13.
-
-    // FIXME: There are some perf issues with the new pass manager
-    // when targeting s390x, so it is temporarily disabled for that
-    // arch, see https://github.com/rust-lang/rust/issues/89609
-    config
-        .new_llvm_pass_manager
-        .unwrap_or_else(|| cgcx.target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0))
-}
-
 pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     diag_handler: &Handler,
@@ -473,6 +458,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
 
     let extra_passes = config.passes.join(",");
 
+    let llvm_plugins = config.llvm_plugins.join(",");
+
     // FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
     // We would have to add upstream support for this first, before we can support
     // config.inline_threshold and our more aggressive default thresholds.
@@ -502,6 +489,8 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
         selfprofile_after_pass_callback,
         extra_passes.as_ptr().cast(),
         extra_passes.len(),
+        llvm_plugins.as_ptr().cast(),
+        llvm_plugins.len(),
     );
     result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes"))
 }
@@ -530,7 +519,10 @@ pub(crate) unsafe fn optimize(
     }
 
     if let Some(opt_level) = config.opt_level {
-        if should_use_new_llvm_pass_manager(cgcx, config) {
+        if llvm_util::should_use_new_llvm_pass_manager(
+            &config.new_llvm_pass_manager,
+            &cgcx.target_arch,
+        ) {
             let opt_stage = match cgcx.lto {
                 Lto::Fat => llvm::OptStage::PreLinkFatLTO,
                 Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2313,6 +2313,8 @@ extern "C" {
         end_callback: SelfProfileAfterPassCallback,
         ExtraPasses: *const c_char,
         ExtraPassesLen: size_t,
+        LLVMPlugins: *const c_char,
+        LLVMPluginsLen: size_t,
     ) -> LLVMRustResult;
     pub fn LLVMRustPrintModule(
         M: &Module,

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -121,14 +121,20 @@ unsafe fn configure_llvm(sess: &Session) {
 
     llvm::LLVMInitializePasses();
 
-    // Register LLVM plugins by loading them into the compiler process.
-    for plugin in &sess.opts.debugging_opts.llvm_plugins {
-        let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e));
-        debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin);
+    // Use the legacy plugin registration if we don't use the new pass manager
+    if !should_use_new_llvm_pass_manager(
+        &sess.opts.debugging_opts.new_llvm_pass_manager,
+        &sess.target.arch,
+    ) {
+        // Register LLVM plugins by loading them into the compiler process.
+        for plugin in &sess.opts.debugging_opts.llvm_plugins {
+            let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e));
+            debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin);
 
-        // Intentionally leak the dynamic library. We can't ever unload it
-        // since the library can make things that will live arbitrarily long.
-        mem::forget(lib);
+            // Intentionally leak the dynamic library. We can't ever unload it
+            // since the library can make things that will live arbitrarily long.
+            mem::forget(lib);
+        }
     }
 
     rustc_llvm::initialize_available_targets();
@@ -411,3 +417,13 @@ pub fn tune_cpu(sess: &Session) -> Option<&str> {
     let name = sess.opts.debugging_opts.tune_cpu.as_ref()?;
     Some(handle_native(name))
 }
+
+pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option<bool>, target_arch: &str) -> bool {
+    // The new pass manager is enabled by default for LLVM >= 13.
+    // This matches Clang, which also enables it since Clang 13.
+
+    // FIXME: There are some perf issues with the new pass manager
+    // when targeting s390x, so it is temporarily disabled for that
+    // arch, see https://github.com/rust-lang/rust/issues/89609
+    user_opt.unwrap_or_else(|| target_arch != "s390x" && llvm_util::get_version() >= (13, 0, 0))
+}