perf(global): cache prefix records, parallelize conda-meta parsing, fast trampoline check

crates/pixi_cli/src/global/install.rs

@@ -248,9 +248,19 @@ async fn setup_environment(
     // Add shortcuts
     if !args.no_shortcuts {
         let prefix = project.environment_prefix(env_name).await?;
+        let prefix_records = project.environment_prefix_records(env_name).await?;
         for spec in specs.iter() {
-            let prefix_record = prefix.find_designated_package(spec.name()).await?;
-            if contains_menuinst_document(&prefix_record, prefix.root()) {
+            let prefix_record = prefix_records
+                .iter()
+                .find(|record| record.name() == spec.name())
+                .ok_or_else(|| {
+                    miette::miette!(
+                        "could not find package {} in environment {}",
+                        spec.name().as_source(),
+                        env_name.fancy_display()
+                    )
+                })?;
+            if contains_menuinst_document(prefix_record, prefix.root()) {
                 project.manifest.add_shortcut(env_name, spec.name())?;
             }
         }

crates/pixi_cli/src/global/remove.rs

@@ -71,14 +71,15 @@ pub async fn execute(args: Args) -> miette::Result<()> {
 
         // Figure out which package the exposed binaries belong to
         let prefix = project.environment_prefix(env_name).await?;
+        let prefix_records = project.environment_prefix_records(env_name).await?;
 
         for spec in specs {
             {
                 let name = spec.name.as_exact().expect("package name must be exact");
                 // If the package is not existent, don't try to remove executables
-                if let Ok(record) = prefix.find_designated_package(name).await {
+                if let Some(record) = prefix_records.iter().find(|record| record.name() == name) {
                     prefix
-                        .find_executables(&[record])
+                        .find_executables(std::slice::from_ref(record))
                         .into_iter()
                         .filter_map(|executable| {
                             ExposedName::from_str(executable.name.as_str()).ok()

crates/pixi_global/Cargo.toml

@@ -45,6 +45,7 @@ rattler_shell = { workspace = true }
 rattler_virtual_packages = { workspace = true }
 regex = { workspace = true }
 rstest = { workspace = true }
+same-file = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }
 serde_json = { workspace = true }

crates/pixi_global/src/common.rs

@@ -183,36 +183,31 @@ pub(crate) fn is_binary(file_path: impl AsRef<Path>) -> miette::Result<bool> {
     Ok(buffer[..bytes_read].contains(&0))
 }
 
-/// Finds the package record from the `conda-meta` directory.
+/// Finds the package records from the `conda-meta` directory.
+///
+/// The records are parsed in parallel on the blocking thread pool since the
+/// record files of a single environment can add up to tens of MBs of JSON.
 pub async fn find_package_records(conda_meta: &Path) -> miette::Result<Vec<PrefixRecord>> {
-    let read_dir = tokio_fs::read_dir(conda_meta).await;
-    let mut records = Vec::new();
-
-    let mut read_dir = match read_dir {
-        Ok(dir) => dir,
-        Err(e) => match e.kind() {
-            std::io::ErrorKind::NotFound => return Ok(records),
-            _ => miette::bail!(
-                "Failed to read conda-meta directory {}: {}",
-                conda_meta.display(),
-                e
-            ),
-        },
+    let Some(prefix_root) = conda_meta
+        .file_name()
+        .is_some_and(|name| name == OsStr::new(pixi_consts::consts::CONDA_META_DIR))
+        .then(|| conda_meta.parent())
+        .flatten()
+        .map(Path::to_path_buf)
+    else {
+        miette::bail!(
+            "Expected a path to a `{}` directory, got {}",
+            pixi_consts::consts::CONDA_META_DIR,
+            conda_meta.display()
+        );
     };
 
-    while let Some(entry) = read_dir.next_entry().await.into_diagnostic()? {
-        let path = entry.path();
-        // Check if the entry is a file and has a .json extension
-        if path.is_file() && path.extension().and_then(OsStr::to_str) == Some("json") {
-            let prefix_record = PrefixRecord::from_path(&path)
-                .into_diagnostic()
-                .wrap_err_with(|| format!("Couldn't parse json from {}", path.display()))?;
-
-            records.push(prefix_record);
-        }
-    }
-
-    Ok(records)
+    let prefix = pixi_utils::prefix::Prefix::new(prefix_root);
+    tokio::task::spawn_blocking(move || prefix.find_installed_packages())
+        .await
+        .into_diagnostic()?
+        .into_diagnostic()
+        .wrap_err_with(|| format!("Couldn't parse records from {}", conda_meta.display()))
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -440,12 +435,15 @@ impl StateChanges {
                     | InstallChange::Reinstalled(_, _)
             ) {
                 // Get the package record from the environment prefix
-                let prefix = project.environment_prefix(env_name).await?;
-                if let Ok(prefix_record) = prefix.find_designated_package(&package_name).await {
+                let prefix_records = project.environment_prefix_records(env_name).await?;
+                if let Some(prefix_record) = prefix_records
+                    .iter()
+                    .find(|record| record.name() == &package_name)
+                {
                     self.insert_change(
                         env_name,
                         StateChange::AddedPackage(Box::new(
-                            prefix_record.repodata_record.package_record,
+                            prefix_record.repodata_record.package_record.clone(),
                         )),
                     );
                 }
@@ -884,17 +882,17 @@ pub(crate) fn channel_url_to_prioritized_channel(
 /// contain menuinst JSON files. It then compares these records with the
 /// requested `shortcuts` to determine which records need to be installed and
 /// which need to be uninstalled.
-pub(crate) fn shortcuts_sync_status(
+pub(crate) fn shortcuts_sync_status<'a>(
     shortcuts: IndexSet<PackageName>,
-    prefix_records: Vec<PrefixRecord>,
+    prefix_records: &'a [PrefixRecord],
     prefix_root: &Path,
-) -> miette::Result<(Vec<PrefixRecord>, Vec<PrefixRecord>)> {
+) -> miette::Result<(Vec<&'a PrefixRecord>, Vec<&'a PrefixRecord>)> {
     let mut remaining_shortcuts = shortcuts;
     let mut records_to_install = Vec::new();
     let mut records_to_uninstall = Vec::new();
 
     let records_with_menuinst = prefix_records
-        .into_iter()
+        .iter()
         .filter(|record| contains_menuinst_document(record, prefix_root));
 
     for record in records_with_menuinst {

crates/pixi_global/src/install.rs

@@ -11,7 +11,7 @@ use pixi_utils::{
     prefix::{Executable, Prefix},
 };
 use rattler_conda_types::{
-    MatchSpec, Matches, PackageName, PackageRecord, ParseMatchSpecOptions, Platform,
+    MatchSpec, Matches, PackageName, PackageRecord, ParseMatchSpecOptions, Platform, PrefixRecord,
     RepodataRevision,
 };
 use rattler_shell::activation::prefix_path_entries;
@@ -344,23 +344,23 @@ pub(crate) fn local_environment_matches_spec(
 /// Finds the package name in the prefix and automatically exposes it if an executable is found.
 /// This is useful for packages like `ansible` and `jupyter` which don't ship executables their own executables.
 /// This function will return the mapping and the package name of the package in which the binary was found.
-pub async fn find_binary_by_name(
+pub fn find_binary_by_name(
     prefix: &Prefix,
     package_name: &PackageName,
-) -> miette::Result<Option<Executable>> {
-    let installed_packages = prefix.find_installed_packages()?;
-    for package in &installed_packages {
+    installed_packages: &[PrefixRecord],
+) -> Option<Executable> {
+    for package in installed_packages {
         let executables = prefix.find_executables(std::slice::from_ref(package));
 
         // Check if any of the executables match the package name
         if let Some(executable) = executables
             .iter()
             .find(|executable| executable.name.as_str() == package_name.as_normalized())
         {
-            return Ok(Some(executable.clone()));
+            return Some(executable.clone());
         }
     }
-    Ok(None)
+    None
 }
 
 #[cfg(test)]

crates/pixi_global/src/list.rs

@@ -4,14 +4,12 @@ use fancy_display::FancyDisplay;
 use indexmap::{IndexMap, IndexSet};
 use itertools::Itertools;
 use miette::{IntoDiagnostic, miette};
-use pixi_consts::consts;
 use pixi_core::environment::list::{PackageToOutput, print_package_table};
 use pixi_spec::PixiSpec;
 use rattler_conda_types::{PackageName, PrefixRecord, Version};
 use serde::Serialize;
 
 use super::{EnvChanges, EnvState, EnvironmentName, Mapping, Project, project::ParsedEnvironment};
-use crate::common::find_package_records;
 
 /// JSON-serializable representation of an exposed mapping.
 #[derive(Serialize)]
@@ -65,9 +63,7 @@ pub async fn list_global_environments_json(
 
     let mut environments = Vec::new();
     for (env_name, env) in project_envs.iter() {
-        let env_dir = project.env_root.path().join(env_name.as_str());
-        let conda_meta = env_dir.join(consts::CONDA_META_DIR);
-        let records = find_package_records(&conda_meta).await?;
+        let records = project.environment_prefix_records(env_name).await?;
 
         let dependencies = env
             .dependencies
@@ -230,14 +226,7 @@ pub async fn list_specific_global_environment(
         .get(environment_name)
         .ok_or_else(|| miette!("Environment {} not found", environment_name.fancy_display()))?;
 
-    let records = find_package_records(
-        &project
-            .env_root
-            .path()
-            .join(environment_name.as_str())
-            .join(consts::CONDA_META_DIR),
-    )
-    .await?;
+    let records = project.environment_prefix_records(environment_name).await?;
 
     let mut packages_to_output = records
         .iter()
@@ -327,9 +316,7 @@ pub async fn list_all_global_environments(
 
     let len = project_envs.len();
     for (idx, (env_name, env)) in project_envs.iter().enumerate() {
-        let env_dir = project.env_root.path().join(env_name.as_str());
-        let conda_meta = env_dir.join(consts::CONDA_META_DIR);
-        let records = find_package_records(&conda_meta).await?;
+        let records = project.environment_prefix_records(env_name).await?;
 
         let last = (idx + 1) == len;