[1/2] Unique the filesystem package cache by build inputs

CHANGELOG.md

@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Rust SDK
+
+- The FPI macro diagnostic for a dependency package missing from a midenc-driven build now names
+  the searched `MIDENC_PACKAGE_CACHE` directory and the expected package file names, instead of
+  an empty candidate list and a `target/miden/<profile>` hint that the cache lookup never
+  consults #1302
+- The FPI dependency package lookup matches the `.masp` extension case-insensitively, aligning
+  the macro-side reader with the compiler's cache writers on case-insensitive filesystems #1302
+- FPI expansions record `option_env!("MIDENC_PACKAGE_CACHE")`, so a consumer crate recompiles —
+  and re-reads its dependency procedure roots — whenever the compiler's fingerprinted package
+  cache path rotates, even if a stale cache directory survives on disk #1302
+
 ## [0.10.0-rc.1]
 
 ### Compiler and `midenc`

benches/src/lib.rs

@@ -191,12 +191,7 @@ impl BenchmarkRunner {
             .with_context(|| format!("failed to parse {}", inputs_path.display()))?;
         let mut executor = Executor::from_config(config);
         let packages_dir = project_dir.join("target/miden/packages");
-        let mut dependencies = fs::read_dir(&packages_dir)
-            .with_context(|| format!("failed to read {}", packages_dir.display()))?
-            .map(|entry| entry.map(|entry| entry.path()))
-            .collect::<std::io::Result<Vec<_>>>()?;
-        dependencies.retain(|path| path.extension().is_some_and(|ext| ext == "masp"));
-        dependencies.sort();
+        let dependencies = collect_dependency_packages(&packages_dir)?;
         for dependency in dependencies {
             executor
                 .with_package(load_package(&dependency)?)
@@ -214,6 +209,42 @@ impl BenchmarkRunner {
     }
 }
 
+/// Collects the dependency packages that the executor must load for an example.
+///
+/// The compiler stores dependency packages in the project package cache. Old compiler versions
+/// write them directly into `target/miden/packages/`. New compiler versions write them into a
+/// fingerprinted subdirectory of that cache. One runner binary drives both compiler versions
+/// during a benchmark comparison, so the scan reads the cache directory and one level of
+/// subdirectories. Package resolution at execution time is digest-addressed, so packages from a
+/// stale cache entry are inert.
+fn collect_dependency_packages(packages_dir: &Path) -> Result<Vec<PathBuf>> {
+    let mut dependencies = Vec::new();
+    for path in read_dir_paths(packages_dir)? {
+        if path.is_dir() {
+            dependencies
+                .extend(read_dir_paths(&path)?.into_iter().filter(|path| is_package_path(path)));
+        } else if is_package_path(&path) {
+            dependencies.push(path);
+        }
+    }
+    dependencies.sort();
+    Ok(dependencies)
+}
+
+/// Lists the entry paths of a directory.
+fn read_dir_paths(dir: &Path) -> Result<Vec<PathBuf>> {
+    fs::read_dir(dir)
+        .with_context(|| format!("failed to read {}", dir.display()))?
+        .map(|entry| entry.map(|entry| entry.path()))
+        .collect::<std::io::Result<Vec<_>>>()
+        .with_context(|| format!("failed to read {}", dir.display()))
+}
+
+/// Returns true when a path names a serialized Miden package file.
+fn is_package_path(path: &Path) -> bool {
+    path.extension().is_some_and(|ext| ext == "masp")
+}
+
 fn discover_cases(workspace_root: &Path) -> Result<Vec<BenchmarkCase>> {
     let examples_dir = workspace_root.join("examples");
     let mut cases = Vec::new();
@@ -358,6 +389,22 @@ mod tests {
         );
     }
 
+    #[test]
+    fn collects_packages_from_flat_and_fingerprinted_cache_layouts() {
+        let cache = tempfile::tempdir().unwrap();
+        fs::write(cache.path().join("legacy.masp"), []).unwrap();
+        fs::write(cache.path().join("1234567890abcdef.lock"), []).unwrap();
+        let fingerprint = cache.path().join("1234567890abcdef");
+        fs::create_dir_all(&fingerprint).unwrap();
+        fs::write(fingerprint.join("miden-core.masp"), []).unwrap();
+        fs::write(fingerprint.join("README.txt"), []).unwrap();
+
+        assert_eq!(
+            collect_dependency_packages(cache.path()).unwrap(),
+            vec![fingerprint.join("miden-core.masp"), cache.path().join("legacy.masp")]
+        );
+    }
+
     #[test]
     fn mast_size_excludes_package_metadata() {
         let mut package = Assembler::default()

midenc-compile/src/pipeline/frontends/masm.rs

@@ -267,7 +267,8 @@ impl MasmProjectFrontend {
     /// `load_target_sources`, and so this frontend, only afterwards (`:428-434`). For an
     /// executable root, `assemble_interruptible` also assembles the project's library target
     /// before either (`:303-320`). So a lint-only run builds the dependency closure and writes
-    /// `.masp` files into `target/miden/packages` that it previously never produced.
+    /// `.masp` files into `target/miden/packages/<build-fingerprint>` that it previously never
+    /// produced.
     ///
     /// That is an accepted consequence of routing project compilation through the assembler,
     /// not an oversight. The frontend is reachable only as an assembler callback, so any goal

midenc-compile/src/pipeline/frontends/rust.rs

@@ -1563,37 +1563,40 @@ pub(crate) mod manifest {
         let rustup_toolchain = crate::rust::rustup_toolchain();
         let cargo_build_args = build_cargo_args(cargo_opts, compiler_opts.optimize);
 
-        // Enable memcopy and 128-bit arithmetic ops
-        let mut extra_rust_flags = String::from("-C target-feature=+bulk-memory,+wide-arithmetic");
-        // Propagate the Miden VM target signal to the entire crate graph so Cargo can use it for
-        // cfg-based dependency selection.
-        extra_rust_flags.push_str(" --cfg miden");
-        // Enable errors on missing stub functions
-        extra_rust_flags.push_str(" -C link-args=--fatal-warnings");
-        // Remove the source file paths in the data segment for panics
-        // https://doc.rust-lang.org/beta/unstable-book/compiler-flags/location-detail.html
-        extra_rust_flags.push_str(" -Zlocation-detail=none");
-        // Build with panic=immediate-abort
-        extra_rust_flags.push_str(" -Zunstable-options");
-        extra_rust_flags.push_str(" -Cpanic=immediate-abort");
-        if let Ok(inherited) = std::env::var("RUSTFLAGS")
-            && !inherited.is_empty()
-        {
-            extra_rust_flags.push(' ');
-            extra_rust_flags.push_str(&inherited);
-        }
-        if let Some(explicit) = compiler_opts.rustflags.as_deref() {
-            extra_rust_flags.push(' ');
-            extra_rust_flags.push_str(explicit);
-        }
+        let mandatory_rust_flags = [
+            // Enable memcopy and 128-bit arithmetic ops
+            "-C",
+            "target-feature=+bulk-memory,+wide-arithmetic",
+            // Propagate the Miden VM target signal to the entire crate graph so Cargo can use it
+            // for cfg-based dependency selection.
+            "--cfg",
+            "miden",
+            // Enable errors on missing stub functions
+            "-C",
+            "link-args=--fatal-warnings",
+            // Remove the source file paths in the data segment for panics
+            // https://doc.rust-lang.org/beta/unstable-book/compiler-flags/location-detail.html
+            "-Zlocation-detail=none",
+            // Build with panic=immediate-abort
+            "-Zunstable-options",
+            "-Cpanic=immediate-abort",
+        ];
+        let inherited_encoded = std::env::var_os("CARGO_ENCODED_RUSTFLAGS");
+        let inherited_plain = std::env::var_os("RUSTFLAGS");
+        let extra_rust_flags = merge_rust_flags(
+            &mandatory_rust_flags,
+            inherited_encoded.as_deref(),
+            inherited_plain.as_deref(),
+            compiler_opts.rustflags.as_deref(),
+        );
 
         let wasi = if compiler_opts.target_requires_protocol() {
             "wasip2"
         } else {
             "wasip1"
         };
 
-        let env = cargo_env(filesystem_cache_dir, extra_rust_flags);
+        let env = cargo_env(filesystem_cache_dir, &extra_rust_flags);
         let mut wasm_outputs =
             run_cargo(wasi, rustup_toolchain.as_deref(), &cargo_build_args, env)?;
 
@@ -1607,24 +1610,64 @@ pub(crate) mod manifest {
     ///
     /// `MIDENC_PACKAGE_CACHE` is the whole reason `filesystem_cache_dir` is threaded down from
     /// the entry point: it tells the nested `midenc` invocations where to publish the packages
-    /// they compile and where to look for the ones their own dependencies already produced.
-    /// Absent a cache directory the variable is *unset* rather than set to an empty path, which
-    /// is what makes the nested build fall back to its own default.
+    /// they compile and where to look for the ones their own dependencies already produced. The
+    /// directory is fingerprinted by the root build's compiler, options, and manifest closure so
+    /// every participant in that build uses the same isolated cache. Absent a cache directory the
+    /// variable is *unset* rather than set to an empty path, which is what makes the nested build
+    /// fall back to its own default.
+    ///
+    /// The composed rust flags are emitted twice: as `RUSTFLAGS` (space-joined, lossy for
+    /// arguments that contain spaces) and, authoritatively, as `CARGO_ENCODED_RUSTFLAGS`
+    /// (0x1f-joined). Cargo prefers the encoded variable, so emitting it explicitly is what keeps
+    /// the mandatory Miden flags — `--cfg miden`, the target features, the panic strategy — from
+    /// being replaced by an inherited value; the caller's own flags survive because
+    /// [`merge_rust_flags`] folds them into the composed list first.
     ///
     /// Named — rather than left inline where it was — so that this can be asserted without
     /// spawning `cargo -Z build-std` against the SDK.
     pub(super) fn cargo_env(
         filesystem_cache_dir: Option<&Path>,
-        extra_rust_flags: String,
+        extra_rust_flags: &[String],
     ) -> Vec<(&'static str, String)> {
+        let mut env = vec![
+            ("RUSTFLAGS", extra_rust_flags.join(" ")),
+            ("CARGO_ENCODED_RUSTFLAGS", extra_rust_flags.join("\x1f")),
+        ];
         if let Some(filesystem_cache_dir) = filesystem_cache_dir {
-            vec![
-                ("RUSTFLAGS", extra_rust_flags),
-                ("MIDENC_PACKAGE_CACHE", filesystem_cache_dir.to_string_lossy().into_owned()),
-            ]
-        } else {
-            vec![("RUSTFLAGS", extra_rust_flags)]
+            env.push(("MIDENC_PACKAGE_CACHE", filesystem_cache_dir.to_string_lossy().into_owned()));
+        }
+        env
+    }
+
+    /// Composes the rustc flag list for the nested cargo build.
+    ///
+    /// Inherited flags follow cargo's own precedence: a non-empty `CARGO_ENCODED_RUSTFLAGS`
+    /// (0x1f-separated; arguments may contain spaces) replaces plain `RUSTFLAGS`
+    /// (whitespace-split). Inherited flags are folded in after the mandatory set and before the
+    /// explicit `--rustflags`, preserving the pre-existing override order — nothing a caller
+    /// passes is dropped.
+    pub(super) fn merge_rust_flags(
+        mandatory: &[&str],
+        inherited_encoded: Option<&std::ffi::OsStr>,
+        inherited_plain: Option<&std::ffi::OsStr>,
+        explicit: Option<&str>,
+    ) -> Vec<String> {
+        let mut args: Vec<String> = mandatory.iter().map(|flag| flag.to_string()).collect();
+        let encoded = inherited_encoded
+            .and_then(|value| value.to_str())
+            .filter(|value| !value.is_empty());
+        let plain = inherited_plain
+            .and_then(|value| value.to_str())
+            .filter(|value| !value.is_empty());
+        if let Some(encoded) = encoded {
+            args.extend(encoded.split('\x1f').filter(|arg| !arg.is_empty()).map(str::to_string));
+        } else if let Some(plain) = plain {
+            args.extend(plain.split_whitespace().map(str::to_string));
+        }
+        if let Some(explicit) = explicit {
+            args.extend(explicit.split_whitespace().map(str::to_string));
         }
+        args
     }
 
     /// Returns the Cargo profile value for a compiler optimization level.
@@ -3152,12 +3195,13 @@ path = "lib.rs"
     /// which is minutes of work and a network fetch.
     #[test]
     fn the_filesystem_cache_directory_is_handed_to_the_nested_cargo_build() {
-        let rustflags = String::from("-C target-feature=+bulk-memory");
+        let rustflags = vec!["-C".to_string(), "target-feature=+bulk-memory".to_string()];
         let cache = std::path::Path::new("/tmp/midenc-package-cache");
 
-        let with = manifest::cargo_env(Some(cache), rustflags.clone());
+        let with = manifest::cargo_env(Some(cache), &rustflags);
         assert!(
-            with.iter().any(|(key, value)| *key == "RUSTFLAGS" && *value == rustflags),
+            with.iter()
+                .any(|(key, value)| *key == "RUSTFLAGS" && *value == rustflags.join(" ")),
             "the rust flags must survive the cache directory being added: {with:?}"
         );
         let (_, found) = with
@@ -3166,17 +3210,86 @@ path = "lib.rs"
             .expect("a cache directory must be delivered to the nested build");
         assert_eq!(found, &cache.display().to_string());
 
-        let without = manifest::cargo_env(None, rustflags.clone());
+        let without = manifest::cargo_env(None, &rustflags);
         assert!(
             !without.iter().any(|(key, _)| *key == "MIDENC_PACKAGE_CACHE"),
             "no cache directory means the variable is unset, not set to nothing: {without:?}"
         );
         assert!(
-            without.iter().any(|(key, value)| *key == "RUSTFLAGS" && *value == rustflags),
+            without
+                .iter()
+                .any(|(key, value)| *key == "RUSTFLAGS" && *value == rustflags.join(" ")),
             "and the rust flags are handed over either way: {without:?}"
         );
     }
 
+    #[test]
+    fn the_encoded_rustflags_override_inherited_values_with_the_same_flags() {
+        let rustflags = vec![
+            "-C".to_string(),
+            "target-feature=+bulk-memory".to_string(),
+            "--cfg".to_string(),
+            "miden".to_string(),
+        ];
+
+        let env = manifest::cargo_env(None, &rustflags);
+        let (_, encoded) = env
+            .iter()
+            .find(|(key, _)| *key == "CARGO_ENCODED_RUSTFLAGS")
+            .expect("the encoded variable must be set so an inherited value cannot override it");
+        // 0x1f-separated units, one per argument.
+        assert_eq!(*encoded, "-C\x1ftarget-feature=+bulk-memory\x1f--cfg\x1fmiden");
+    }
+
+    #[test]
+    fn merged_rust_flags_preserve_inherited_encoded_arguments() {
+        use std::ffi::OsStr;
+
+        let mandatory = ["--cfg", "miden"];
+
+        // A non-empty encoded value wins over the plain spelling, mirroring cargo's precedence,
+        // and its 0x1f-separated arguments survive verbatim — including ones with spaces.
+        let merged = manifest::merge_rust_flags(
+            &mandatory,
+            Some(OsStr::new("-C\x1flink-args=--flag one --flag two")),
+            Some(OsStr::new("--cfg plain_ignored")),
+            Some("--cfg explicit"),
+        );
+        assert_eq!(
+            merged,
+            vec![
+                "--cfg".to_string(),
+                "miden".to_string(),
+                "-C".to_string(),
+                "link-args=--flag one --flag two".to_string(),
+                "--cfg".to_string(),
+                "explicit".to_string(),
+            ]
+        );
+
+        // Without an encoded value the plain spelling is whitespace-split, as cargo does.
+        let merged =
+            manifest::merge_rust_flags(&mandatory, None, Some(OsStr::new("-C opt-level=3")), None);
+        assert_eq!(
+            merged,
+            vec![
+                "--cfg".to_string(),
+                "miden".to_string(),
+                "-C".to_string(),
+                "opt-level=3".to_string(),
+            ]
+        );
+
+        // Empty inherited values are treated as unset.
+        let merged = manifest::merge_rust_flags(
+            &mandatory,
+            Some(OsStr::new("")),
+            Some(OsStr::new("")),
+            None,
+        );
+        assert_eq!(merged, vec!["--cfg".to_string(), "miden".to_string()]);
+    }
+
     /// A WebAssembly module with a body, for the lowering half of the entry point.
     const MANIFEST_WAT: &str = r#"
 (module

midenc-session/Cargo.toml

@@ -51,3 +51,6 @@ smallvec.workspace = true
 parking_lot = { workspace = true, optional = true }
 termcolor = { version = "1.4.1", optional = true }
 thiserror.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true