diff --git a/Cargo.lock b/Cargo.lock index 690d40a0936..5f485a6febe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2446,9 +2446,11 @@ dependencies = [ "gix-status", "gix-testtools", "gix-worktree", + "hashbrown 0.16.1", "portable-atomic", "pretty_assertions", "thiserror 2.0.18", + "windows-sys 0.61.2", ] [[package]] diff --git a/gitoxide-core/src/repository/status.rs b/gitoxide-core/src/repository/status.rs index 04abb15eea3..c425277e55b 100644 --- a/gitoxide-core/src/repository/status.rs +++ b/gitoxide-core/src/repository/status.rs @@ -41,6 +41,7 @@ pub struct Options { pub statistics: bool, pub allow_write: bool, pub index_worktree_renames: Option, + pub untracked: Option, } pub fn show( @@ -58,6 +59,7 @@ pub fn show( allow_write, statistics, index_worktree_renames, + untracked, }: Options, ) -> anyhow::Result<()> { if output_format != OutputFormat::Human { @@ -70,9 +72,13 @@ pub fn show( let start = std::time::Instant::now(); let prefix = repo.prefix()?.unwrap_or(Path::new("")); let index_progress = progress.add_child("traverse index"); - let mut iter = repo + let mut status = repo .status(index_progress)? - .should_interrupt_shared(&gix::interrupt::IS_INTERRUPTED) + .should_interrupt_shared(&gix::interrupt::IS_INTERRUPTED); + if let Some(untracked) = untracked { + status = status.untracked_files(untracked); + } + let mut iter = status .index_worktree_options_mut(|opts| { if let Some((opts, ignored)) = opts.dirwalk_options.as_mut().zip(ignored) { opts.set_emit_ignored(Some(match ignored { diff --git a/gix-index/src/entry/mode.rs b/gix-index/src/entry/mode.rs index 3b29548485f..58d66631d4c 100644 --- a/gix-index/src/entry/mode.rs +++ b/gix-index/src/entry/mode.rs @@ -46,22 +46,57 @@ impl Mode { stat: &crate::fs::Metadata, has_symlinks: bool, executable_bit: bool, + ) -> Option { + self.change_to_match_fs_with_values( + stat.is_file(), + stat.is_dir(), + stat.is_symlink(), + stat.is_executable(), + has_symlinks, + executable_bit, + ) + } + + /// Like [`change_to_match_fs`](Self::change_to_match_fs) but accepts pre-extracted + /// file-type and permission bits, for callers that already have them (e.g. cached + /// metadata from a batched directory enumeration). + /// + /// Note that some parameters are mutually exclusive, but inconsistency won't be a problem + /// if the source of the data is consistent. + /// + /// * `is_file` is `true` if the file-system entry is a regular file. + /// * `is_dir` is `true` if the file-system entry is a directory. + /// * `is_symlink` is `true` if the file-system entry is a symbolic link. + /// * `is_executable` is `true` if the file-system entry has executable permissions. + /// + /// These parameters match [`change_to_match_fs`](Self::change_to_match_fs). + /// + /// * `has_symlinks` is `true` if the file system represents symbolic links faithfully. + /// * `executable_bit` is `true` if the file system represents executable permissions faithfully. + pub fn change_to_match_fs_with_values( + self, + is_file: bool, + is_dir: bool, + is_symlink: bool, + is_executable: bool, + has_symlinks: bool, + executable_bit: bool, ) -> Option { match self { - Mode::FILE if !stat.is_file() => (), - Mode::SYMLINK if stat.is_symlink() => return None, - Mode::SYMLINK if has_symlinks && !stat.is_symlink() => (), - Mode::SYMLINK if !has_symlinks && !stat.is_file() => (), - Mode::COMMIT | Mode::DIR if !stat.is_dir() => (), - Mode::FILE if executable_bit && stat.is_executable() => return Some(Change::ExecutableBit), - Mode::FILE_EXECUTABLE if executable_bit && !stat.is_executable() => return Some(Change::ExecutableBit), + Mode::FILE if !is_file => (), + Mode::SYMLINK if is_symlink => return None, + Mode::SYMLINK if has_symlinks && !is_symlink => (), + Mode::SYMLINK if !has_symlinks && !is_file => (), + Mode::COMMIT | Mode::DIR if !is_dir => (), + Mode::FILE if executable_bit && is_executable => return Some(Change::ExecutableBit), + Mode::FILE_EXECUTABLE if executable_bit && !is_executable => return Some(Change::ExecutableBit), _ => return None, } - let new_mode = if stat.is_dir() { + let new_mode = if is_dir { Mode::COMMIT - } else if executable_bit && stat.is_executable() { + } else if executable_bit && is_executable { Mode::FILE_EXECUTABLE - } else if has_symlinks && stat.is_symlink() { + } else if has_symlinks && is_symlink { Mode::SYMLINK } else { Mode::FILE diff --git a/gix-status/Cargo.toml b/gix-status/Cargo.toml index 5263d2ceae2..0a4ab054a5a 100644 --- a/gix-status/Cargo.toml +++ b/gix-status/Cargo.toml @@ -47,6 +47,16 @@ document-features = { version = "0.2.0", optional = true } [target.'cfg(not(target_has_atomic = "64"))'.dependencies] portable-atomic = "1" +[target.'cfg(windows)'.dependencies] +hashbrown = "0.16.0" +windows-sys = { version = "0.61.1", features = [ + "Win32_Foundation", + "Win32_Storage_FileSystem", + # Needed for `SECURITY_ATTRIBUTES` in the signature of `CreateFileW`, + # which is gated behind this windows-sys feature even when the call passes null. + "Win32_Security", +] } + [dev-dependencies] gix-status = { path = ".", features = ["worktree-rewrites", "parallel"] } gix-hash = { path = "../gix-hash", features = ["sha1"] } diff --git a/gix-status/src/fscache.rs b/gix-status/src/fscache.rs new file mode 100644 index 00000000000..eec8f1c700b --- /dev/null +++ b/gix-status/src/fscache.rs @@ -0,0 +1,526 @@ +//! **Windows-only** worktree metadata caching. During per-entry modification +//! checks, parent directories can be enumerated lazily so that +//! `index_as_worktree` can reuse stat results instead of issuing a per-file +//! `lstat`. This trade only pays off where per-file stat is expensive +//! and where the dirwalk returns basic stat information, so only on Windows. +//! +//! [`FsCache`] lazily enumerates parent directories with +//! `GetFileInformationByHandleEx` and keeps the results in the status worker +//! that asked for them. It is not a long-lived cache: it is built during one +//! status call and discarded with the worker state. Lookups are transparent: +//! misses fall through to a live syscall, so cache misses affect speed only. + +use std::path::{Path, PathBuf}; + +use bstr::{BStr, BString, ByteSlice}; + +/// File metadata produced by the lazy cache for one worktree entry. +/// +/// Carries enough information to determine file type, detect mode changes, +/// build a [`gix_index::entry::Stat`] for comparison, and short-circuit content +/// reads via file size. +/// +/// Windows-only fields: this module is `#[cfg(windows)]`, and Windows batch +/// directory enumeration doesn't expose `dev`/`ino`/`uid`/`gid` or the +/// executable bit. The status pipeline's stat comparison on Windows compares +/// those `Stat` fields against matching zeros from +/// [`gix_index::entry::Stat::from_fs`]'s Windows branch, and git on Windows +/// defaults to `core.filemode=false`, so all five are simply omitted here. +#[derive(Debug, Clone, Copy, Default)] +pub struct Stat { + /// Whether this is a directory, with the same semantics as + /// `std::fs::symlink_metadata`: `false` for directory symlinks and junctions + /// even though the filesystem also marks those with the directory attribute. + pub is_dir: bool, + /// Whether this is a symlink, with the same semantics as + /// `std::fs::symlink_metadata`: `true` only for name-surrogate reparse points + /// (symlinks and junctions/mount points). Other reparse points — OneDrive + /// cloud placeholders, ProjFS, app-exec links — are regular files/dirs, as + /// the live `lstat` fallback would report them. + pub is_symlink: bool, + /// File size in bytes. + pub size: u64, + /// Modification time — seconds since Unix epoch. + pub mtime_secs: u32, + /// Modification time — nanoseconds component. + pub mtime_nsecs: u32, + /// Status/creation time — seconds since Unix epoch. + /// + /// On Windows this must be populated from the real `CreationTime`, not `mtime`: + /// the stat comparison in the status pipeline compares `ctime.secs` by default + /// (`trust_ctime=true`), and faking `ctime=mtime` causes spurious mismatches + /// for any file whose creation-time and modification-time differ. + pub ctime_secs: u32, + /// Status/creation time — nanoseconds component. + pub ctime_nsecs: u32, +} + +impl Stat { + /// Convert to gitoxide's [`Stat`](gix_index::entry::Stat) struct for index comparison. + /// + /// Truncates `size` from 64 to 32 bits — matching what + /// [`gix_index::entry::stat::Stat::from_fs`] does on Unix, so both code + /// paths compare the same quantities. `dev`/`ino`/`uid`/`gid` are zeroed + /// here to match what `from_fs` produces on Windows. + pub(crate) fn to_stat(&self) -> gix_index::entry::Stat { + gix_index::entry::Stat { + mtime: gix_index::entry::stat::Time { + secs: self.mtime_secs, + nsecs: self.mtime_nsecs, + }, + ctime: gix_index::entry::stat::Time { + secs: self.ctime_secs, + nsecs: self.ctime_nsecs, + }, + dev: 0, + ino: 0, + uid: 0, + gid: 0, + size: self.size as u32, + } + } +} + +/// Metadata for one enumerated directory, keyed by direct filename. +type DirectoryEntries = hashbrown::HashMap; + +/// Either a live `lstat` result or a precomputed [`Stat`] from the lazy +/// cache. Lets [`crate::index_as_worktree`] treat both shapes uniformly without +/// branching at every per-entry use site. +pub(crate) enum Metadata { + Live(gix_index::fs::Metadata), + Cached(Stat), +} + +impl Metadata { + pub(crate) fn is_dir(&self) -> bool { + match self { + Self::Live(m) => m.is_dir(), + Self::Cached(c) => c.is_dir, + } + } + + pub(crate) fn is_symlink(&self) -> bool { + match self { + Self::Live(m) => m.is_symlink(), + Self::Cached(c) => c.is_symlink, + } + } + + pub(crate) fn len(&self) -> u64 { + match self { + Self::Live(m) => m.len(), + Self::Cached(c) => c.size, + } + } + + pub(crate) fn to_stat(&self) -> Result { + match self { + Self::Live(m) => gix_index::entry::Stat::from_fs(m), + Self::Cached(c) => Ok(c.to_stat()), + } + } + + pub(crate) fn mode_change( + &self, + entry_mode: gix_index::entry::Mode, + has_symlinks: bool, + executable_bit: bool, + ) -> Option { + match self { + Self::Live(m) => entry_mode.change_to_match_fs(m, has_symlinks, executable_bit), + // Windows batch enumeration doesn't expose the executable bit; pass `false`. + // Git on Windows defaults to `core.filemode=false` so this is unused anyway. + // TODO(correctness): it seems we'd have to try to support this at some point, + // depending on Git for Windows' implementation. + Self::Cached(c) => entry_mode.change_to_match_fs_with_values( + !c.is_dir && !c.is_symlink, // is_file: regular file (not dir, not symlink) + c.is_dir, + c.is_symlink, + false, + has_symlinks, + executable_bit, + ), + } + } +} + +/// Thread-local, lazily populated worktree metadata cache. +/// +/// Each lookup caches the entry's parent directory, keyed by the worktree +/// relative directory path. Directory entries are keyed by filename only. This +/// keeps the cache independent from pathspecs and excludes while avoiding a +/// live `lstat` for every tracked file in directories that have already been +/// enumerated. +pub struct FsCache { + /// Absolute worktree root used to turn repository-relative directory keys + /// into filesystem paths for Windows directory enumeration. + /// + /// The cache never changes this root and never stores absolute paths as + /// keys. Keeping the root separate lets lookups operate on Git-style + /// forward-slashed paths while `directory_entries()` performs the + /// platform-specific conversion at the boundary. + worktree: PathBuf, + /// Cache of direct directory listings, keyed by worktree-relative parent + /// directory path. + /// + /// Keys use the same byte representation as index paths: forward slashes, + /// no leading slash, and an empty key for the worktree root. Values are + /// `Some(entries)` when that directory was enumerated successfully and + /// `None` when enumeration failed, for example because the directory was + /// removed or inaccessible. Caching failed enumerations prevents repeated + /// directory-open attempts for multiple tracked entries in the same missing + /// directory; callers still fall through to live per-file metadata and + /// preserve correctness. + /// + /// `DirectoryEntries` itself is keyed only by direct filename, not by full + /// worktree-relative path. This keeps each cached listing small and mirrors + /// the Windows API result shape: first resolve the parent directory, then + /// look up the requested basename in that listing. + directories: hashbrown::HashMap>, +} + +impl FsCache { + /// Create an empty cache rooted at `worktree`. + pub fn new(worktree: &Path) -> Self { + FsCache { + worktree: worktree.to_owned(), + directories: Default::default(), + } + } + + /// Return cached metadata for `rela_path`, populating its parent directory + /// on first use. `None` means the cache couldn't help and callers should + /// fall back to a live `lstat`. + pub(crate) fn get(&mut self, rela_path: &BStr) -> Option { + let (dir, filename) = match rela_path.rfind_byte(b'/') { + Some(pos) => (&rela_path[..pos], &rela_path[pos + 1..]), + None => (BStr::new(b""), rela_path), + }; + if filename.is_empty() { + return None; + } + + if let Some(entries) = self.directories.get(dir) { + return entries.as_ref().and_then(|entries| entries.get(filename)).copied(); + } + + let entries = windows::directory_entries(&self.worktree, dir.into()); + let stat = entries.as_ref().and_then(|entries| entries.get(filename)).copied(); + self.directories.insert(dir.into(), entries); + stat + } +} + +mod windows { + use super::*; + use std::ffi::c_void; + use std::os::windows::ffi::OsStrExt; + + use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; + use windows_sys::Win32::Storage::FileSystem::{ + CreateFileW, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_ID_BOTH_DIR_INFO, FILE_LIST_DIRECTORY, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + FileIdBothDirectoryInfo, GetFileInformationByHandleEx, OPEN_EXISTING, SYNCHRONIZE, + }; + + /// Convert FILE_ID_BOTH_DIR_INFO to a [`Stat`]. + /// + /// File-type flags must mirror what the live fallback + /// (`gix_index::fs::Metadata`, i.e. `std::fs::symlink_metadata`) reports for + /// the same path, or cache hits and misses would disagree on the type and + /// misreport unchanged entries as type-changed or removed. std only calls a + /// reparse point a symlink if its tag is a *name surrogate* (symlinks and + /// junctions), and reports `is_dir=false` for those; non-surrogate reparse + /// points (OneDrive cloud placeholders, ProjFS, app-exec links, WOF) are + /// plain files/directories to std. Directory enumeration stores the reparse + /// tag in `EaSize` when `FILE_ATTRIBUTE_REPARSE_POINT` is set (MS-FSCC + /// 2.4.17, same convention as `WIN32_FIND_DATA::dwReserved0`), so we can + /// replicate that logic without extra syscalls. + fn stat_from_info(info: &FILE_ID_BOTH_DIR_INFO) -> Stat { + let size = info.EndOfFile as u64; + + // FILETIME values are LARGE_INTEGER holding 100ns intervals since 1601-01-01 UTC. + // `ctime` must come from `CreationTime`: `gix_index::entry::stat::from_fs` + // on Windows populates ctime from `Metadata::created()`, which is CreationTime. + let (mtime_secs, mtime_nsecs) = filetime_to_unix(info.LastWriteTime as u64); + let (ctime_secs, ctime_nsecs) = filetime_to_unix(info.CreationTime as u64); + + let (is_dir, is_symlink) = type_flags_from_attributes(info.FileAttributes, info.EaSize); + Stat { + is_dir, + is_symlink, + size, + mtime_secs, + mtime_nsecs, + ctime_secs, + ctime_nsecs, + } + } + + /// Derive `(is_dir, is_symlink)` exactly like `std::fs::FileType` does for + /// `symlink_metadata`, given raw directory-enumeration data. `reparse_tag` + /// is `EaSize` reinterpreted, only meaningful when the reparse attribute is set. + pub(super) fn type_flags_from_attributes(attributes: u32, reparse_tag: u32) -> (bool, bool) { + /// `IsReparseTagNameSurrogate`: the tag names another filesystem object + /// (symlink, junction/mount point), as opposed to tags that overlay data + /// onto the file itself (cloud placeholders, WOF, ProjFS, ...). + const NAME_SURROGATE_BIT: u32 = 0x2000_0000; + let is_reparse = attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0; + let is_symlink = is_reparse && (reparse_tag & NAME_SURROGATE_BIT) != 0; + let is_dir = !is_symlink && (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + (is_dir, is_symlink) + } + + /// Read a record's UTF-16 `FileName` without assuming the record is aligned. + /// + /// `FILE_ID_BOTH_DIR_INFO` records can be returned at unaligned offsets by some + /// filesystem drivers (see the call site and rust-lang/rust#104530), so forming a + /// `&[u16]` over the buffer would be UB. Borrow in place in the common aligned case; + /// copy element-by-element with unaligned reads otherwise. Mirrors std's + /// `from_maybe_unaligned` helper. + /// + /// # Safety + /// + /// `p` must point at `len` `u16`s of an in-bounds, initialized directory record. + #[allow(unsafe_code)] + unsafe fn name_from_maybe_unaligned<'a>(p: *const u16, len: usize) -> std::borrow::Cow<'a, [u16]> { + if p.is_aligned() { + std::borrow::Cow::Borrowed(unsafe { std::slice::from_raw_parts(p, len) }) + } else { + std::borrow::Cow::Owned((0..len).map(|i| unsafe { p.add(i).read_unaligned() }).collect()) + } + } + + /// Convert a Windows FILETIME (100ns intervals since 1601-01-01 UTC) to Unix (secs, nsecs). + fn filetime_to_unix(ft: u64) -> (u32, u32) { + const EPOCH_DIFF: u64 = 116_444_736_000_000_000; + let unix_100ns = ft.saturating_sub(EPOCH_DIFF); + let secs = (unix_100ns / 10_000_000) as u32; + let nsecs = ((unix_100ns % 10_000_000) * 100) as u32; + (secs, nsecs) + } + + /// Convert a filesystem path into a null-terminated UTF-16 buffer suitable for `CreateFileW`. + fn utf16_null_terminated(path: &Path) -> Vec { + let mut v: Vec = path.as_os_str().encode_wide().collect(); + v.push(0); + v + } + + /// Read the direct entries in `rel_dir`, keyed by filename. + /// + /// Returns `None` if the directory can't be read. Status callers treat that + /// as a cache miss and use the live syscall path for the requested entry. + pub(super) fn directory_entries(worktree: &Path, rel_dir: &BStr) -> Option { + let dir = if rel_dir.is_empty() { + worktree.to_owned() + } else { + worktree.join(gix_path::from_bstr(rel_dir)) + }; + let dir_path = utf16_null_terminated(&dir); + #[allow(unsafe_code)] + let handle = unsafe { + CreateFileW( + dir_path.as_ptr(), + FILE_LIST_DIRECTORY | SYNCHRONIZE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + std::ptr::null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + std::ptr::null_mut(), + ) + }; + + if handle == INVALID_HANDLE_VALUE { + return None; + } + + let mut entries = DirectoryEntries::default(); + // 64 KiB, u64-aligned: `FILE_ID_BOTH_DIR_INFO` contains LARGE_INTEGER + // fields that require 8-byte alignment, and `Vec` guarantees it. + const DIRECTORY_BUFFER_U64_WORDS: usize = 8 * 1024; + let mut buffer = vec![0u64; DIRECTORY_BUFFER_U64_WORDS]; + let buffer_bytes = (buffer.len() * 8) as u32; + + loop { + #[allow(unsafe_code)] + let success = unsafe { + GetFileInformationByHandleEx( + handle, + FileIdBothDirectoryInfo, + buffer.as_mut_ptr().cast::(), + buffer_bytes, + ) + }; + if success == 0 { + // End of enumeration (ERROR_NO_MORE_FILES) or access denied / similar. + // Either way, stop: the preprocess is best-effort and correctness falls back + // to per-file syscalls in `index_as_worktree`. + break; + } + + let mut offset = 0usize; + loop { + #[allow(clippy::cast_ptr_alignment)] + #[allow(unsafe_code)] + let info_ptr = unsafe { buffer.as_ptr().cast::().add(offset).cast::() }; + #[allow(unsafe_code)] + let info = unsafe { info_ptr.read_unaligned() }; + let info = &info; + + let name_len = (info.FileNameLength / 2) as usize; + #[allow(unsafe_code)] + let name_ptr = unsafe { (&raw const (*info_ptr).FileName).cast::() }; + #[allow(unsafe_code)] + let name = unsafe { name_from_maybe_unaligned(name_ptr, name_len) }; + let name_slice: &[u16] = &name; + + let is_dot = name_len == 1 && name_slice[0] == b'.' as u16; + let is_dotdot = name_len == 2 && name_slice[0] == b'.' as u16 && name_slice[1] == b'.' as u16; + if !is_dot && !is_dotdot { + let mut filename = String::with_capacity(name_len); + let mut valid = true; + for ch in char::decode_utf16(name_slice.iter().copied()) { + match ch { + Ok(ch) => filename.push(ch), + Err(_) => { + valid = false; + break; + } + } + } + if valid { + entries.insert(filename.into_bytes().into(), stat_from_info(info)); + } + } + + if info.NextEntryOffset == 0 { + break; + } + offset += info.NextEntryOffset as usize; + } + } + + #[allow(unsafe_code)] + unsafe { + CloseHandle(handle) + }; + Some(entries) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cache_stat_to_index_stat() { + let stat = Stat { + is_dir: false, + is_symlink: false, + size: 1234, + mtime_secs: 1700000000, + mtime_nsecs: 500_000_000, + ctime_secs: 1699999999, + ctime_nsecs: 100_000_000, + }; + let s = stat.to_stat(); + assert_eq!(s.size, 1234); + assert_eq!(s.mtime.secs, 1700000000); + assert_eq!(s.mtime.nsecs, 500_000_000); + assert_eq!(s.ctime.secs, 1699999999); + assert_eq!(s.ctime.nsecs, 100_000_000); + // dev/ino/uid/gid are always zero on Windows — `Stat::from_fs` zeros them too. + assert_eq!(s.dev, 0); + assert_eq!(s.ino, 0); + assert_eq!(s.uid, 0); + assert_eq!(s.gid, 0); + } + + #[test] + fn type_flags_match_std_symlink_metadata_semantics() { + use windows_sys::Win32::Storage::FileSystem::{FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT}; + + let flags = super::windows::type_flags_from_attributes; + const FILE_ATTRIBUTE_NORMAL: u32 = 0x80; + const IO_REPARSE_TAG_SYMLINK: u32 = 0xA000_000C; + const IO_REPARSE_TAG_MOUNT_POINT: u32 = 0xA000_0003; + const IO_REPARSE_TAG_CLOUD: u32 = 0x9000_001A; + const IO_REPARSE_TAG_WOF: u32 = 0x8000_0017; + + assert_eq!(flags(FILE_ATTRIBUTE_NORMAL, 0), (false, false), "plain file"); + assert_eq!(flags(FILE_ATTRIBUTE_DIRECTORY, 0), (true, false), "plain directory"); + assert_eq!( + flags(FILE_ATTRIBUTE_REPARSE_POINT, IO_REPARSE_TAG_SYMLINK), + (false, true), + "file symlinks are symlinks, never directories, like std" + ); + assert_eq!( + flags( + FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT, + IO_REPARSE_TAG_SYMLINK + ), + (false, true), + "directory symlinks are symlinks, never directories, like std" + ); + assert_eq!( + flags( + FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT, + IO_REPARSE_TAG_MOUNT_POINT + ), + (false, true), + "junctions are treated as symlinks by std" + ); + assert_eq!( + flags(FILE_ATTRIBUTE_REPARSE_POINT, IO_REPARSE_TAG_CLOUD), + (false, false), + "non-surrogate reparse-point files are regular files" + ); + assert_eq!( + flags( + FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT, + IO_REPARSE_TAG_CLOUD + ), + (true, false), + "non-surrogate reparse-point directories are regular directories" + ); + assert_eq!( + flags(FILE_ATTRIBUTE_REPARSE_POINT, IO_REPARSE_TAG_WOF), + (false, false), + "WOF reparse points are regular files" + ); + } + + #[test] + fn lazy_cache_populates_parent_directories_on_demand() { + let temp_dir = unique_temp_dir(); + let worktree = temp_dir.path(); + + std::fs::write(worktree.join("test.txt"), b"hello").expect("root file can be written"); + std::fs::create_dir(worktree.join("subdir")).expect("subdirectory can be created"); + std::fs::write(worktree.join("subdir").join("nested.txt"), b"world").expect("nested file can be written"); + + let mut cache = FsCache::new(worktree); + assert!( + cache.get(b"test.txt".as_slice().into()).is_some(), + "root files are found by lazily enumerating the worktree root" + ); + assert!( + cache.get(b"subdir/nested.txt".as_slice().into()).is_some(), + "nested files are found by lazily enumerating their parent directory" + ); + assert!( + cache.get(b"Test.txt".as_slice().into()).is_none(), + "mixed-case lookups miss rather than silently aliasing onto a different path" + ); + assert!( + cache.get(b"does-not-exist".as_slice().into()).is_none(), + "missing files are not a problem (and cause a fall-through to the live path)" + ); + } + + fn unique_temp_dir() -> gix_testtools::tempfile::TempDir { + gix_testtools::tempfile::TempDir::new().expect("temporary directory can be created") + } +} diff --git a/gix-status/src/index_as_worktree/function.rs b/gix-status/src/index_as_worktree/function.rs index ee8a15b9a7e..3527fa2f21d 100644 --- a/gix-status/src/index_as_worktree/function.rs +++ b/gix-status/src/index_as_worktree/function.rs @@ -11,6 +11,8 @@ use gix_features::parallel::{Reduce, in_parallel_if}; use gix_filter::pipeline::convert::ToGitOutcome; use gix_object::FindExt; +#[cfg(windows)] +use crate::fscache::{FsCache, Metadata as FsCacheMetadata}; use crate::index_as_worktree::types::ConflictIndexEntry; use crate::{ AtomicU64, SymlinkCheck, @@ -122,6 +124,8 @@ where path_backing, filter, options, + #[cfg(windows)] + fscache: options.fscache.then(|| FsCache::new(worktree)), skipped_by_pathspec, skipped_by_entry_flags, @@ -236,6 +240,10 @@ struct State<'a, 'b> { filter: gix_filter::Pipeline, path_backing: &'b gix_index::PathStorageRef, options: &'a Options, + /// Optional lazy worktree stats cache for faster status checks on Windows. + /// Lookups happen before falling back to per-file syscalls. + #[cfg(windows)] + fscache: Option, skipped_by_pathspec: &'a AtomicUsize, skipped_by_entry_flags: &'a AtomicUsize, @@ -382,53 +390,74 @@ impl<'index> State<'_, 'index> { } Err(err) => return Err(Error::Io(err.into())), }; - self.symlink_metadata_calls.fetch_add(1, Ordering::Relaxed); - let metadata = match gix_index::fs::Metadata::from_path_no_follow(worktree_path) { - Ok(metadata) if metadata.is_dir() => { - // index entries are normally only for files/symlinks - // if a file turned into a directory it was removed - // the only exception here are submodules which are - // part of the index despite being directories - if entry.mode.is_submodule() { - let status = submodule - .status(entry, rela_path) - .map_err(|err| Error::SubmoduleStatus { - rela_path: rela_path.into(), - source: Box::new(err), - })?; - return Ok(status.map(|status| Change::SubmoduleModification(status).into())); - } else { - return Ok(Some(Change::Removed.into())); - } - } - Ok(metadata) => metadata, - Err(err) if gix_fs::io_err::is_not_found(err.kind(), err.raw_os_error()) => { - return Ok(Some(Change::Removed.into())); + + // Acquire metadata. On Windows we consult the precomputed stats first and + // only fall back to a syscall on miss; on other platforms per-file + // `lstat` is already fast, so we just do the syscall directly. + #[cfg(windows)] + let metadata = if let Some(cached) = self.fscache.as_mut().and_then(|c| c.get(rela_path)) { + FsCacheMetadata::Cached(cached) + } else { + self.symlink_metadata_calls.fetch_add(1, Ordering::Relaxed); + match live_metadata(worktree_path)? { + Some(md) => FsCacheMetadata::Live(md), + None => return Ok(Some(Change::Removed.into())), } - Err(err) => { - return Err(Error::Io(err.into())); + }; + #[cfg(not(windows))] + let metadata = { + self.symlink_metadata_calls.fetch_add(1, Ordering::Relaxed); + match live_metadata(worktree_path)? { + Some(md) => md, + None => return Ok(Some(Change::Removed.into())), } }; + + // Handle directory: index entries are normally only for files/symlinks. + // If a file turned into a directory it was removed. + // The only exception here are submodules which are part of the index despite being directories. + if metadata.is_dir() { + if entry.mode.is_submodule() { + let status = submodule + .status(entry, rela_path) + .map_err(|err| Error::SubmoduleStatus { + rela_path: rela_path.into(), + source: Box::new(err), + })?; + return Ok(status.map(|status| Change::SubmoduleModification(status).into())); + } else { + return Ok(Some(Change::Removed.into())); + } + } + if entry.flags.contains(gix_index::entry::Flags::INTENT_TO_ADD) { return Ok(Some(EntryStatus::IntentToAdd)); } + + #[cfg(windows)] + let new_stat = metadata.to_stat()?; + #[cfg(not(windows))] let new_stat = gix_index::entry::Stat::from_fs(&metadata)?; - let executable_bit_changed = - match entry + + #[cfg(windows)] + let mode_change = metadata.mode_change(entry.mode, self.options.fs.symlink, self.options.fs.executable_bit); + #[cfg(not(windows))] + let mode_change = + entry .mode - .change_to_match_fs(&metadata, self.options.fs.symlink, self.options.fs.executable_bit) - { - Some(gix_index::entry::mode::Change::Type { new_mode }) => { - return Ok(Some( - Change::Type { - worktree_mode: new_mode, - } - .into(), - )); - } - Some(gix_index::entry::mode::Change::ExecutableBit) => true, - None => false, - }; + .change_to_match_fs(&metadata, self.options.fs.symlink, self.options.fs.executable_bit); + let executable_bit_changed = match mode_change { + Some(gix_index::entry::mode::Change::Type { new_mode }) => { + return Ok(Some( + Change::Type { + worktree_mode: new_mode, + } + .into(), + )); + } + Some(gix_index::entry::mode::Change::ExecutableBit) => true, + None => false, + }; // We implement racy-git. See racy-git.txt in the git documentation for detailed documentation. // @@ -690,3 +719,11 @@ impl Conflict { }) } } + +fn live_metadata(worktree_path: &Path) -> Result, Error> { + match gix_index::fs::Metadata::from_path_no_follow(worktree_path) { + Ok(md) => Ok(Some(md)), + Err(err) if gix_fs::io_err::is_not_found(err.kind(), err.raw_os_error()) => Ok(None), + Err(err) => Err(Error::Io(err.into())), + } +} diff --git a/gix-status/src/index_as_worktree/types.rs b/gix-status/src/index_as_worktree/types.rs index 26d26981412..1a3d833827b 100644 --- a/gix-status/src/index_as_worktree/types.rs +++ b/gix-status/src/index_as_worktree/types.rs @@ -33,6 +33,11 @@ pub struct Options { pub thread_limit: Option, /// Options that control how stat comparisons are made when checking if a file is fresh. pub stat: gix_index::entry::stat::Options, + /// Use the internal lazy worktree metadata cache. + /// + /// Misses fall through to a live `lstat`, so this only affects performance. + /// Effective only on Windows. + pub fscache: bool, } /// The context for [index_as_worktree()`](crate::index_as_worktree()). diff --git a/gix-status/src/index_as_worktree_with_renames/mod.rs b/gix-status/src/index_as_worktree_with_renames/mod.rs index 9eff6084a25..22fb1223dbd 100644 --- a/gix-status/src/index_as_worktree_with_renames/mod.rs +++ b/gix-status/src/index_as_worktree_with_renames/mod.rs @@ -62,6 +62,8 @@ pub(super) mod function { E: std::error::Error + Send + Sync + 'static, Find: gix_object::Find + gix_object::FindHeader + Send + Clone, { + let mut tracked_file_modifications = options.tracked_file_modifications; + tracked_file_modifications.fscache = options.fscache; gix_features::parallel::threads(|scope| -> Result { let (tx, rx) = std::sync::mpsc::channel(); let walk_outcome = options @@ -154,7 +156,7 @@ pub(super) mod function { filter, should_interrupt: ctx.should_interrupt, }, - options.tracked_file_modifications, + tracked_file_modifications, ) .map_err(Error::TrackedFileModifications) } diff --git a/gix-status/src/index_as_worktree_with_renames/types.rs b/gix-status/src/index_as_worktree_with_renames/types.rs index d0e528c1e4b..77ed478348b 100644 --- a/gix-status/src/index_as_worktree_with_renames/types.rs +++ b/gix-status/src/index_as_worktree_with_renames/types.rs @@ -295,6 +295,11 @@ pub struct Options<'a> { pub object_hash: gix_hash::Kind, /// Options to configure how modifications to tracked files should be obtained. pub tracked_file_modifications: crate::index_as_worktree::Options, + /// Use the internal lazy worktree metadata cache. + /// + /// See [`crate::index_as_worktree::Options::fscache`] for details. + /// Ineffective on non-Windows. + pub fscache: bool, /// Options to control the directory walk that informs about untracked files. /// /// Note that we forcefully disable emission of tracked files to avoid any overlap diff --git a/gix-status/src/lib.rs b/gix-status/src/lib.rs index b49a086182f..8598ba278ed 100644 --- a/gix-status/src/lib.rs +++ b/gix-status/src/lib.rs @@ -37,6 +37,9 @@ use portable_atomic::AtomicU64; pub mod index_as_worktree; pub use index_as_worktree::function::index_as_worktree; +#[cfg(windows)] +pub(crate) mod fscache; + #[cfg(feature = "worktree-rewrites")] pub mod index_as_worktree_with_renames; #[cfg(feature = "worktree-rewrites")] diff --git a/gix-status/tests/status/index_as_worktree.rs b/gix-status/tests/status/index_as_worktree.rs index 57e36674498..fc51a141906 100644 --- a/gix-status/tests/status/index_as_worktree.rs +++ b/gix-status/tests/status/index_as_worktree.rs @@ -41,6 +41,7 @@ fn fixture(name: &str, expected_status: &[Expectation<'_>]) -> Outcome { fixture_filtered(name, &[], expected_status) } +#[cfg(unix)] fn nonfile_fixture(name: &str, expected_status: &[Expectation<'_>]) -> Outcome { fixture_filtered_detailed( "status_nonfile", diff --git a/gix-status/tests/status/index_as_worktree_with_renames.rs b/gix-status/tests/status/index_as_worktree_with_renames.rs index 878c58c6590..c6a693123b1 100644 --- a/gix-status/tests/status/index_as_worktree_with_renames.rs +++ b/gix-status/tests/status/index_as_worktree_with_renames.rs @@ -1,6 +1,5 @@ use bstr::ByteSlice; use gix_diff::{blob::pipeline::WorktreeRoots, rewrites::CopySource}; -use gix_index::entry; use gix_status::{ index_as_worktree::{Change, EntryStatus, traits::FastEq}, index_as_worktree_with_renames, @@ -8,7 +7,7 @@ use gix_status::{ }; use pretty_assertions::assert_eq; -use crate::{fixture_path, fixture_path_rw_slow}; +use crate::fixture_path; #[test] fn changed_and_untracked_and_renamed() { @@ -134,7 +133,7 @@ fn tracked_changed_to_non_file() { &[Expectation::Modification { rela_path: "file", status: Change::Type { - worktree_mode: entry::Mode::FILE, + worktree_mode: gix_index::entry::Mode::FILE, } .into(), }], @@ -259,6 +258,7 @@ fn unreadable_untracked() { enum Fixture { ReadOnly, + #[cfg(unix)] WritableExecuted, } @@ -282,10 +282,11 @@ fn fixture_filtered_detailed( let (worktree, _tmp) = match fixture { Fixture::ReadOnly => { let dir = fixture_path(script).join(subdir); - (dir, None) + (dir, None::) } + #[cfg(unix)] Fixture::WritableExecuted => { - let tmp = fixture_path_rw_slow(script); + let tmp = crate::fixture_path_rw_slow(script); let dir = tmp.path().join(subdir); (dir, Some(tmp)) } @@ -347,6 +348,7 @@ fn fixture_filtered_detailed( stat: crate::index_as_worktree::TEST_OPTIONS, ..Default::default() }, + fscache: false, dirwalk, sorting: Some(Sorting::ByPathCaseSensitive), rewrites, diff --git a/gix/src/config/tree/sections/core.rs b/gix/src/config/tree/sections/core.rs index d6dd1e00767..1bf3dcf4ad5 100644 --- a/gix/src/config/tree/sections/core.rs +++ b/gix/src/config/tree/sections/core.rs @@ -26,6 +26,8 @@ impl Core { pub const EDITOR: keys::Program = keys::Program::new_program("editor", &config::Tree::CORE); /// The `core.fileMode` key. pub const FILE_MODE: keys::Boolean = keys::Boolean::new_boolean("fileMode", &config::Tree::CORE); + /// The `core.fsCache` key. + pub const FS_CACHE: keys::Boolean = keys::Boolean::new_boolean("fsCache", &config::Tree::CORE); /// The `core.ignoreCase` key. pub const IGNORE_CASE: keys::Boolean = keys::Boolean::new_boolean("ignoreCase", &config::Tree::CORE); /// The `core.filesRefLockTimeout` key. @@ -110,6 +112,7 @@ impl Section for Core { &Self::DISAMBIGUATE, &Self::EDITOR, &Self::FILE_MODE, + &Self::FS_CACHE, &Self::IGNORE_CASE, &Self::FILES_REF_LOCK_TIMEOUT, &Self::PACKED_REFS_TIMEOUT, diff --git a/gix/src/repository/attributes.rs b/gix/src/repository/attributes.rs index 7cea9383a04..e46680eb300 100644 --- a/gix/src/repository/attributes.rs +++ b/gix/src/repository/attributes.rs @@ -15,7 +15,8 @@ impl Repository { /// Configure a file-system cache for accessing git attributes *and* excludes on a per-path basis. /// /// Use `attribute_source` to specify where to read attributes from. Also note that exclude information will - /// always try to read `.gitignore` files from disk before trying to read it from the `index`. + /// always try to read `.gitignore` files from disk before trying to read it from the `index`, + /// configurable via `ignore_source`. /// /// Note that no worktree is required for this to work, even though access to in-tree `.gitattributes` and `.gitignore` files /// would require a non-empty `index` that represents a git tree. diff --git a/gix/src/status/index_worktree.rs b/gix/src/status/index_worktree.rs index 9b7dc060f29..a8056b4f990 100644 --- a/gix/src/status/index_worktree.rs +++ b/gix/src/status/index_worktree.rs @@ -1,12 +1,12 @@ use std::sync::atomic::AtomicBool; -use gix_status::index_as_worktree::traits::{CompareBlobs, SubmoduleStatus}; - use crate::{ Repository, bstr::{BStr, BString}, config, + config::cache::util::ApplyLeniencyDefault, }; +use gix_status::index_as_worktree::traits::{CompareBlobs, SubmoduleStatus}; /// The error returned by [Repository::index_worktree_status()]. #[derive(Debug, thiserror::Error)] @@ -121,6 +121,15 @@ impl Repository { let cwd = self.current_dir(); let git_dir_realpath = crate::path::realpath_opts(self.git_dir(), cwd, crate::path::realpath::MAX_SYMLINKS)?; let fs_caps = self.filesystem_options()?; + let fscache = self + .config + .resolved + .boolean(config::tree::Core::FS_CACHE) + .map(|res| config::tree::Core::FS_CACHE.enrich_error(res)) + .transpose() + .with_lenient_default(self.config.lenient_config)? + // if unset, default to enabled on Windows. Good for missing Git installations that would turn it on by installation config + .unwrap_or(cfg!(windows)); let accelerate_lookup = fs_caps.ignore_case.then(|| index.prepare_icase_backing()); let resource_cache = crate::diff::resource_cache( self, @@ -157,7 +166,9 @@ impl Repository { fs: fs_caps, thread_limit: options.thread_limit, stat: self.stat_options()?, + fscache, }, + fscache, dirwalk: options.dirwalk_options.map(Into::into), rewrites: options.rewrites, }, diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs index 2b6e5a21a57..4066b5aae93 100644 --- a/src/plumbing/main.rs +++ b/src/plumbing/main.rs @@ -390,6 +390,7 @@ pub fn main() -> Result<()> { ), Subcommands::Status(crate::plumbing::options::status::Platform { ignored, + untracked, format: status_format, statistics, submodules, @@ -428,6 +429,13 @@ pub fn main() -> Result<()> { core::repository::status::Ignored::Collapsed } }), + untracked: untracked.map(|mode| match mode.unwrap_or_default() { + crate::plumbing::options::status::Untracked::No => gix::status::UntrackedFiles::None, + crate::plumbing::options::status::Untracked::Normal => { + gix::status::UntrackedFiles::Collapsed + } + crate::plumbing::options::status::Untracked::All => gix::status::UntrackedFiles::Files, + }), output_format: format, statistics, thread_limit: thread_limit.or(cfg!(target_os = "macos").then_some(3)), // TODO: make this a configurable when in `gix`, this seems to be optimal on MacOS, linux scales though! MacOS also scales if reading a lot of files for refresh index diff --git a/src/plumbing/options/mod.rs b/src/plumbing/options/mod.rs index f21eff79194..1867fc85847 100644 --- a/src/plumbing/options/mod.rs +++ b/src/plumbing/options/mod.rs @@ -290,6 +290,17 @@ pub mod status { // allowing to ignore directories, naturally traversing the entire content. } + #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, clap::ValueEnum)] + pub enum Untracked { + /// Do not show untracked files. + No, + /// Collapse untracked directories when possible. + Normal, + /// Show individual untracked files. + #[default] + All, + } + #[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, clap::ValueEnum)] pub enum Format { /// A basic format that is easy to read, and useful for a first glimpse as flat list. @@ -308,6 +319,9 @@ pub mod status { /// If enabled, show ignored files and directories. #[clap(long)] pub ignored: Option>, + /// Define how to show untracked files. If set without a value, show all individual untracked files. + #[clap(long, short = 'u', require_equals = true)] + pub untracked: Option>, /// Define how to display the submodule status. Defaults to git configuration if unset. #[clap(long)] pub submodules: Option,