diff --git a/src/uu/cut/locales/fr-FR.ftl b/src/uu/cut/locales/fr-FR.ftl index b311a723415..e58588c35bd 100644 --- a/src/uu/cut/locales/fr-FR.ftl +++ b/src/uu/cut/locales/fr-FR.ftl @@ -97,7 +97,7 @@ cut-help-characters = alias pour le mode caractère cut-help-delimiter = spécifier le caractère délimiteur qui sépare les champs dans la source d'entrée. Par défaut Tab. cut-help-whitespace-delimited = Utiliser tout nombre d'espaces (Espace, Tab) pour séparer les champs dans la source d'entrée (extension FreeBSD). cut-help-fields = filtrer les colonnes de champs depuis la source d'entrée -cut-help-fields-merged = comme -f, mais fusionne les délimiteurs adjacents ; le délimiteur par défaut est l'espacement et le délimiteur de sortie une espace +cut-help-fields-merged = comme -f, mais fusionne les délimiteurs adjacents ; le délimiteur par défaut est l'espacement et le délimiteur de sortie un espace cut-help-complement = inverser le filtre - au lieu d'afficher seulement les colonnes filtrées, afficher toutes sauf ces colonnes cut-help-only-delimited = en mode champ, afficher seulement les lignes qui contiennent le délimiteur cut-help-zero-terminated = au lieu de filtrer les colonnes basées sur la ligne, filtrer les colonnes basées sur \\0 (caractère NULL) diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 415a7e2bca8..29a4b513ba0 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -18,7 +18,7 @@ use uucore::line_ending::LineEnding; use uucore::os_str_as_bytes; use self::searcher::Searcher; -use matcher::{ExactMatcher, Matcher, WhitespaceMatcher}; +use matcher::{ExactMatcher, Matcher, MbExactMatcher, WhitespaceMatcher}; use uucore::ranges::Range; use uucore::translate; use uucore::{format_usage, show_error, show_if_err}; @@ -322,6 +322,10 @@ impl CharCut<'_> { // already consumed. The ranges are sorted and disjoint, so one pass // over the line is enough and each range maps to a contiguous slice. let (mut idx, mut pos) = (0, 0); + // End of the previous range, and end of the last range before the run + // of adjacent ranges the current one belongs to (1-based, `0` for + // none), plus whether a delimiter is owed to the next character. + let (mut prev_high, mut before_run_high, mut delim_pending) = (0, 0, false); for &Range { low, high } in self.ranges { // A character position is never below its own byte offset, so a // range starting past the last byte selects nothing, and so do the @@ -334,21 +338,56 @@ impl CharCut<'_> { if idx == line.len() { break; } - if print_delim { - out.write_all(self.out_delim)?; - } else if self.explicit_delim { - print_delim = true; + // Adjacent ranges select a contiguous stretch of the line, so they + // form one run as far as characters are concerned. + if prev_high + 1 != low { + before_run_high = prev_high; + } + // A range boundary only separates characters when it falls between + // two of them; the delimiter is owed to whichever character prints + // next, possibly from a later range. Positions are byte offsets + // here, so all this only concerns `-b -n`: a character position + // can never land inside a character. + if self.by_char || idx >= prev_high { + delim_pending = true; } + // The selected bytes of a character must reach its end without a + // hole, so one an earlier run reached into is dropped even though + // its last byte falls here, taking the owed delimiter with it. + if !self.by_char && idx < before_run_high { + delim_pending = false; + let len = self.encoding.char_len(&line[idx..]); + idx += len; + pos += len; + if idx >= line.len() { + break; + } + } + prev_high = high; let start = idx; if high >= line.len() { // The range reaches past the end of the line, so it covers // every character left and none of them needs to be decoded. // The ranges after it start further away still. + if print_delim && delim_pending { + out.write_all(self.out_delim)?; + } return out.write_all(&line[start..]); } - // At least one character is taken: `pos` is below `high` and there - // are bytes left, so this always moves `idx` forward. (idx, pos) = self.advance(line, idx, pos, high); + // A range covering only part of a multi-byte character selects + // nothing, and leaves the delimiter it owes to the next one. + if idx == start { + continue; + } + if print_delim { + if delim_pending { + out.write_all(self.out_delim)?; + } + } else if self.explicit_delim { + print_delim = true; + } + delim_pending = false; out.write_all(&line[start..idx])?; } Ok(()) @@ -743,6 +782,38 @@ fn cut_fields_whitespace_trimmed( Ok(()) } +/// Run field cutting with the given matcher, choosing the explicit- or +/// implicit-output-delimiter routine. +fn cut_fields_with_matcher( + reader: R, + out: &mut W, + matcher: &M, + ranges: &[Range], + only_delimited: bool, + newline_char: u8, + out_delimiter: Option<&[u8]>, +) -> UResult<()> { + match out_delimiter { + Some(out_delim) => cut_fields_explicit_out_delim( + reader, + out, + matcher, + ranges, + only_delimited, + newline_char, + out_delim, + ), + None => cut_fields_implicit_out_delim( + reader, + out, + matcher, + ranges, + only_delimited, + newline_char, + ), + } +} + fn cut_fields( reader: R, out: &mut W, @@ -763,28 +834,27 @@ fn cut_fields( field_opts.only_delimited, ) } - Delimiter::Slice(delim) => { - let matcher = ExactMatcher::new(delim); - match opts.out_delimiter { - Some(out_delim) => cut_fields_explicit_out_delim( - reader, - out, - &matcher, - ranges, - field_opts.only_delimited, - newline_char, - out_delim, - ), - None => cut_fields_implicit_out_delim( - reader, - out, - &matcher, - ranges, - field_opts.only_delimited, - newline_char, - ), - } - } + // An ASCII single-byte delimiter can never occur inside a multi-byte + // character, so the fast byte-wise matcher is correct in every locale. + // Otherwise match the delimiter as a whole character. + Delimiter::Slice(delim) if delim.len() == 1 && delim[0] <= 0x7F => cut_fields_with_matcher( + reader, + out, + &ExactMatcher::new(delim), + ranges, + field_opts.only_delimited, + newline_char, + opts.out_delimiter, + ), + Delimiter::Slice(delim) => cut_fields_with_matcher( + reader, + out, + &MbExactMatcher::new(delim), + ranges, + field_opts.only_delimited, + newline_char, + opts.out_delimiter, + ), Delimiter::Whitespace => { let out_delim = opts.out_delimiter.unwrap_or(b"\t"); if field_opts.whitespace_trimmed { @@ -891,14 +961,14 @@ fn get_delimiters(matches: &ArgMatches) -> UResult<(Delimiter<'_>, Option<&[u8]> if os_string.is_empty() { Delimiter::Slice(b"\0") } else { - // The delimiter must be a single character. We accept a single - // UTF-8 character (e.g. an emoji), a single byte (including a - // non-UTF-8 byte like `b"\xFF"`), or a single character of the - // current locale's encoding (e.g. a 2-byte GB18030 character). + // The delimiter must be a single character in the *current locale's* + // encoding: a multi-byte UTF-8 character in a UTF-8 locale, a 2-byte + // GB18030 character, or any single byte like `b"\xAD"`. In a C/POSIX + // (single-byte) locale even a valid UTF-8 multibyte sequence counts + // as several characters, matching GNU. let bytes = os_str_as_bytes(os_string)?; - let single_utf8_char = os_string.to_str().is_some_and(|s| s.chars().count() == 1); - let single_locale_char = mb_char_len(bytes) == bytes.len(); - if !single_utf8_char && !single_locale_char { + let is_single_char = mb_char_len(bytes) == bytes.len(); + if !is_single_char { return Err(UUsageError::new( 1, translate!("cut-error-delimiter-must-be-single-character"), @@ -1039,8 +1109,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { Ok(()) } -// Only one, and only one of cutting mode arguments, i.e. `-b`, `-c`, `-f`, -// `-F`, is expected. +// Exactly one of the cutting mode arguments `-b`, `-c`, `-f` or `-F` must be +// given. // // Returns `options::BYTES`, `options::CHARACTERS`, `options::FIELDS`, or // `options::FIELDS_MERGED`. diff --git a/src/uu/cut/src/matcher.rs b/src/uu/cut/src/matcher.rs index c1be9fb5ee7..036daa6c027 100644 --- a/src/uu/cut/src/matcher.rs +++ b/src/uu/cut/src/matcher.rs @@ -3,7 +3,10 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +// spell-checker:ignore clen + use memchr::{memchr, memchr2}; +use uucore::i18n::charmap::{Encoding, locale_encoding}; // Find the next matching byte sequence positions // Return (first, last) where haystack[first..last] corresponds to the matched pattern @@ -39,22 +42,71 @@ impl Matcher for ExactMatcher<'_> { } } -// Matches for any number of SPACE or TAB +// Matches the delimiter as a whole character, never inside a multi-byte +// character. Used for delimiters that could be a continuation byte (e.g. a +// lone `0xa9`) or a multi-byte character; ASCII delimiters use `ExactMatcher`. +pub struct MbExactMatcher<'a> { + needle: &'a [u8], +} + +impl<'a> MbExactMatcher<'a> { + pub fn new(needle: &'a [u8]) -> Self { + assert!(!needle.is_empty()); + Self { needle } + } +} + +impl Matcher for MbExactMatcher<'_> { + fn next_match(&self, haystack: &[u8]) -> Option<(usize, usize)> { + // Resolved once per line: character boundaries have to be walked from + // the start, so there is no way to skip ahead with `memchr` here. + let encoding = locale_encoding(); + let mut pos = 0; + while pos < haystack.len() { + let clen = encoding.char_len(&haystack[pos..]); + if clen == self.needle.len() && &haystack[pos..pos + clen] == self.needle { + return Some((pos, pos + clen)); + } + pos += clen; + } + None + } +} + +// Matches any number of whitespace characters. ASCII space and tab are always +// recognized; in a UTF-8 locale Unicode space separators are too. pub struct WhitespaceMatcher {} impl Matcher for WhitespaceMatcher { fn next_match(&self, haystack: &[u8]) -> Option<(usize, usize)> { - let match_idx = memchr2(b' ', b'\t', haystack)?; - let mut skip = match_idx + 1; - - while skip < haystack.len() { - match haystack[skip] { - b' ' | b'\t' => skip += 1, - _ => break, + // In a single-byte locale SPACE and TAB are the only blanks, so the run + // can be found with a SIMD scan instead of decoding every character. + let encoding = locale_encoding(); + if encoding == Encoding::SingleByte { + let start = memchr2(b' ', b'\t', haystack)?; + let mut end = start + 1; + while end < haystack.len() && matches!(haystack[end], b' ' | b'\t') { + end += 1; } + return Some((start, end)); } - Some((match_idx, skip)) + let mut pos = 0; + while pos < haystack.len() { + if let Some(blank_len) = encoding.blank_len(&haystack[pos..]) { + let start = pos; + pos += blank_len; + while pos < haystack.len() { + match encoding.blank_len(&haystack[pos..]) { + Some(len) => pos += len, + None => break, + } + } + return Some((start, pos)); + } + pos += encoding.char_len(&haystack[pos..]); + } + None } } diff --git a/src/uu/paste/src/paste.rs b/src/uu/paste/src/paste.rs index ff467ee622e..0d3913097c1 100644 --- a/src/uu/paste/src/paste.rs +++ b/src/uu/paste/src/paste.rs @@ -249,7 +249,7 @@ fn parse_delimiters(delimiters: &OsString) -> UResult]>> { _ => { // Unknown escape: strip backslash, use the following character(s) let remaining = &bytes[i..]; - let len = mb_char_len(remaining).min(remaining.len()); + let len = mb_char_len(remaining); vec.push(Box::from(&bytes[i..i + len])); i += len; continue; @@ -258,7 +258,7 @@ fn parse_delimiters(delimiters: &OsString) -> UResult]>> { i += 1; } else { let remaining = &bytes[i..]; - let len = mb_char_len(remaining).min(remaining.len()); + let len = mb_char_len(remaining); vec.push(Box::from(&bytes[i..i + len])); i += len; } diff --git a/src/uucore/src/lib/features/i18n/charmap.rs b/src/uucore/src/lib/features/i18n/charmap.rs index 2c8bc0d1006..8c394cc677f 100644 --- a/src/uucore/src/lib/features/i18n/charmap.rs +++ b/src/uucore/src/lib/features/i18n/charmap.rs @@ -3,7 +3,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore langinfo charmap eucjp euckr euctw CTYPE HKSCS hkscs localedata +// spell-checker:ignore langinfo charmap eucjp euckr euctw CTYPE HKSCS hkscs localedata iswblank feff //! Locale-aware multi-byte character length detection via `LC_CTYPE`. @@ -13,6 +13,7 @@ use std::sync::OnceLock; /// concerned. `SingleByte` covers `C`/`POSIX` and every 8-bit encoding. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Encoding { + /// C/POSIX and every 8-bit encoding: each byte is its own character. SingleByte, Utf8, Gb18030, @@ -36,6 +37,7 @@ fn encoding_from_name(enc: &str) -> Encoding { /// /// Callers that decode more than one character should hold on to the returned /// value: it turns the per-character encoding lookup into a register read. +#[inline] pub fn locale_encoding() -> Encoding { static ENCODING: OnceLock = OnceLock::new(); *ENCODING.get_or_init(|| { @@ -44,6 +46,8 @@ pub fn locale_encoding() -> Encoding { .find_map(|&k| std::env::var(k).ok().filter(|v| !v.is_empty())); let s = match val.as_deref() { Some(s) if s != "C" && s != "POSIX" => s, + // Explicit C/POSIX locale, or no locale set at all: the POSIX + // default is `C`, which is byte-oriented. _ => return Encoding::SingleByte, }; if let Some(enc) = s.split('.').nth(1) { @@ -70,16 +74,42 @@ impl Encoding { if b0 <= 0x7F { return 1; } - match self { - // `C`/`POSIX` and unknown encodings have `MB_CUR_MAX == 1`, but we - // still decode UTF-8 there as a sensible default for byte-length - // detection. - Self::SingleByte | Self::Utf8 => utf8_len(bytes, b0), + let len = match self { + // `C`/`POSIX` and 8-bit encodings have `MB_CUR_MAX == 1`, so a byte + // is never part of a longer character, even when it would form a + // valid UTF-8 sequence. + Self::SingleByte => 1, + Self::Utf8 => utf8_len(bytes, b0), Self::Gb18030 => gb18030_len(bytes, b0), Self::EucJp => eucjp_len(bytes, b0), Self::EucKr => euckr_len(bytes, b0), Self::Big5 => big5_len(bytes, b0), + }; + debug_assert!((1..=bytes.len()).contains(&len)); + len + } + + /// If the first character in `bytes` is horizontal whitespace ("blank"), + /// return its byte length; otherwise `None`. + /// + /// ASCII space and tab always count. Under UTF-8 the Unicode space + /// separators are also recognized, except no-break ones (e.g. U+00A0), + /// matching glibc's `iswblank`. + #[inline] + pub fn blank_len(self, bytes: &[u8]) -> Option { + let len = self.char_len(bytes); + if len == 1 { + return (bytes[0] == b' ' || bytes[0] == b'\t').then_some(1); + } + // `char_len` only looks at the shape of the sequence, so decode it for + // real here: an overlong or surrogate encoding is not a character at + // all, and must not pass for one of the blanks it decodes to. + if self == Self::Utf8 + && std::str::from_utf8(&bytes[..len]).is_ok_and(|s| s.starts_with(is_unicode_blank)) + { + return Some(len); } + None } } @@ -88,6 +118,13 @@ pub fn mb_char_len(bytes: &[u8]) -> usize { locale_encoding().char_len(bytes) } +/// Horizontal whitespace characters (glibc `iswblank`): excludes the +/// no-break variants U+00A0, U+2007 and U+202F. +fn is_unicode_blank(c: char) -> bool { + matches!(c, + '\u{09}' | '\u{20}' | '\u{1680}' | '\u{2000}'..='\u{2006}' | '\u{2008}'..='\u{200A}' | '\u{205F}' | '\u{3000}') +} + // All helpers below assume b0 > 0x7F (ASCII already handled by caller). fn utf8_len(b: &[u8], b0: u8) -> usize { @@ -171,3 +208,58 @@ fn big5_len(b: &[u8], b0: u8) -> usize { 1 } } + +#[cfg(test)] +mod tests { + use super::{Encoding, is_unicode_blank}; + + #[test] + fn blank_len_ascii_is_encoding_independent() { + for encoding in [Encoding::SingleByte, Encoding::Utf8, Encoding::Gb18030] { + assert_eq!(encoding.blank_len(b" x"), Some(1)); + assert_eq!(encoding.blank_len(b"\tx"), Some(1)); + assert_eq!(encoding.blank_len(b"Qx"), None); + // Vertical whitespace is not a blank. + assert_eq!(encoding.blank_len(b"\nx"), None); + assert_eq!(encoding.blank_len(b"\x0bx"), None); + } + } + + #[test] + fn blank_len_unicode_blanks_only_under_utf8() { + // U+2002 EN SPACE and U+3000 IDEOGRAPHIC SPACE. + for blank in ["\u{2002}", "\u{3000}"] { + assert_eq!(Encoding::Utf8.blank_len(blank.as_bytes()), Some(3)); + // A byte-oriented locale sees only the individual lead byte. + assert_eq!(Encoding::SingleByte.blank_len(blank.as_bytes()), None); + } + } + + #[test] + fn blank_len_rejects_no_break_blanks() { + // U+00A0, U+2007 and U+2009 differ from glibc's iswblank only in that + // the first two are non-breaking; U+2009 is a real blank. + assert_eq!(Encoding::Utf8.blank_len("\u{a0}".as_bytes()), None); + assert_eq!(Encoding::Utf8.blank_len("\u{2007}".as_bytes()), None); + assert_eq!(Encoding::Utf8.blank_len("\u{2009}".as_bytes()), Some(3)); + } + + #[test] + fn blank_len_rejects_ill_formed_sequences() { + // Overlong encoding of U+0020: shaped like a 2-byte sequence, but it is + // not a character and must not pass for a space. + assert_eq!(Encoding::Utf8.blank_len(&[0xc0, 0xa0]), None); + // Truncated lead byte falls back to a single byte, which is not blank. + assert_eq!(Encoding::Utf8.blank_len(&[0xe2, 0x80]), None); + // Surrogate half. + assert_eq!(Encoding::Utf8.blank_len(&[0xed, 0xa0, 0x80]), None); + } + + #[test] + fn unicode_blank_set_matches_iswblank() { + assert!(is_unicode_blank('\u{1680}')); + assert!(is_unicode_blank('\u{205f}')); + assert!(!is_unicode_blank('\u{202f}')); + assert!(!is_unicode_blank('\u{feff}')); + } +} diff --git a/tests/by-util/test_cut.rs b/tests/by-util/test_cut.rs index 2c0882b6067..07e9f61be9c 100644 --- a/tests/by-util/test_cut.rs +++ b/tests/by-util/test_cut.rs @@ -746,6 +746,84 @@ fn test_whitespace_delimited_long_and_trimmed() { .fails_with_code(1); } +#[test] +fn test_whitespace_delimited_trimmed_zero_terminated() { + // The record terminator must not count as a non-blank when trimming, or the + // trailing blanks of a record would survive and add a phantom field. NUL is + // the interesting case: unlike `\n` it is not whitespace to begin with. + new_ucmd!() + .args(&["-z", "--whitespace-delimited=trimmed", "-f2"]) + .pipe_in(&b" red blue \0 green pink \0"[..]) + .succeeds() + .stdout_only_bytes(&b"blue\0pink\0"[..]); + // Asking past the last field yields an empty record, not the stray blanks. + new_ucmd!() + .args(&["-z", "--whitespace-delimited=trimmed", "-f3"]) + .pipe_in(&b" red blue \0"[..]) + .succeeds() + .stdout_only_bytes(&b"\0"[..]); + // A blank-only record has no delimiter left after trimming, so -s drops it. + new_ucmd!() + .args(&["-z", "-s", "--whitespace-delimited=trimmed", "-f1"]) + .pipe_in(&b"\t \0 amber violet \0"[..]) + .succeeds() + .stdout_only_bytes(&b"amber\0"[..]); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] +#[cfg(target_os = "linux")] +fn test_byte_no_split_partially_selected_char() { + // -b -n: the selected bytes of a character must reach its end without a + // hole. "🗿" (f0 9f 97 bf) spans bytes 1-4, "w" is byte 5. + let stone = &b"\xf0\x9f\x97\xbfw\n"[..]; + // Byte 3 is left out, so the character is split and dropped. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b1-2,4-5", "-n"]) + .pipe_in(stone) + .succeeds() + .stdout_only_bytes(b"w\n"); + // The same holds when the hole comes from a single-byte range. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b2,4", "-n"]) + .pipe_in(stone) + .succeeds() + .stdout_only_bytes(b"\n"); + // Selecting only the tail of the character still prints it whole. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b3-", "-n"]) + .pipe_in(stone) + .succeeds() + .stdout_only_bytes(stone); + // Adjacent ranges cover it without a hole, and the boundary inside the + // character emits no output delimiter. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b1-3,4-5", "-n", "--output-d=|"]) + .pipe_in(stone) + .succeeds() + .stdout_only_bytes(stone); + // "q€é r": q is byte 1, € (e2 82 ac) bytes 2-4, é (c3 a9) bytes 5-6. + let mixed = &b"q\xe2\x82\xac\xc3\xa9r\n"[..]; + // The delimiter a range owes is carried to whatever prints next, and only + // a boundary between two printed characters produces one. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b1,2,5-7", "-n", "--output-d=|"]) + .pipe_in(mixed) + .succeeds() + .stdout_only_bytes(&b"q|\xc3\xa9r\n"[..]); + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b1,2-4,5-6", "-n", "--output-d=|"]) + .pipe_in(mixed) + .succeeds() + .stdout_only_bytes(&b"q|\xe2\x82\xac|\xc3\xa9\n"[..]); +} + #[test] fn test_unset_locale_is_byte_oriented() { // With no locale set the POSIX default is C, so characters are bytes. @@ -769,13 +847,90 @@ fn test_newline_delim_suppress_missing_field() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] +#[cfg(target_os = "linux")] +fn test_byte_no_split_with_output_delimiter() { + // -b -n with an output delimiter: a range covering only part of a + // multibyte character contributes nothing and emits no delimiter. + // "ü" (c3 bc) spans bytes 1-2; byte 1 alone selects no whole character. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-b1,3", "-n", "--output-d=|"]) + .pipe_in(&b"\xc3\xbcZ\n"[..]) + .succeeds() + .stdout_only_bytes(b"Z\n"); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] +#[cfg(target_os = "linux")] +fn test_field_delimiter_not_split_inside_multibyte_char() { + use std::os::unix::ffi::OsStrExt; + // In a UTF-8 locale, a delimiter byte that is part of a multibyte character + // must not split it. Here U+20AC (€ = e2 82 ac) contains 0xac, and 0xac is + // also used as a standalone delimiter byte. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .arg("-d") + .arg(std::ffi::OsStr::from_bytes(b"\xac")) + .arg("-f2") + .pipe_in(&b"1\xe2\x82\xac2\xac3\n"[..]) + .succeeds() + .stdout_only_bytes(b"3\n"); + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .arg("-d") + .arg(std::ffi::OsStr::from_bytes(b"\xac")) + .arg("-f1") + .pipe_in(&b"1\xe2\x82\xac2\xac3\n"[..]) + .succeeds() + .stdout_only_bytes(b"1\xe2\x82\xac2\n"); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] +#[cfg(target_os = "linux")] +fn test_whitespace_delimiter_unicode_blank() { + // U+2002 (EN SPACE) is a Unicode blank and splits fields under -w. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-w", "-f2"]) + .pipe_in(&b"x\xe2\x80\x82y\n"[..]) + .succeeds() + .stdout_only_bytes(b"y\n"); + // U+2007 (FIGURE SPACE) is not a blank: the line stays a single field and + // is suppressed by -s. + new_ucmd!() + .env("LC_ALL", "C.UTF-8") + .args(&["-s", "-w", "-f2"]) + .pipe_in(&b"x\xe2\x80\x87y\n"[..]) + .succeeds() + .stdout_only_bytes(b""); +} + +#[test] +fn test_delimiter_multibyte_rejected_in_c_locale() { + // In the C locale a valid UTF-8 multibyte sequence is several characters. + // The delimiter is passed as ordinary text so this also runs on Windows. + new_ucmd!() + .env("LC_ALL", "C") + .args(&["-d", "\u{20ac}", "-f1"]) + .fails_with_code(1) + .stderr_contains("cut: the delimiter must be a single character"); +} + +#[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] fn test_emoji_delim() { + // A multibyte delimiter is only a single character in a UTF-8 locale. new_ucmd!() + .env("LC_ALL", "C.UTF-8") .args(&["-d🗿", "-f1"]) .pipe_in("💐🗿🌹\n") .succeeds() .stdout_only("💐\n"); new_ucmd!() + .env("LC_ALL", "C.UTF-8") .args(&["-d🗿", "-f2"]) .pipe_in("💐🗿🌹\n") .succeeds() diff --git a/tests/by-util/test_paste.rs b/tests/by-util/test_paste.rs index 260aa21d605..77c79fe3929 100644 --- a/tests/by-util/test_paste.rs +++ b/tests/by-util/test_paste.rs @@ -393,9 +393,12 @@ fn test_gnu_escape_sequences() { // As of 2024-10-09, only bsdutils (https://github.com/dcantrell/bsdutils, derived from FreeBSD) and toybox handle // multibyte delimiter characters in the way a user would likely expect. BusyBox and GNU Core Utilities do not. #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] fn test_multi_byte_delimiter() { for option_style in ["-d", "--delimiters"] { new_ucmd!() + // A multibyte delimiter is stepped per character in a UTF-8 locale. + .env("LC_ALL", "C.UTF-8") .args(&[option_style, "!ß@", "-s"]) .pipe_in( "\ @@ -417,6 +420,7 @@ fn test_multi_byte_delimiter() { } #[test] +#[cfg_attr(wasi_runner, ignore = "WASI: the guest does not inherit LC_ALL")] fn test_data() { for example in EXAMPLE_DATA { let (at, mut ucmd) = at_and_ucmd!(); @@ -427,7 +431,9 @@ fn test_data() { ins.push(file); } println!("{}", example.name); - ucmd.args(example.args) + // Some examples use multibyte delimiters, which need a UTF-8 locale. + ucmd.env("LC_ALL", "C.UTF-8") + .args(example.args) .args(&ins) .succeeds() .stdout_is(example.out);