diff --git a/.changeset/doc-scaffold-fp-guards.md b/.changeset/doc-scaffold-fp-guards.md new file mode 100644 index 00000000..1b345c63 --- /dev/null +++ b/.changeset/doc-scaffold-fp-guards.md @@ -0,0 +1,5 @@ +--- +"@stll/anonymize": patch +--- + +Reject section-marker addresses and numbered page footers misclassified as organizations. diff --git a/crates/anonymize-adapter-contract/src/assemble/deny_list.rs b/crates/anonymize-adapter-contract/src/assemble/deny_list.rs index ffb89230..a7c3a47a 100644 --- a/crates/anonymize-adapter-contract/src/assemble/deny_list.rs +++ b/crates/anonymize-adapter-contract/src/assemble/deny_list.rs @@ -88,6 +88,39 @@ fn scoped_language_word_file( Ok(lower_sorted_unique(words)) } +fn scoped_page_footer_markers( + selected: Option<&[String]>, +) -> Result, AssembleError> { + let data: OrderedMap = + parse_ordered_data_file("page-footer-markers.json")?; + let mut markers = Vec::new(); + for (language, value) in &data { + if language.starts_with('_') + || !super::language::language_config_matches(language, selected) + { + continue; + } + let Some(group) = value.as_object() else { + continue; + }; + let mut page_words = Vec::new(); + append_word_array(&mut page_words, group.get("pageWords")); + let mut count_words = Vec::new(); + append_word_array(&mut count_words, group.get("countWords")); + for page_word in &page_words { + markers.push(page_word.to_lowercase()); + for count_word in &count_words { + markers.push(format!( + "{} {}", + page_word.to_lowercase(), + count_word.to_lowercase() + )); + } + } + } + Ok(lower_sorted_unique(markers.iter().map(String::as_str))) +} + /// `languageWordValues(record)`: `collectLanguageWordValues` of a sub-record, /// or empty when the value is not a record. fn language_word_values(value: Option<&Value>) -> Vec { @@ -210,6 +243,7 @@ pub(super) fn build_deny_list_filter_data( let generic_roles_file: GenericRoles = parse_data_file("generic-roles.json")?; let mut generic_roles = set_ordered(generic_roles_file.roles); generic_roles.extend(legal_forms::role_heads(content_languages)?); + let page_footer_markers = scoped_page_footer_markers(content_languages)?; // trailingAddressWordExclusions: lowerSortedUnique union, then Set (no-op). let mut trailing_union: Vec = Vec::new(); @@ -242,6 +276,7 @@ pub(super) fn build_deny_list_filter_data( ambiguous_street_type_terms: shape("ambiguousStreetTypeTerms"), first_names: corpus.first_names_list.clone(), generic_roles, + page_footer_markers, number_abbrev_prefixes: shape("numberAbbrevPrefixes"), sentence_starters: deny_list_filter_static("sentenceStarters")?, trailing_address_word_exclusions, @@ -1349,6 +1384,65 @@ mod tests { ); } + #[test] + fn page_footer_markers_follow_content_language_scope() { + let coverage = [ + ("cs", "strana", "stran celkem"), + ("de", "seite", "seiten insgesamt"), + ("en", "page", "pages of"), + ("pt-br", "página", "páginas de"), + ("es", "página", "páginas de"), + ("fr", "page", "pages sur"), + ("hu", "oldal", "oldalak összesen"), + ("it", "pagina", "pagine di"), + ("lv", "lapa", "lappuses no"), + ("pl", "strona", "stron łącznie"), + ("ro", "pagina", "pagini din"), + ("sk", "strana", "strán celkom"), + ("sv", "sida", "sidor av"), + ]; + let scope_data: Value = parse_data_file("language-scopes.json") + .expect("language scopes should parse"); + let scope_languages: HashSet = scope_data + .get("languages") + .and_then(Value::as_object) + .expect("language scopes should contain a language map") + .keys() + .cloned() + .collect(); + let covered_languages: HashSet = coverage + .iter() + .map(|(language, _, _)| String::from(*language)) + .collect(); + assert_eq!( + covered_languages, scope_languages, + "every supported content language needs reviewed page-footer vocabulary" + ); + + for (language, standalone, paired) in coverage { + let selected = [String::from(language)]; + let filters = + build_deny_list_filter_data(&test_corpus(), Some(&selected)) + .expect("language-scoped deny-list filters should assemble"); + for expected in [standalone, paired] { + assert!( + filters + .page_footer_markers + .contains(&String::from(expected)), + "{language} should include {expected}" + ); + } + if language != "en" { + assert!( + !filters + .page_footer_markers + .contains(&String::from("page of")), + "{language} should not import English page markers" + ); + } + } + } + /// The assemble-time exemption is only effective while the exempted word /// stays *off* the match-time allow list: `curated_labels_for_match` /// (processors.rs) unconditionally rejects any keyword present in diff --git a/crates/anonymize-adapter-contract/src/config.rs b/crates/anonymize-adapter-contract/src/config.rs index 4112f28b..9ff31779 100644 --- a/crates/anonymize-adapter-contract/src/config.rs +++ b/crates/anonymize-adapter-contract/src/config.rs @@ -472,6 +472,7 @@ fn deny_list_filters_from_binding( ambiguous_street_type_terms: lower_set(filters.ambiguous_street_type_terms), first_names: lower_set(filters.first_names), generic_roles: lower_set(filters.generic_roles), + page_footer_markers: lower_set(filters.page_footer_markers), number_abbrev_prefixes: lower_set(filters.number_abbrev_prefixes), sentence_starters: lower_set(filters.sentence_starters), trailing_address_word_exclusions: lower_set( diff --git a/crates/anonymize-adapter-contract/src/package/format.rs b/crates/anonymize-adapter-contract/src/package/format.rs index a4640c52..9cdb75d5 100644 --- a/crates/anonymize-adapter-contract/src/package/format.rs +++ b/crates/anonymize-adapter-contract/src/package/format.rs @@ -9,8 +9,8 @@ use crate::error::{Result, invalid_prepared_search_package}; use super::PackageCompression; -pub(crate) const BINDING_PACKAGE_SCHEMA_VERSION: u32 = 5; -pub(crate) const CORE_PACKAGE_SCHEMA_VERSION: u32 = 5; +pub(crate) const BINDING_PACKAGE_SCHEMA_VERSION: u32 = 6; +pub(crate) const CORE_PACKAGE_SCHEMA_VERSION: u32 = 6; pub(crate) const PREPARED_SEARCH_PACKAGE_HEADER: [u8; 8] = *b"ANONPKG2"; pub(crate) const PREPARED_SEARCH_COMPRESSED_PACKAGE_HEADER: [u8; 8] = diff --git a/crates/anonymize-adapter-contract/src/package/mod.rs b/crates/anonymize-adapter-contract/src/package/mod.rs index c06c59ad..3bffe8fe 100644 --- a/crates/anonymize-adapter-contract/src/package/mod.rs +++ b/crates/anonymize-adapter-contract/src/package/mod.rs @@ -735,13 +735,13 @@ mod tests { #[test] fn prepared_package_schema_versions_track_the_current_payload_shape() { - assert_eq!(BINDING_PACKAGE_SCHEMA_VERSION, 5); - assert_eq!(CORE_PACKAGE_SCHEMA_VERSION, 5); + assert_eq!(BINDING_PACKAGE_SCHEMA_VERSION, 6); + assert_eq!(CORE_PACKAGE_SCHEMA_VERSION, 6); } #[test] fn prepared_package_readers_reject_previous_schema_payloads() { - const PREVIOUS_SCHEMA_VERSION: u32 = 4; + const PREVIOUS_SCHEMA_VERSION: u32 = 5; let binding_payload = prepared_search_package_payload_to_bytes( &package_test_config(), b"artifacts", diff --git a/crates/anonymize-adapter-contract/src/types.rs b/crates/anonymize-adapter-contract/src/types.rs index 68000f6a..12ddb19a 100644 --- a/crates/anonymize-adapter-contract/src/types.rs +++ b/crates/anonymize-adapter-contract/src/types.rs @@ -495,6 +495,8 @@ pub struct BindingDenyListFilterData { pub first_names: Vec, pub generic_roles: Vec, #[serde(default)] + pub page_footer_markers: Vec, + #[serde(default)] pub number_abbrev_prefixes: Vec, pub sentence_starters: Vec, pub trailing_address_word_exclusions: Vec, diff --git a/crates/anonymize-binding-core/src/lib.rs b/crates/anonymize-binding-core/src/lib.rs index bf419ea8..0cff334a 100644 --- a/crates/anonymize-binding-core/src/lib.rs +++ b/crates/anonymize-binding-core/src/lib.rs @@ -100,28 +100,36 @@ impl PreparedBinding { verification: PackageVerification, ) -> Result { if prepared_search_package_has_core_payload(bytes) { - let (package, _) = match verification { - PackageVerification::Trusted => { - prepared_search_core_package_view_trusted_from_bytes_with_timings( - bytes, - ) - } - PackageVerification::Verified => { - prepared_search_core_package_view_from_bytes_with_timings(bytes) - } - }?; - let artifacts = - PreparedEngineArtifactsView::from_bytes(package.artifacts.as_bytes())?; - let prepared = PreparedEngine::new_with_artifact_view_diagnostics( - package.config, - &artifacts, - )?; - return Ok(Self { - engine: prepared.prepared, - diagnostics: prepared.diagnostics, - }); + return Self::from_core_package_bytes(bytes, verification); } + Self::from_binding_package_bytes(bytes) + } + + fn from_core_package_bytes( + bytes: &[u8], + verification: PackageVerification, + ) -> Result { + let (package, _) = match verification { + PackageVerification::Trusted => { + prepared_search_core_package_view_trusted_from_bytes_with_timings(bytes) + } + PackageVerification::Verified => { + prepared_search_core_package_view_from_bytes_with_timings(bytes) + } + }?; + let artifacts = + PreparedEngineArtifactsView::from_bytes(package.artifacts.as_bytes())?; + let prepared = PreparedEngine::new_with_artifact_view_diagnostics( + package.config, + &artifacts, + )?; + Ok(Self { + engine: prepared.prepared, + diagnostics: prepared.diagnostics, + }) + } + fn from_binding_package_bytes(bytes: &[u8]) -> Result { let package = prepared_search_package_from_bytes(bytes)?; let config = prepared_search_config_from_binding(package.config)?; let artifacts = diff --git a/crates/anonymize-core/src/false_positives.rs b/crates/anonymize-core/src/false_positives.rs index 9687d9eb..cadcfc11 100644 --- a/crates/anonymize-core/src/false_positives.rs +++ b/crates/anonymize-core/src/false_positives.rs @@ -1,11 +1,14 @@ use std::collections::BTreeSet; +use std::ops::Range; use std::sync::LazyLock; use regex::Regex; use crate::byte_offsets::ByteOffsets; use crate::processors::DenyListFilterData; -use crate::resolution::{DetectionSource, PipelineEntity, SourceDetail}; +use crate::resolution::{ + DetectionSource, PipelineEntity, ResolutionDocument, SourceDetail, +}; use crate::types::{Error, Result}; use crate::labels::{ @@ -19,18 +22,67 @@ const ALL_CAPS_LINE_LETTER_THRESHOLD: usize = 5; const ALL_CAPS_LINE_RATIO: f64 = 0.95; const ALL_CAPS_LINE_PROSE_EXTRA_LETTERS: usize = 20; const ALL_CAPS_LINE_HEADING_WORD_LIMIT: usize = 5; +const MAX_PAGE_FOOTER_TOTAL: u32 = 1_000; static POSTAL_CODE_RE: LazyLock> = LazyLock::new(|| Regex::new(r"\d{3}\s?\d{2}").ok()); static SECTION_NUMBER_RE: LazyLock> = LazyLock::new(|| Regex::new(r"^(?:§\s*)?\d{1,3}(?:\.\d{1,3}){0,4}\.?$").ok()); +struct LineContext<'a> { + line: &'a str, + before: &'a str, + after: &'a str, + entity: Range, +} + +fn line_context<'a>( + document: &'a ResolutionDocument<'a>, + offsets: &ByteOffsets<'_>, + entity: &PipelineEntity, +) -> Result>> { + let full_text = document.text(); + let start = offsets.validate_offset(entity.start)?; + let end = offsets.validate_offset(entity.end)?; + if start > end { + return Err(Error::InvalidSpan { + start: entity.start, + end: entity.end, + }); + } + let Some(line_range) = document.line_range(start, end) else { + return Ok(None); + }; + let line = full_text + .get(line_range.clone()) + .ok_or(Error::InvalidSpan { + start: entity.start, + end: entity.end, + })?; + let relative_start = start.saturating_sub(line_range.start); + let relative_end = end.saturating_sub(line_range.start); + let before = line.get(..relative_start).ok_or(Error::InvalidSpan { + start: entity.start, + end: entity.end, + })?; + let after = line.get(relative_end..).ok_or(Error::InvalidSpan { + start: entity.start, + end: entity.end, + })?; + Ok(Some(LineContext { + line, + before, + after, + entity: relative_start..relative_end, + })) +} + pub(crate) fn filter_entity_false_positives( entities: Vec, - full_text: &str, + document: &ResolutionDocument<'_>, filters: Option<&DenyListFilterData>, ) -> Result> { - let offsets = ByteOffsets::new(full_text); + let offsets = document.offsets(); let mut filtered = Vec::with_capacity(entities.len()); for entity in entities { if is_caller_owned(&entity) { @@ -41,7 +93,7 @@ pub(crate) fn filter_entity_false_positives( let Some(normalized) = normalize_entity(entity, &offsets, filters)? else { continue; }; - if should_reject_entity(&normalized, full_text, &offsets, filters)? { + if should_reject_entity(&normalized, document, &offsets, filters)? { continue; } filtered.push(normalized); @@ -132,10 +184,11 @@ fn normalize_entity( fn should_reject_entity( entity: &PipelineEntity, - full_text: &str, + document: &ResolutionDocument<'_>, offsets: &ByteOffsets<'_>, filters: Option<&DenyListFilterData>, ) -> Result { + let full_text = document.text(); let text = entity.text.trim(); if is_template_placeholder(text) { return Ok(true); @@ -146,7 +199,17 @@ fn should_reject_entity( if exceeds_open_ended_word_count(entity) { return Ok(true); } + // Explicit section markers (`§ 6`, `6.1`, `3.2.4`) are never addresses, + // including when a place-of-performance cue extracts them as trigger values. + // A single dotted number needs heading context because the same shape is + // valid for sentence-final house numbers and postal codes. + if entity.label == ADDRESS_LABEL + && is_explicit_address_section(document, offsets, entity)? + { + return Ok(true); + } if entity.label != IP_ADDRESS_LABEL + && entity.label != ADDRESS_LABEL && is_section_number(text) && entity.source != DetectionSource::Trigger { @@ -199,7 +262,7 @@ fn should_reject_entity( } if entity.label == ORGANIZATION_LABEL && is_all_caps_candidate(text) - && is_all_caps_boilerplate_line(full_text, offsets, entity)? + && is_all_caps_boilerplate_line(document, offsets, entity)? { return Ok(true); } @@ -209,6 +272,12 @@ fn should_reject_entity( { return Ok(true); } + if entity.label == ORGANIZATION_LABEL + && let Some(filters) = filters + && is_numbered_page_footer(document, offsets, entity, filters)? + { + return Ok(true); + } if entity.label == ADDRESS_LABEL && should_reject_address(entity, filters) { return Ok(true); } @@ -321,6 +390,41 @@ fn is_section_number(text: &str) -> bool { regex_is_match(&SECTION_NUMBER_RE, text.trim()) } +fn is_explicit_address_section( + document: &ResolutionDocument<'_>, + offsets: &ByteOffsets<'_>, + entity: &PipelineEntity, +) -> Result { + let trimmed = entity.text.trim(); + if let Some(section) = trimmed.strip_prefix('§') { + return Ok( + !section.trim().is_empty() + && section + .trim() + .chars() + .all(|ch| ch.is_ascii_digit() || ch == '.'), + ); + } + if !is_section_number(trimmed) { + return Ok(false); + } + let without_terminal = trimmed.trim_end_matches('.'); + if without_terminal.contains('.') { + return Ok(true); + } + if !trimmed.ends_with('.') { + return Ok(false); + } + + let Some(context) = line_context(document, offsets, entity)? else { + return Ok(false); + }; + if !context.before.trim().is_empty() { + return Ok(false); + } + Ok(starts_with_section_heading_prefix(context.line)) +} + fn is_standalone_year(text: &str) -> bool { let trimmed = text.trim(); trimmed.len() == 4 @@ -454,6 +558,67 @@ fn role_exact_match( .contains(&entity.text.trim().to_lowercase()) } +fn is_numbered_page_footer( + document: &ResolutionDocument<'_>, + offsets: &ByteOffsets<'_>, + entity: &PipelineEntity, + filters: &DenyListFilterData, +) -> Result { + if entity.source != DetectionSource::Trigger { + return Ok(false); + } + let Some((head, page)) = words_and_number(&entity.text) else { + return Ok(false); + }; + let head = head.to_lowercase(); + + let Some(context) = line_context(document, offsets, entity)? else { + return Ok(false); + }; + if !context.before.trim().is_empty() { + return Ok(false); + } + + let line_remainder = context.after.trim(); + if line_remainder.is_empty() { + return Ok( + filters.page_footer_markers.contains(&head) + && page <= MAX_PAGE_FOOTER_TOTAL, + ); + } + let counter = + bracketed_inner(line_remainder, '(', ')').unwrap_or(line_remainder); + let Some((counter_head, total)) = words_and_number(counter) else { + return Ok(false); + }; + let marker = format!("{head} {}", counter_head.to_lowercase()); + if !filters.page_footer_markers.contains(&marker) { + return Ok(false); + } + Ok(page <= total && total <= MAX_PAGE_FOOTER_TOTAL) +} + +fn words_and_number(text: &str) -> Option<(&str, u32)> { + let trimmed = text.trim().trim_start_matches(',').trim_start(); + let split = trimmed.rfind(char::is_whitespace)?; + let words = trimmed + .get(..split)? + .trim_end() + .trim_end_matches(':') + .trim_end(); + let number = trimmed.get(split..)?.trim(); + if words.is_empty() + || (words != "/" + && !words + .split_whitespace() + .all(|word| word.chars().all(char::is_alphabetic))) + || !number.chars().all(|ch| ch.is_ascii_digit()) + { + return None; + } + Some((words, number.parse().ok()?)) +} + fn is_all_caps_candidate(text: &str) -> bool { let mut has_upper = false; for ch in text.chars().filter(|ch| ch.is_alphabetic()) { @@ -466,39 +631,18 @@ fn is_all_caps_candidate(text: &str) -> bool { } fn is_all_caps_boilerplate_line( - full_text: &str, + document: &ResolutionDocument<'_>, offsets: &ByteOffsets<'_>, entity: &PipelineEntity, ) -> Result { - let start = offsets.validate_offset(entity.start)?; - let end = offsets.validate_offset(entity.end)?; - let before = full_text.get(..start).ok_or(Error::InvalidSpan { - start: entity.start, - end: entity.end, - })?; - let line_start = before - .rfind('\n') - .map_or(0usize, |index| index.saturating_add('\n'.len_utf8())); - let after = full_text.get(end..).ok_or(Error::InvalidSpan { - start: entity.start, - end: entity.end, - })?; - let line_end = after - .find('\n') - .map_or(full_text.len(), |index| end.saturating_add(index)); - let line = full_text - .get(line_start..line_end) - .ok_or(Error::InvalidSpan { - start: entity.start, - end: entity.end, - })?; - let entity_rel_start = start.saturating_sub(line_start); - let entity_rel_end = end.saturating_sub(line_start); + let Some(context) = line_context(document, offsets, entity)? else { + return Ok(false); + }; let mut letter_count = 0usize; let mut upper_count = 0usize; let mut outside_entity_letters = 0usize; - for (index, ch) in line.char_indices() { + for (index, ch) in context.line.char_indices() { if !ch.is_alphabetic() { continue; } @@ -506,7 +650,7 @@ fn is_all_caps_boilerplate_line( if ch.is_uppercase() { upper_count = upper_count.saturating_add(1); } - if index < entity_rel_start || index >= entity_rel_end { + if index < context.entity.start || index >= context.entity.end { outside_entity_letters = outside_entity_letters.saturating_add(1); } } @@ -517,7 +661,7 @@ fn is_all_caps_boilerplate_line( if !uppercase_ratio_at_least(upper_count, letter_count) { return Ok(false); } - if starts_with_section_heading_prefix(line) { + if starts_with_section_heading_prefix(context.line) { return Ok(true); } if outside_entity_letters >= ALL_CAPS_LINE_PROSE_EXTRA_LETTERS { @@ -1224,6 +1368,18 @@ mod tests { use super::*; + fn filter_entity_false_positives( + entities: Vec, + full_text: &str, + filters: Option<&DenyListFilterData>, + ) -> Result> { + super::filter_entity_false_positives( + entities, + &ResolutionDocument::new(full_text), + filters, + ) + } + #[test] fn normalization_reuses_unchanged_text_allocation() -> Result<()> { let full_text = "Alice"; @@ -1897,6 +2053,148 @@ mod tests { assert!(entities.is_empty()); } + #[test] + fn keeps_multiline_all_caps_organizations() { + let text = "ACME\nCORP"; + let entities = filter_entity_false_positives( + vec![entity( + text, + text, + ORGANIZATION_LABEL, + DetectionSource::Regex, + )], + text, + Some(&DenyListFilterData::default()), + ) + .unwrap(); + + assert_eq!(entities.len(), 1); + assert_eq!(entities[0].text, "ACME CORP"); + } + + #[test] + fn rejects_explicit_address_sections_but_keeps_address_numbers() { + for (full_text, marker) in [ + ("6. Heading", "6."), + ("6.1", "6.1"), + ("3.2.4", "3.2.4"), + ("§ 1983", "§ 1983"), + ] { + let section = filter_entity_false_positives( + vec![entity( + full_text, + marker, + ADDRESS_LABEL, + DetectionSource::Trigger, + )], + full_text, + Some(&DenyListFilterData::default()), + ) + .unwrap(); + assert!(section.is_empty(), "{marker}"); + } + + for (full_text, value) in + [("123", "123"), ("č.p. 6.", "6."), ("C.P. 28001.", "28001.")] + { + let address_number = filter_entity_false_positives( + vec![entity( + full_text, + value, + ADDRESS_LABEL, + DetectionSource::Trigger, + )], + full_text, + Some(&DenyListFilterData::default()), + ) + .unwrap(); + + assert_eq!(address_number.len(), 1, "{full_text}"); + } + } + + #[test] + fn rejects_numbered_page_footers_without_hiding_numbered_names() { + let filters = DenyListFilterData { + page_footer_markers: set([ + "oldal /", + "oldal összesen", + "strana", + "stran celkem", + "strana celkem", + "strany celkem", + "strona łącznie", + ]), + ..DenyListFilterData::default() + }; + let text = "Strana 7 (celkem 7)\rStrany 4 (celkem 9)\r\nStran celkem 9\nStrana 8\nStrona 4 (łącznie 9)\nOldal 1 / 2\nOldal: 1 (összesen: 7)\nStudio 54 (Group 100)\nAcme Industries"; + let entities = filter_entity_false_positives( + vec![ + entity( + text, + "Strana 7", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Strany 4", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Stran celkem 9", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Strana 8", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Strona 4", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Oldal 1", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Oldal: 1", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Studio 54", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + entity( + text, + "Acme Industries", + ORGANIZATION_LABEL, + DetectionSource::Trigger, + ), + ], + text, + Some(&filters), + ) + .unwrap(); + + assert_eq!(entities.len(), 2); + assert_eq!(entities[0].text, "Studio 54"); + assert_eq!(entities[1].text, "Acme Industries"); + } + #[test] fn keeps_ipv4_addresses_that_resemble_section_numbers() { let text = "192.0.2.1"; @@ -1917,9 +2215,11 @@ mod tests { label: &str, source: DetectionSource, ) -> PipelineEntity { + let start = full_text.find(text).expect("entity text is in fixture"); + let end = start.saturating_add(text.len()); PipelineEntity::detected( - 0, - u32::try_from(full_text.len()).expect("fixture length fits u32"), + u32::try_from(start).expect("fixture offset fits u32"), + u32::try_from(end).expect("fixture offset fits u32"), label, text, 0.8, diff --git a/crates/anonymize-core/src/prepared/resolution_phase.rs b/crates/anonymize-core/src/prepared/resolution_phase.rs index ba6841a3..e7279ee0 100644 --- a/crates/anonymize-core/src/prepared/resolution_phase.rs +++ b/crates/anonymize-core/src/prepared/resolution_phase.rs @@ -97,7 +97,7 @@ impl PreparedEngine { let mut resolved_entities = filter_entities_for_config( filter_entity_false_positives( sanitized_entities, - full_text, + &document, false_positive_filters, )?, self.policy.threshold, @@ -253,7 +253,7 @@ impl PreparedEngine { let sanitized = sanitize_entities_with_document(consistent, document)?; let filtered = filter_entity_false_positives( sanitized, - full_text, + document, false_positive_filters, )?; Ok(filter_entities_for_labels( diff --git a/crates/anonymize-core/src/processors.rs b/crates/anonymize-core/src/processors.rs index 8314f2cb..2f190aa8 100644 --- a/crates/anonymize-core/src/processors.rs +++ b/crates/anonymize-core/src/processors.rs @@ -713,6 +713,7 @@ pub struct DenyListFilterData { pub ambiguous_street_type_terms: BTreeSet, pub first_names: BTreeSet, pub generic_roles: BTreeSet, + pub page_footer_markers: BTreeSet, pub number_abbrev_prefixes: BTreeSet, pub sentence_starters: BTreeSet, pub trailing_address_word_exclusions: BTreeSet, diff --git a/crates/anonymize-core/src/resolution/document.rs b/crates/anonymize-core/src/resolution/document.rs index ed6ce02c..973a3d09 100644 --- a/crates/anonymize-core/src/resolution/document.rs +++ b/crates/anonymize-core/src/resolution/document.rs @@ -1,4 +1,5 @@ use std::collections::BTreeSet; +use std::ops::Range; use std::sync::OnceLock; use crate::byte_offsets::ByteOffsets; @@ -19,6 +20,9 @@ pub(super) struct WordAnalysis { pub(crate) struct ResolutionDocument<'a> { text: &'a str, + line_starts: OnceLock>, + #[cfg(test)] + line_operations: std::cell::Cell, word_analysis: OnceLock, } @@ -26,6 +30,9 @@ impl<'a> ResolutionDocument<'a> { pub(crate) const fn new(text: &'a str) -> Self { Self { text, + line_starts: OnceLock::new(), + #[cfg(test)] + line_operations: std::cell::Cell::new(0), word_analysis: OnceLock::new(), } } @@ -42,6 +49,74 @@ impl<'a> ResolutionDocument<'a> { self.offsets().slice_ref(start, end) } + pub(crate) fn line_range( + &self, + start: usize, + end: usize, + ) -> Option> { + if start > end || end > self.text.len() { + return None; + } + let starts = self.line_starts.get_or_init(|| { + let mut starts = vec![0]; + let bytes = self.text.as_bytes(); + let mut index = 0_usize; + while let Some(byte) = bytes.get(index) { + #[cfg(test)] + self + .line_operations + .set(self.line_operations.get().saturating_add(1)); + let delimiter_len = match byte { + b'\r' + if bytes.get(index.saturating_add(1)).copied() == Some(b'\n') => + { + 2 + } + b'\r' | b'\n' => 1, + _ => { + index = index.saturating_add(1); + continue; + } + }; + index = index.saturating_add(delimiter_len); + starts.push(index); + } + starts + }); + let mut left = 0_usize; + let mut right = starts.len(); + while left < right { + #[cfg(test)] + self + .line_operations + .set(self.line_operations.get().saturating_add(1)); + let middle = left.midpoint(right); + if *starts.get(middle)? <= start { + left = middle.saturating_add(1); + } else { + right = middle; + } + } + let line_index = left.checked_sub(1)?; + let line_start = *starts.get(line_index)?; + let line_end = starts.get(line_index.saturating_add(1)).map_or( + self.text.len(), + |next_start| { + let delimiter_len = + if next_start.checked_sub(2).and_then(|delimiter_start| { + self.text.as_bytes().get(delimiter_start..*next_start) + }) == Some(b"\r\n".as_slice()) + { + 2 + } else { + 1 + }; + next_start.saturating_sub(delimiter_len) + }, + ); + (end <= line_end).then_some(line_start..line_end) + } + pub(super) fn word_analysis(&self) -> &WordAnalysis { self.word_analysis.get_or_init(|| { let spans = char_spans(self.text); @@ -137,8 +212,61 @@ const fn is_combining_mark(ch: char) -> bool { #[cfg(test)] mod tests { + use proptest::prelude::*; + use super::ResolutionDocument; + fn generated_lines( + segments: &[String], + ending_codes: &[u8], + ) -> (String, Vec>) { + let mut text = String::new(); + let mut ranges = Vec::with_capacity(segments.len()); + for (index, segment) in segments.iter().enumerate() { + let start = text.len(); + text.push_str(segment); + ranges.push(start..text.len()); + if index.saturating_add(1) == segments.len() { + continue; + } + let ending = ending_codes + .get(index.checked_rem(ending_codes.len()).unwrap_or_default()) + .copied() + .unwrap_or_default(); + text.push_str(match ending % 3 { + 0 => "\n", + 1 => "\r\n", + _ => "\r", + }); + } + (text, ranges) + } + + proptest! { + #[test] + fn generated_line_index_matches_reference_model( + segments in proptest::collection::vec("[A-Za-z0-9 ]{0,16}", 1..32), + ending_codes in proptest::collection::vec(any::(), 0..32), + queries in proptest::collection::vec((any::(), any::()), 0..64), + ) { + let (text, ranges) = generated_lines(&segments, &ending_codes); + let document = ResolutionDocument::new(&text); + for (first, second) in queries { + let divisor = text.len().saturating_add(1); + let left = first.checked_rem(divisor).unwrap_or_default(); + let right = second.checked_rem(divisor).unwrap_or_default(); + let start = left.min(right); + let end = left.max(right); + let expected = ranges + .iter() + .find(|range| start >= range.start && end <= range.end) + .cloned(); + + prop_assert_eq!(document.line_range(start, end), expected); + } + } + } + #[test] fn word_analysis_is_built_once_and_reused() { let document = ResolutionDocument::new("Jean d’Arc"); @@ -149,4 +277,50 @@ mod tests { assert!(first.boundaries.contains(&0)); assert!(first.boundaries.contains(&12)); } + + #[test] + fn line_ranges_are_built_lazily_and_reused() { + let document = ResolutionDocument::new("first\nsecond"); + + assert!(document.line_starts.get().is_none()); + assert_eq!(document.line_range(6, 12), Some(6..12)); + let first = document.line_starts.get().map(Vec::as_ptr); + assert!(first.is_some()); + assert_eq!(document.line_range(0, 5), Some(0..5)); + assert_eq!(first, document.line_starts.get().map(Vec::as_ptr)); + } + + #[test] + fn line_ranges_support_all_line_endings() { + for (text, range) in [ + ("first\nsecond", 6..12), + ("first\r\nsecond", 7..13), + ("first\rsecond", 6..12), + ] { + let document = ResolutionDocument::new(text); + assert_eq!(document.line_range(range.start, range.end), Some(range)); + } + } + + #[test] + fn dense_line_queries_have_bounded_structural_work() { + const LINE_COUNT: usize = 10_000; + let text = "x\n".repeat(LINE_COUNT); + let document = ResolutionDocument::new(&text); + + for index in 0..LINE_COUNT { + let start = index.saturating_mul(2); + assert_eq!( + document.line_range(start, start + 1), + Some(start..start + 1) + ); + } + + let linear_build_work = text.len(); + let logarithmic_query_work = LINE_COUNT.saturating_mul(20); + assert!( + document.line_operations.get() + <= linear_build_work.saturating_add(logarithmic_query_work) + ); + } } diff --git a/crates/anonymize-core/tests/assemble_parity.rs b/crates/anonymize-core/tests/assemble_parity.rs index 69d7f219..3628bdbd 100644 --- a/crates/anonymize-core/tests/assemble_parity.rs +++ b/crates/anonymize-core/tests/assemble_parity.rs @@ -394,7 +394,7 @@ fn compare_filters( match (got, want) { (None, None) => Ok(()), (Some(got), Some(want)) => { - let fields: [(&str, &Vec, &Vec); 18] = [ + let fields: [(&str, &Vec, &Vec); 19] = [ ("stopwords", &got.stopwords, &want.stopwords), ("allow_list", &got.allow_list, &want.allow_list), ( @@ -435,6 +435,11 @@ fn compare_filters( ), ("first_names", &got.first_names, &want.first_names), ("generic_roles", &got.generic_roles, &want.generic_roles), + ( + "page_footer_markers", + &got.page_footer_markers, + &want.page_footer_markers, + ), ( "number_abbrev_prefixes", &got.number_abbrev_prefixes, diff --git a/crates/anonymize-core/tests/fixtures/assemble/baseline-all-on.expected.json b/crates/anonymize-core/tests/fixtures/assemble/baseline-all-on.expected.json index f5da81b8..c1a262f3 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/baseline-all-on.expected.json +++ b/crates/anonymize-core/tests/fixtures/assemble/baseline-all-on.expected.json @@ -80489,7 +80489,89 @@ "suffix_phrases": ["a", "el"] } ], - "address_trailing_nouns": ["day"] + "address_trailing_nouns": ["day"], + "page_footer_markers": [ + "lapa", + "lapa kopā", + "lapa no", + "lapas", + "lapas kopā", + "lapas no", + "lappuse", + "lappuse kopā", + "lappuse no", + "lappuses", + "lappuses kopā", + "lappuses no", + "oldal", + "oldal /", + "oldal összesen", + "oldalak", + "oldalak /", + "oldalak összesen", + "page", + "page of", + "page sur", + "page total", + "pages", + "pages of", + "pages sur", + "pages total", + "pagina", + "pagina di", + "pagina din", + "pagina total", + "pagina totale", + "pagine", + "pagine di", + "pagine totale", + "pagini", + "pagini din", + "pagini total", + "página", + "página de", + "página total", + "páginas", + "páginas de", + "páginas total", + "seite", + "seite gesamt", + "seite insgesamt", + "seite von", + "seiten", + "seiten gesamt", + "seiten insgesamt", + "seiten von", + "sida", + "sida av", + "sida totalt", + "sidor", + "sidor av", + "sidor totalt", + "stran", + "stran celkem", + "stran z", + "strana", + "strana celkem", + "strana celkom", + "strana z", + "strany", + "strany celkem", + "strany celkom", + "strany z", + "stron", + "stron z", + "stron łącznie", + "strona", + "strona z", + "strona łącznie", + "strony", + "strony z", + "strony łącznie", + "strán", + "strán celkom", + "strán z" + ] } }, "false_positive_filters": { @@ -92915,7 +92997,89 @@ "suffix_phrases": ["a", "el"] } ], - "address_trailing_nouns": ["day"] + "address_trailing_nouns": ["day"], + "page_footer_markers": [ + "lapa", + "lapa kopā", + "lapa no", + "lapas", + "lapas kopā", + "lapas no", + "lappuse", + "lappuse kopā", + "lappuse no", + "lappuses", + "lappuses kopā", + "lappuses no", + "oldal", + "oldal /", + "oldal összesen", + "oldalak", + "oldalak /", + "oldalak összesen", + "page", + "page of", + "page sur", + "page total", + "pages", + "pages of", + "pages sur", + "pages total", + "pagina", + "pagina di", + "pagina din", + "pagina total", + "pagina totale", + "pagine", + "pagine di", + "pagine totale", + "pagini", + "pagini din", + "pagini total", + "página", + "página de", + "página total", + "páginas", + "páginas de", + "páginas total", + "seite", + "seite gesamt", + "seite insgesamt", + "seite von", + "seiten", + "seiten gesamt", + "seiten insgesamt", + "seiten von", + "sida", + "sida av", + "sida totalt", + "sidor", + "sidor av", + "sidor totalt", + "stran", + "stran celkem", + "stran z", + "strana", + "strana celkem", + "strana celkom", + "strana z", + "strany", + "strany celkem", + "strany celkom", + "strany z", + "stron", + "stron z", + "stron łącznie", + "strona", + "strona z", + "strona łącznie", + "strony", + "strony z", + "strony łącznie", + "strán", + "strán celkom", + "strán z" + ] }, "name_corpus_data": { "first_names": [ diff --git a/crates/anonymize-core/tests/fixtures/assemble/language-cs-sk.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/language-cs-sk.expected.delta.json index 80b4ae9a..be86ab4a 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/language-cs-sk.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/language-cs-sk.expected.delta.json @@ -1009,6 +1009,22 @@ "path": ["deny_list_data", "filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 57, + "end": 68 + }, + { + "type": "copy", + "start": 77, + "end": 80 + } + ] + }, { "type": "array", "path": ["false_positive_filters", "title_tokens"], @@ -1292,6 +1308,22 @@ "path": ["false_positive_filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 57, + "end": 68 + }, + { + "type": "copy", + "start": 77, + "end": 80 + } + ] + }, { "type": "array", "path": ["name_corpus_data", "title_tokens"], diff --git a/crates/anonymize-core/tests/fixtures/assemble/language-cs.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/language-cs.expected.delta.json index 9b5f40ad..77cd13bd 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/language-cs.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/language-cs.expected.delta.json @@ -991,6 +991,27 @@ "path": ["deny_list_data", "filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 57, + "end": 62 + }, + { + "type": "copy", + "start": 63, + "end": 66 + }, + { + "type": "copy", + "start": 67, + "end": 68 + } + ] + }, { "type": "array", "path": ["false_positive_filters", "title_tokens"], @@ -1279,6 +1300,27 @@ "path": ["false_positive_filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 57, + "end": 62 + }, + { + "type": "copy", + "start": 63, + "end": 66 + }, + { + "type": "copy", + "start": 67, + "end": 68 + } + ] + }, { "type": "array", "path": ["name_corpus_data", "title_tokens"], diff --git a/crates/anonymize-core/tests/fixtures/assemble/language-de.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/language-de.expected.delta.json index 4187c7ca..59619a06 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/language-de.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/language-de.expected.delta.json @@ -1102,6 +1102,17 @@ "path": ["deny_list_data", "filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 43, + "end": 51 + } + ] + }, { "type": "array", "path": ["false_positive_filters", "title_tokens"], @@ -1375,6 +1386,17 @@ "path": ["false_positive_filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 43, + "end": 51 + } + ] + }, { "type": "array", "path": ["name_corpus_data", "title_tokens"], diff --git a/crates/anonymize-core/tests/fixtures/assemble/language-en.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/language-en.expected.delta.json index 03aa39a9..13d022ad 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/language-en.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/language-en.expected.delta.json @@ -871,6 +871,27 @@ } ] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 18, + "end": 20 + }, + { + "type": "copy", + "start": 21, + "end": 24 + }, + { + "type": "copy", + "start": 25, + "end": 26 + } + ] + }, { "type": "array", "path": ["false_positive_filters", "title_tokens"], @@ -1169,6 +1190,27 @@ } ] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 18, + "end": 20 + }, + { + "type": "copy", + "start": 21, + "end": 24 + }, + { + "type": "copy", + "start": 25, + "end": 26 + } + ] + }, { "type": "array", "path": ["name_corpus_data", "title_tokens"], diff --git a/crates/anonymize-core/tests/fixtures/assemble/language-ja.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/language-ja.expected.delta.json index 304c1705..96910a97 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/language-ja.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/language-ja.expected.delta.json @@ -921,6 +921,11 @@ "path": ["deny_list_data", "filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [] + }, { "type": "array", "path": ["false_positive_filters", "person_trailing_nouns"], @@ -1148,6 +1153,11 @@ "path": ["false_positive_filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [] + }, { "type": "set", "path": ["slices", "regex", "end"], diff --git a/crates/anonymize-core/tests/fixtures/assemble/legal-forms-multilang.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/legal-forms-multilang.expected.delta.json index 5e8dd3d0..34c2a706 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/legal-forms-multilang.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/legal-forms-multilang.expected.delta.json @@ -1358,6 +1358,37 @@ "path": ["deny_list_data", "filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 18, + "end": 19 + }, + { + "type": "copy", + "start": 20, + "end": 23 + }, + { + "type": "copy", + "start": 24, + "end": 26 + }, + { + "type": "copy", + "start": 43, + "end": 51 + }, + { + "type": "copy", + "start": 68, + "end": 77 + } + ] + }, { "type": "array", "path": ["false_positive_filters", "title_tokens"], @@ -1766,6 +1797,37 @@ "path": ["false_positive_filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 18, + "end": 19 + }, + { + "type": "copy", + "start": 20, + "end": 23 + }, + { + "type": "copy", + "start": 24, + "end": 26 + }, + { + "type": "copy", + "start": 43, + "end": 51 + }, + { + "type": "copy", + "start": 68, + "end": 77 + } + ] + }, { "type": "array", "path": ["name_corpus_data", "title_tokens"], diff --git a/crates/anonymize-core/tests/fixtures/assemble/manifest.json b/crates/anonymize-core/tests/fixtures/assemble/manifest.json index 8daabe98..f01b115f 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/manifest.json +++ b/crates/anonymize-core/tests/fixtures/assemble/manifest.json @@ -9,247 +9,247 @@ "fixtures": [ { "name": "baseline-all-on", - "packageDigest": "83ee44a0a0c13f5576d30fc8f84339269b38865c7837c6f29b912e32c784bef2", + "packageDigest": "d6c6d645b8691f0b9e1df70d916caa02c03fbaa176189cc39898f0f8f15ef4b2", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableTriggerPhrases", - "packageDigest": "624b7cd4c7abb44e368efd5dd18fbaecd533042f1a5d04484d48b777b504151b", + "packageDigest": "01e37c0ffff0e7126bf8238c677c81c57f3c8919de88bc2b5e4e0e30cb17897c", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableRegex", - "packageDigest": "03791765371566b88ef7d36a1a5dcd44c6e9c76bf58d13dca0588322ffbd2846", + "packageDigest": "5af2f615c9e183059e8cac74379a754e6ae8395e739f998b0c48ebd18bf7eed2", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableLegalForms", - "packageDigest": "f8f1e52a6dce98a765ce049293b92e8d39b0d62cd9859be3b86677a1d9ced531", + "packageDigest": "129cb916fe1af76f7eb09a5de87fd9c300e02dcef4720a247742b2af85e11d58", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableNameCorpus", - "packageDigest": "cc23fa0e8b99b01aa0d24a97d0cc8b6a7d4c4135d3dc8c70ab2ba6ef4365231e", + "packageDigest": "0ac55548ef26713fa034f59daee800b782eac8c9f32cc8a8a312fbae744036fe", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableDenyList", - "packageDigest": "677a570829078ac6c2b9098cbbba6efe07406cf7ee750f56def0b90d7dab3bcd", + "packageDigest": "2a352df9dbd2b6b1522f10995b169e5c9db2997e8245eaadf59e595eca726dc8", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableGazetteer", - "packageDigest": "83ee44a0a0c13f5576d30fc8f84339269b38865c7837c6f29b912e32c784bef2", + "packageDigest": "d6c6d645b8691f0b9e1df70d916caa02c03fbaa176189cc39898f0f8f15ef4b2", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableCountries", - "packageDigest": "b4748c86325cf90c8c8ad4ff92fb7af32ec841f5308cdb4ba6488c7334f8dbbe", + "packageDigest": "1a26e0c53fd0e9ad129e7da7d00443fc09611bccc121b0b034d96449a2651691", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableNer", - "packageDigest": "83ee44a0a0c13f5576d30fc8f84339269b38865c7837c6f29b912e32c784bef2", + "packageDigest": "d6c6d645b8691f0b9e1df70d916caa02c03fbaa176189cc39898f0f8f15ef4b2", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableConfidenceBoost", - "packageDigest": "8cd37a66a476a78c42a107cf68338906ff370dc59e2fac0d1311c41db23e64f3", + "packageDigest": "df4e6b21214de70dabd6cde2573a7041401260f6d36782d638b14eea99e8d0e3", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableCoreference", - "packageDigest": "be618013b273cd03e893525a56b2f9fb8a21f51addc3116d0af6c6c06bfb23c3", + "packageDigest": "8f562d00c3475fa5c8b9ad18da5ccbca7db1e94f7f596725a440a05eaf371d83", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableZoneClassification", - "packageDigest": "c50c8e33c87c34d2613d2d4b64abd4de05951448d1aac276dc59a748e351f8df", + "packageDigest": "6752d0f31b1a9d0017cfe894ff87999ab1df83b2229828ac6e8d9c52c0619728", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "toggle-off-enableHotwordRules", - "packageDigest": "8bae3a694a7ed2380005266c379a4eac7f4bae0fc567e62f28f433bba1088465", + "packageDigest": "7fcd5786c280e76faa57f98e6dd7dd68c6cb317c5eb52a481aa15eaf18668da4", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableTriggerPhrases", - "packageDigest": "9c72f39d02b6e0a3815300927130a0103b2117e4c41ea516902f3432d26df7ce", + "packageDigest": "8191d570a9858dda69c9d2bb2ec405d621b9b4ac123d5d4a3f0411426809676a", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableRegex", - "packageDigest": "6887669bab941ab2d27b612873429066124b4d338087aea1b8e6a40b2a18bf6d", + "packageDigest": "353080dcfd8123dec56722c2300f4825bd80d1390614c20c04da6dcbdf0f522f", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableLegalForms", - "packageDigest": "5d7c32d5021d2d5e3c45ffe24fa7c1428a0ea6748f00c80256e504257b072283", + "packageDigest": "68d3ca65d49faf1c091708aa7c78b18f196940939d56308a28de788c94c92fc5", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableNameCorpus", - "packageDigest": "529abc1eb60b27f503ce3fdd49a109e55a2caa45f0dc4f7799bfcafdaf8d8870", + "packageDigest": "a3d78329be2bb180c60a74463c3b5b59f315ae64cb7fb8768a7a2f4f0b459d56", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableDenyList", - "packageDigest": "b752a57357401b7539e123161aef6af6961907d5d0ec98c2a8ebaf75f5e07e7e", + "packageDigest": "793e83aeec4ee3009a27685ebc6571adef87a51e8ebb2495463fe3478e1a5565", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableGazetteer", - "packageDigest": "b752a57357401b7539e123161aef6af6961907d5d0ec98c2a8ebaf75f5e07e7e", + "packageDigest": "793e83aeec4ee3009a27685ebc6571adef87a51e8ebb2495463fe3478e1a5565", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableCountries", - "packageDigest": "b14a4d399f912c3d8555c60deb184177cec86d6f2fe41789fee28367b17ca48d", + "packageDigest": "73c53b2a65c471c2b30275366eabecadade7b7213c5af5cd1721eee8d3bae6f9", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "isolate-enableCoreference", - "packageDigest": "f702081e830fc75dda88966278dd7d1ab3e8ddbba664fc046c0150bea084cc33", + "packageDigest": "cef80f231e46039eae103f0060002b9f14f304ce50a53a5b8a22c29e9df5db37", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "language-all", - "packageDigest": "83ee44a0a0c13f5576d30fc8f84339269b38865c7837c6f29b912e32c784bef2", + "packageDigest": "d6c6d645b8691f0b9e1df70d916caa02c03fbaa176189cc39898f0f8f15ef4b2", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "language-cs", - "packageDigest": "720ea5190fc2e1fcf595478301fa2118477f701e1bb89dfdc214b8fa27b6c16e", + "packageDigest": "2144b023343f010737605c82fa086885e681fa2c34978845b4daff991526f8f7", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "language-de", - "packageDigest": "29dd329f0725ff9e2af4a4ecce39d217cba31a9f67b8b484818850ff4776c5f9", + "packageDigest": "a9d9ea2c0e0c6da8f0f173d95110d17ba8a580989449b436cdb42d9384c93ddd", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "language-en", - "packageDigest": "af5fa7fc33be395410329d5df698de4af9aa85cbad3fae0eb44b89f5c7fbcee8", + "packageDigest": "87ea4f37c29d90187f55cdf0cef9e3f8802471fe3e8993803e268e0324643c3b", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "language-ja", - "packageDigest": "59bdb27f317fd80d44db8b054364f28681f28a6de83b911234f8193ed244b89b", + "packageDigest": "715bf880f5749c0b4b0585636d994f5c9f670a24ed0dab91fc001d1657e6ef26", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "language-cs-sk", - "packageDigest": "e8d2226c772666706c5890e82bf3fdbd5dc7d8b37145f77c3408db50c1b1f6e3", + "packageDigest": "11cb655ee791628971e8b9be26b9dc9a213910a86e743cd7068e9ba19972a2a9", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-regex-basic", - "packageDigest": "a950cd4ec52ce2eb805d19de8d04835eac41deaa2a6fc5006c40e1b20dfb5827", + "packageDigest": "c34ab1ba11ba61128e4bb8163d9b185f879a829ca85a9b1aad6ef12929e1a2fa", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-regex-prepared-omit", - "packageDigest": "1d9a5439302ed979554c31122b18b97ec45bf006adb43d34142237501fa75b54", + "packageDigest": "4fc604e637a6193ae317cd7b85830ede3297cd7f41b2d4393da686d1b6f91bc7", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-regex-prepared-include", - "packageDigest": "ca065b08a5c8ac220b7150d649bcadf5217d98958e3f5d3fd7922d932233e9ab", + "packageDigest": "199b8416652ae73caba6af1403b93bec3a93d8441681af0d98ef09960b6d9e06", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-regex-label-filtered", - "packageDigest": "19a91ad18eb102917bdc71612e1091728522586deed6952b640eb7e03dfe5a43", + "packageDigest": "2770c2da55bf8a9fd53260290d58cc1d0d63db39b0c1ee4c91d26158e9d4daa1", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-regex-disabled", - "packageDigest": "03791765371566b88ef7d36a1a5dcd44c6e9c76bf58d13dca0588322ffbd2846", + "packageDigest": "5af2f615c9e183059e8cac74379a754e6ae8395e739f998b0c48ebd18bf7eed2", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-deny-boundary-override", - "packageDigest": "4986ce2ac49926ec2209da2952e131c2f051d922203222da9b666a4ab29fc6d2", + "packageDigest": "2d00a173cab3a5a979a4010679450e5a20a73507df6fa24b4379f26135d757eb", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-deny-non-override", - "packageDigest": "7e02fb497f7e1a581991d9aecf586eda3cd4c763bed87a9b77b052fb9d8ce112", + "packageDigest": "22aec58c7de52c6959deb863f3f40618a10cffd21baf158c4341f09cb2eef08b", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "gazetteer-fuzzy", - "packageDigest": "c534fd54dc11b0bfcab1cbfeaf3599c6b67af9e392833515f2043f0a64c80bc7", + "packageDigest": "df9e39b90035306431926f1f7958356efb20a6e5a6ac26e069de0e49111f4eeb", "hasDictionaries": false, "gazetteerCount": 1 }, { "name": "labels-and-threshold", - "packageDigest": "99e5b7a416eb26a070e2b6afb679fcb95c4b0765b9c90449e604e6c575ebdb9d", + "packageDigest": "b7d241205650e77a7903759a8eb7a15c6ce1e1019196777f50ebb43032b9b7ef", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "confidence-boost-off", - "packageDigest": "8cd37a66a476a78c42a107cf68338906ff370dc59e2fac0d1311c41db23e64f3", + "packageDigest": "df4e6b21214de70dabd6cde2573a7041401260f6d36782d638b14eea99e8d0e3", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "custom-regex-hotword-reclassify", - "packageDigest": "e9ab3b67656edea16fdf85bd245a0e0454402066632246b6647db3badcb74594", + "packageDigest": "00d2d2f97cf61085e6d143d17eb4abfe77fd4575317bdb2155056e78215e892d", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "regex-only-no-triggers", - "packageDigest": "9079d7c14177cd254cb1cbbd88c1fc188d6dcea8b4320c1b748673c1851716eb", + "packageDigest": "fddc48a2e64ec81abe55a9a28060965867c9e63893e56fd7b9674059550b4c49", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "legal-forms-multilang", - "packageDigest": "066e89017fe3c084295a3755b6a40492ed37730f68deea2ba5b87370333aaaf6", + "packageDigest": "13089f01976ed0b0108f829e6549d9b3ade75a60a28751320a6991a32b1be9cf", "hasDictionaries": false, "gazetteerCount": 0 }, { "name": "with-test-dictionaries", - "packageDigest": "f875ffa50f1113572610be5a7f0d5209bafb7479436b5a8900a18fc54a9b6737", + "packageDigest": "dd14956d113ede2ba7cffbfb6fc9759eea358bccfb9b251e97bb3d035f95babe", "hasDictionaries": true, "gazetteerCount": 0 } diff --git a/crates/anonymize-core/tests/fixtures/assemble/with-test-dictionaries.expected.delta.json b/crates/anonymize-core/tests/fixtures/assemble/with-test-dictionaries.expected.delta.json index 82df93f0..1c5bd7bb 100644 --- a/crates/anonymize-core/tests/fixtures/assemble/with-test-dictionaries.expected.delta.json +++ b/crates/anonymize-core/tests/fixtures/assemble/with-test-dictionaries.expected.delta.json @@ -5055,6 +5055,27 @@ "path": ["deny_list_data", "filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["deny_list_data", "filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 57, + "end": 62 + }, + { + "type": "copy", + "start": 63, + "end": 66 + }, + { + "type": "copy", + "start": 67, + "end": 68 + } + ] + }, { "type": "array", "path": ["false_positive_filters", "title_tokens"], @@ -5343,6 +5364,27 @@ "path": ["false_positive_filters", "address_trailing_nouns"], "segments": [] }, + { + "type": "array", + "path": ["false_positive_filters", "page_footer_markers"], + "segments": [ + { + "type": "copy", + "start": 57, + "end": 62 + }, + { + "type": "copy", + "start": 63, + "end": 66 + }, + { + "type": "copy", + "start": 67, + "end": 68 + } + ] + }, { "type": "array", "path": ["name_corpus_data", "surnames"], diff --git a/crates/anonymize-napi/src/lib.rs b/crates/anonymize-napi/src/lib.rs index b5ab0858..f0b89d0b 100644 --- a/crates/anonymize-napi/src/lib.rs +++ b/crates/anonymize-napi/src/lib.rs @@ -423,6 +423,7 @@ pub struct JsDenyListFilterData { pub street_types: Vec, pub first_names: Vec, pub generic_roles: Vec, + pub page_footer_markers: Vec, pub sentence_starters: Vec, pub trailing_address_word_exclusions: Vec, pub defined_term_cues: Vec, diff --git a/packages/anonymize/src/__test__/document-scaffold-fp.test.ts b/packages/anonymize/src/__test__/document-scaffold-fp.test.ts new file mode 100644 index 00000000..fee875b3 --- /dev/null +++ b/packages/anonymize/src/__test__/document-scaffold-fp.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, setDefaultTimeout, test } from "bun:test"; +import { DEFAULT_ENTITY_LABELS } from "../constants"; +import type { Dictionaries, PipelineConfig } from "../types"; +import { detectNative } from "./native-detect"; +import { loadTestDictionaries } from "./load-dictionaries"; + +setDefaultTimeout(60_000); + +const CONFIG: PipelineConfig = { + threshold: 0.3, + languages: ["cs"], + enableTriggerPhrases: true, + enableRegex: true, + enableLegalForms: true, + enableNameCorpus: true, + enableDenyList: true, + enableGazetteer: false, + enableConfidenceBoost: true, + enableCoreference: true, + enableHotwordRules: true, + denyListCountries: ["CZ"], + nameCorpusLanguages: ["cs"], + labels: [...DEFAULT_ENTITY_LABELS], + workspaceId: "doc-scaffold-fp-test", +}; + +let cachedDictionaries: Dictionaries | undefined; +const detect = async ( + text: string, + languages: NonNullable = ["cs"], + denyListCountries: NonNullable = ["CZ"], +) => { + cachedDictionaries ??= await loadTestDictionaries(); + return detectNative( + { + ...CONFIG, + languages, + denyListCountries, + dictionaries: cachedDictionaries, + }, + text, + ); +}; + +describe("document scaffolding false positives", () => { + test("section markers and numbered page footers are not PII", async () => { + const text = + "Místo plnění\n\n6.1. Místem plnění se rozumí: Technická správa města\n\nStrana 7 (celkem 7)\nStrany 4 (celkem 9)\nStran celkem 9\nStrana 8"; + const entities = await detect(text); + expect( + entities.some((e) => e.label === "address" && e.text.trim() === "6.1"), + ).toBe(false); + expect( + entities.some( + (e) => + e.label === "organization" && + /^(?:Stran(?:a|y)?\s+\d+|Stran celkem \d+)$/u.test(e.text), + ), + ).toBe(false); + }); + + test("ordinary municipality organization trigger is unchanged", async () => { + const text = "město Brandýs nad Labem, IČO: 00240066"; + const entities = await detect(text); + expect( + entities.some( + (e) => + e.label === "organization" && e.text.includes("Brandýs nad Labem"), + ), + ).toBe(true); + }); + + test.each([ + ["Czech house number", ["cs"], ["CZ"], "č.p. 6.", "6"], + ["Spanish postal code", ["es"], ["ES"], "C.P. 28001.", "28001"], + ] as const)( + "%s remains PII at sentence end", + async (_name, languages, countries, text, expected) => { + const entities = await detect(text, [...languages], [...countries]); + expect( + entities.some( + (entity) => + entity.label === "address" && entity.text.trim() === expected, + ), + ).toBe(true); + }, + ); +}); diff --git a/packages/anonymize/src/native-search-config.ts b/packages/anonymize/src/native-search-config.ts index b8956341..83988f1a 100644 --- a/packages/anonymize/src/native-search-config.ts +++ b/packages/anonymize/src/native-search-config.ts @@ -75,6 +75,7 @@ export type NativeDenyListFilterData = { ambiguous_street_type_terms: string[]; first_names: string[]; generic_roles: string[]; + page_footer_markers?: string[]; number_abbrev_prefixes: string[]; sentence_starters: string[]; trailing_address_word_exclusions: string[]; diff --git a/packages/data/config/page-footer-markers.json b/packages/data/config/page-footer-markers.json new file mode 100644 index 00000000..af7cab14 --- /dev/null +++ b/packages/data/config/page-footer-markers.json @@ -0,0 +1,55 @@ +{ + "_comment": "Language-scoped page/footer vocabulary. The assembler creates standalone page-word markers and page-word/count-word pairs; the false-positive filter only rejects structurally numbered footers whose marker belongs to an enabled language. Keep this matrix aligned with every language in language-scopes.json.", + "cs": { + "pageWords": ["strana", "strany", "stran"], + "countWords": ["celkem", "z"] + }, + "de": { + "pageWords": ["seite", "seiten"], + "countWords": ["gesamt", "insgesamt", "von"] + }, + "en": { + "pageWords": ["page", "pages"], + "countWords": ["of", "total"] + }, + "pt-br": { + "pageWords": ["página", "páginas"], + "countWords": ["de", "total"] + }, + "es": { + "pageWords": ["página", "páginas"], + "countWords": ["de", "total"] + }, + "fr": { + "pageWords": ["page", "pages"], + "countWords": ["sur", "total"] + }, + "hu": { + "pageWords": ["oldal", "oldalak"], + "countWords": ["/", "összesen"] + }, + "it": { + "pageWords": ["pagina", "pagine"], + "countWords": ["di", "totale"] + }, + "lv": { + "pageWords": ["lapa", "lapas", "lappuse", "lappuses"], + "countWords": ["kopā", "no"] + }, + "pl": { + "pageWords": ["strona", "strony", "stron"], + "countWords": ["łącznie", "z"] + }, + "ro": { + "pageWords": ["pagina", "pagini"], + "countWords": ["din", "total"] + }, + "sk": { + "pageWords": ["strana", "strany", "strán"], + "countWords": ["celkom", "z"] + }, + "sv": { + "pageWords": ["sida", "sidor"], + "countWords": ["av", "totalt"] + } +}