diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d3cae9..3be2a21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,24 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). dialog lists each source's status with per-source actions (open, relink, leave out, remove, or open-available-only); and quitting or closing a project with unsaved workspace changes prompts to save first. +- **Data dictionary** — document what each column MEANS. Every column + carries an optional entry (display name, description, analytical role, + unit, source, sensitivity, allowed values, example, owner, notes) keyed + by its stable column ID, so the documentation survives renames and + reorders and is restored by undo/redo; deleting a column reports its + entry as orphaned and keeps it until you explicitly discard it (an undo + re-attaches it). Editing the dictionary is pure metadata: it has its own + revision, like the schema, and never rewrites a cell or marks the + document dirty. The editor prefills each column's technical name and + inferred F31 type. Dictionaries import and export as versioned CEESVEE + JSON, Markdown documentation, and tabular CSV documentation; an import + merges incoming metadata by column ID (or by mapped column name when the + IDs differ) and surfaces every field-level conflict for explicit + resolution before it replaces anything. File profiles can require + documentation fields (e.g. a description and owner on every column) as + ordinary validation issues, and columns classified confidential or + restricted are folded into the PII scan preflight even when no detector + matches them. - **Explicit schemas and typed columns** (palette → "Edit schema…", or a column header's menu): declare an explicit logical type per column — text, integer, decimal, float, boolean, date, datetime, UUID, or JSON — diff --git a/README.md b/README.md index 7cf1510..c5d6acf 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,20 @@ and faithful on large, real-world delimited files.** conversion applies as one previewed, undoable step. Schemas key columns by stable IDs so they survive renames and reorders; a violet header badge marks a declared type. +- **Data dictionary** — document what each column MEANS: display name, + description, analytical role, unit, source, sensitivity, allowed values, + example, owner, and notes, each keyed by the stable column ID so the + documentation survives renames and reorders (deleting a column reports + its entry as orphaned and keeps it). The searchable editor prefills every + column's technical name and inferred type, shows a per-column + completeness indicator, and surfaces the description, unit, and a + sensitivity badge as a column-header tooltip. Editing the dictionary is + pure metadata — it has its own revision and never dirties the document. + Import and export as versioned CEESVEE JSON, Markdown, or CSV + documentation; an import merges by column ID (or mapped name) and every + field-level conflict is resolved explicitly before it replaces anything. + File profiles can require documentation fields, and columns marked + confidential or restricted feed the personal-data preflight. - **Cross-column validation** — relational rules between columns (typed comparisons, date order, conditional required, sum equality with tolerance, allowed combinations, …) with violation samples, jump-to-row, diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index afe1998..4985e9b 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -21,6 +21,10 @@ use crate::crossval::{self, CrossRule, CrossValCache, CrossValReport}; use crate::dedup::{self, DedupCache, DedupSpec, DuplicateKeepStrategy, DuplicateReport}; use crate::diagnostics::{self, DiagnosticsCache, DiagnosticsReport}; use crate::dialect::{self, CsvDialectOptions, DialectPreview}; +use crate::dictionary::{ + self, DictionaryField, DictionaryFormat, DictionaryImportOutcome, DictionaryView, MergeMatchBy, + MergePlan, MergeResolution, +}; use crate::document::{ChangeSummary, Document}; use crate::dto::{ BackupPolicy, CellRect, ColumnSummary, DocumentMeta, EncodingCompatibility, @@ -582,8 +586,10 @@ pub async fn start_reindex( fresh.set_revision(doc.revision() + 1); fresh.set_fingerprint(fingerprint); // Schema entries key on stable IDs, which restart positionally on - // a reload — they re-attach to the same columns (F31). + // a reload — they re-attach to the same columns (F31). The data + // dictionary (F38) carries across on the same principle. fresh.inherit_schema(&doc); + fresh.inherit_dictionary(&doc); *doc = fresh; Ok(()) }) @@ -666,8 +672,10 @@ pub async fn apply_reparse( fresh.set_revision(doc.revision() + 1); fresh.set_fingerprint(fingerprint); // Schema entries key on stable IDs, which restart positionally on a - // reparse — they re-attach to the same columns (F31). + // reparse — they re-attach to the same columns (F31). The data + // dictionary (F38) carries across on the same principle. fresh.inherit_schema(doc); + fresh.inherit_dictionary(doc); // Journaling continues against the NEW interpretation. attach_journal_if_enabled(&app, &mut fresh); let meta = fresh.meta(); @@ -1886,6 +1894,166 @@ pub async fn apply_redaction( Ok(meta) } +// ----- data dictionary (F38) ---------------------------------------------------- + +/// The dictionary editor surface: one row per current column (technical name + +/// inferred F31 type prefilled, stored entry when documented) plus any orphaned +/// entries. `dictionaryRevision` is the metadata revision used to guard edits; +/// documentation edits never move the document `revision` or the dirty flag. +#[tauri::command] +pub fn get_dictionary(doc_id: u64, state: Db<'_>) -> AppResult { + read_doc(&state, doc_id, |doc| Ok(dictionary::view(doc))) +} + +/// Insert or replace one column's documentation. An entry with no populated +/// field is removed rather than stored empty. Metadata only: not undoable, +/// never dirties the document. Guarded by the dictionary revision. +#[tauri::command] +pub fn set_dictionary_field( + doc_id: u64, + field: DictionaryField, + expected_dictionary_revision: u64, + state: Db<'_>, +) -> AppResult { + write_doc(&state, doc_id, |doc| { + doc.check_dictionary_revision(expected_dictionary_revision)?; + dictionary::validate_field(&field)?; + // The entry must key on a column that exists (present or orphaned is + // fine — it is keyed by stable ID either way). + if field.is_documented() { + doc.set_dictionary_field(field); + } else { + doc.remove_dictionary_field(&field.column_id); + } + Ok(dictionary::view(doc)) + }) +} + +/// Drop one column's documentation entry (clearing a column, or discarding a +/// single orphan). Guarded by the dictionary revision. +#[tauri::command] +pub fn remove_dictionary_field( + doc_id: u64, + column_id: String, + expected_dictionary_revision: u64, + state: Db<'_>, +) -> AppResult { + write_doc(&state, doc_id, |doc| { + doc.check_dictionary_revision(expected_dictionary_revision)?; + doc.remove_dictionary_field(&column_id); + Ok(dictionary::view(doc)) + }) +} + +/// Discard EVERY orphaned entry (documentation whose column is gone). The +/// user's explicit "clean up orphans" action. Guarded by the dictionary +/// revision. +#[tauri::command] +pub fn discard_dictionary_orphans( + doc_id: u64, + expected_dictionary_revision: u64, + state: Db<'_>, +) -> AppResult { + write_doc(&state, doc_id, |doc| { + doc.check_dictionary_revision(expected_dictionary_revision)?; + let orphan_ids: Vec = dictionary::orphans(doc) + .into_iter() + .map(|o| o.column_id) + .collect(); + if !orphan_ids.is_empty() { + let mut dict = doc.dictionary().clone(); + for id in &orphan_ids { + dict.remove(id); + } + doc.set_dictionary(dict); + } + Ok(dictionary::view(doc)) + }) +} + +/// Export the dictionary as versioned JSON, Markdown documentation, or tabular +/// CSV documentation (atomic write via the F03 pipeline). +#[tauri::command] +pub async fn export_dictionary( + doc_id: u64, + path: String, + format: DictionaryFormat, + state: Db<'_>, +) -> AppResult<()> { + let handle = doc_handle(&state, doc_id)?; + tauri::async_runtime::spawn_blocking(move || { + let rendered = { + let doc = handle.read().map_err(poisoned)?; + dictionary::export_as(&doc, format)? + }; + save_mod::atomic_write(Path::new(&path), BackupPolicy::None, |file| { + use std::io::Write; + file.write_all(rendered.as_bytes())?; + Ok(rendered.len() as u64) + })?; + Ok(()) + }) + .await + .map_err(|e| AppError::Other(format!("background task failed: {e}")))? +} + +/// Plan a dictionary import: parse the CEESVEE dictionary JSON at `path`, match +/// its entries to current columns by ID or mapped name, and return the merge +/// plan (clean additions + the field-level conflicts that must be resolved +/// before applying). Read-only — nothing changes. +#[tauri::command] +pub async fn preview_dictionary_import( + doc_id: u64, + path: String, + match_by: MergeMatchBy, + state: Db<'_>, +) -> AppResult { + let handle = doc_handle(&state, doc_id)?; + tauri::async_runtime::spawn_blocking(move || { + let json = std::fs::read_to_string(&path)?; + let imported = dictionary::parse_import(&json)?; + let doc = handle.read().map_err(poisoned)?; + Ok(dictionary::plan_merge(&doc, &imported, match_by)) + }) + .await + .map_err(|e| AppError::Other(format!("background task failed: {e}")))? +} + +/// Apply a dictionary import under an explicit conflict resolution. Fails +/// (changing nothing) if any reported conflict is left unresolved, or if the +/// dictionary moved since the plan was taken. Metadata only: never dirties the +/// document. +#[tauri::command] +pub async fn apply_dictionary_import( + doc_id: u64, + path: String, + match_by: MergeMatchBy, + resolution: MergeResolution, + expected_dictionary_revision: u64, + state: Db<'_>, +) -> AppResult { + let handle = doc_handle(&state, doc_id)?; + tauri::async_runtime::spawn_blocking(move || { + let json = std::fs::read_to_string(&path)?; + let imported = dictionary::parse_import(&json)?; + let mut doc = handle.write().map_err(poisoned)?; + doc.check_dictionary_revision(expected_dictionary_revision)?; + let applied = dictionary::apply_merge(&doc, &imported, match_by, &resolution)?; + doc.set_dictionary(applied.dictionary); + Ok(DictionaryImportOutcome { + matched_columns: applied.matched_columns, + new_entries: applied.new_entries, + updated_entries: applied.updated_entries, + fields_added: applied.fields_added, + conflicts_resolved: applied.conflicts_resolved, + unmatched: applied.unmatched, + view: dictionary::view(&doc), + }) + }) + .await + .map_err(|e| AppError::Other(format!("background task failed: {e}")))? +} + // ----- batch recipes (F25) ------------------------------------------------------ /// Validate a batch (recipe version, steps, templates, distinct output diff --git a/src-tauri/src/dictionary.rs b/src-tauri/src/dictionary.rs new file mode 100644 index 0000000..41f3505 --- /dev/null +++ b/src-tauri/src/dictionary.rs @@ -0,0 +1,1673 @@ +//! Data dictionary (F38): human documentation of what each column MEANS, +//! linked to the F31 stable column ID so it survives renames, reorders and +//! undo/redo. The dictionary is pure metadata: it lives on the [`Document`] +//! beside the schema, has its OWN revision, and editing it never rewrites a +//! cell or makes the document dirty. +//! +//! This module owns the documentation model ([`DictionaryField`]), the +//! per-document container ([`Dictionary`]), the editor view (with technical +//! names + inferred F31 types prefilled), versioned JSON / Markdown / CSV +//! export, and the import MERGE engine — which matches incoming entries by +//! column ID (or by mapped column name when IDs are absent) and produces a +//! field-level conflict report that must be explicitly resolved before it can +//! replace anything. It also exposes two integration hooks consumed elsewhere: +//! required-documentation checks for F08 file profiles, and the sensitive +//! (confidential/restricted) column set for the F28 PII preflight. + +use std::collections::{BTreeMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +use crate::document::Document; +use crate::error::{AppError, AppResult}; +use crate::schema::LogicalType; + +/// Import/export envelope version. Bumped only on an incompatible format +/// change; unknown fields within a version are ignored (forward-tolerant). +pub const DICTIONARY_VERSION: u32 = 1; + +// --------------------------------------------------------------------------- +// Documentation model (wire DTOs, camelCase) +// --------------------------------------------------------------------------- + +/// The analytical role a column plays. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FieldRole { + Identifier, + Dimension, + Measure, + Timestamp, + Label, +} + +impl FieldRole { + pub fn label(self) -> &'static str { + match self { + FieldRole::Identifier => "identifier", + FieldRole::Dimension => "dimension", + FieldRole::Measure => "measure", + FieldRole::Timestamp => "timestamp", + FieldRole::Label => "label", + } + } +} + +/// Data-sensitivity classification, ordered least → most sensitive so +/// `>= Confidential` selects the values the PII preflight flags. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Sensitivity { + Public, + Internal, + Confidential, + Restricted, +} + +impl Sensitivity { + pub fn label(self) -> &'static str { + match self { + Sensitivity::Public => "public", + Sensitivity::Internal => "internal", + Sensitivity::Confidential => "confidential", + Sensitivity::Restricted => "restricted", + } + } + + /// Whether this classification makes a column PII-sensitive regardless of + /// pattern hits (confidential or restricted). Consumed by [`sensitive_columns`]. + pub fn is_sensitive(self) -> bool { + self >= Sensitivity::Confidential + } +} + +/// One column's documentation. Every descriptive field is optional; the entry +/// is keyed by the STABLE column ID (F12), never by position or header text. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryField { + pub column_id: String, + /// Human-friendly name (the technical header stays the source of truth). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub display_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + /// Unit of measure ("USD", "ms", "kg"). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unit: Option, + /// Where the values originate (system of record, upstream table). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sensitivity: Option, + /// Enumerated permitted values, when the column is categorical. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub allowed_values: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub example: Option, + /// Data owner / steward. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub notes: Option, +} + +impl DictionaryField { + /// An empty entry for `column_id` (used to prefill the editor and as the + /// merge base for a column with no existing documentation). + pub fn empty(column_id: impl Into) -> Self { + DictionaryField { + column_id: column_id.into(), + ..DictionaryField::default() + } + } + + /// Whether any documentation field carries a real (non-blank) value. + pub fn is_documented(&self) -> bool { + ALL_FIELD_KEYS.iter().any(|&k| value_of(self, k).is_some()) + } +} + +/// Every documentable field, as a closed enum: used by the merge engine, the +/// conflict report and the F08 required-documentation profile rule. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DictionaryFieldKey { + DisplayName, + Description, + Role, + Unit, + Source, + Sensitivity, + AllowedValues, + Example, + Owner, + Notes, +} + +/// All field keys, in stable presentation order. +pub const ALL_FIELD_KEYS: [DictionaryFieldKey; 10] = [ + DictionaryFieldKey::DisplayName, + DictionaryFieldKey::Description, + DictionaryFieldKey::Role, + DictionaryFieldKey::Unit, + DictionaryFieldKey::Source, + DictionaryFieldKey::Sensitivity, + DictionaryFieldKey::AllowedValues, + DictionaryFieldKey::Example, + DictionaryFieldKey::Owner, + DictionaryFieldKey::Notes, +]; + +impl DictionaryFieldKey { + /// Human label, used in conflict reports, profile issues and MD/CSV headers. + pub fn label(self) -> &'static str { + match self { + DictionaryFieldKey::DisplayName => "display name", + DictionaryFieldKey::Description => "description", + DictionaryFieldKey::Role => "role", + DictionaryFieldKey::Unit => "unit", + DictionaryFieldKey::Source => "source", + DictionaryFieldKey::Sensitivity => "sensitivity", + DictionaryFieldKey::AllowedValues => "allowed values", + DictionaryFieldKey::Example => "example", + DictionaryFieldKey::Owner => "owner", + DictionaryFieldKey::Notes => "notes", + } + } +} + +/// Canonical (trimmed, non-blank) string value of one field, or `None` when it +/// carries no real documentation. Drives presence, equality and conflict +/// display uniformly across the differently-typed fields. +fn value_of(f: &DictionaryField, key: DictionaryFieldKey) -> Option { + fn text(o: &Option) -> Option { + o.as_deref() + .map(str::trim) + .filter(|v| !v.is_empty()) + .map(str::to_string) + } + match key { + DictionaryFieldKey::DisplayName => text(&f.display_name), + DictionaryFieldKey::Description => text(&f.description), + DictionaryFieldKey::Role => f.role.map(|r| r.label().to_string()), + DictionaryFieldKey::Unit => text(&f.unit), + DictionaryFieldKey::Source => text(&f.source), + DictionaryFieldKey::Sensitivity => f.sensitivity.map(|s| s.label().to_string()), + DictionaryFieldKey::AllowedValues => { + let vals: Vec<&str> = f + .allowed_values + .iter() + .map(|v| v.trim()) + .filter(|v| !v.is_empty()) + .collect(); + (!vals.is_empty()).then(|| vals.join(", ")) + } + DictionaryFieldKey::Example => text(&f.example), + DictionaryFieldKey::Owner => text(&f.owner), + DictionaryFieldKey::Notes => text(&f.notes), + } +} + +/// Copy one field's raw (typed) value from `src` into `dst`. +fn copy_field(dst: &mut DictionaryField, src: &DictionaryField, key: DictionaryFieldKey) { + match key { + DictionaryFieldKey::DisplayName => dst.display_name = src.display_name.clone(), + DictionaryFieldKey::Description => dst.description = src.description.clone(), + DictionaryFieldKey::Role => dst.role = src.role, + DictionaryFieldKey::Unit => dst.unit = src.unit.clone(), + DictionaryFieldKey::Source => dst.source = src.source.clone(), + DictionaryFieldKey::Sensitivity => dst.sensitivity = src.sensitivity, + DictionaryFieldKey::AllowedValues => dst.allowed_values = src.allowed_values.clone(), + DictionaryFieldKey::Example => dst.example = src.example.clone(), + DictionaryFieldKey::Owner => dst.owner = src.owner.clone(), + DictionaryFieldKey::Notes => dst.notes = src.notes.clone(), + } +} + +/// Whether `field` populates `key` with a real value. +pub fn field_present(field: &DictionaryField, key: DictionaryFieldKey) -> bool { + value_of(field, key).is_some() +} + +// --------------------------------------------------------------------------- +// Per-document container +// --------------------------------------------------------------------------- + +/// A document's data dictionary: per-column entries keyed by stable column ID. +/// Columns without an entry are simply undocumented. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Dictionary { + pub fields: BTreeMap, +} + +impl Dictionary { + pub fn field(&self, column_id: &str) -> Option<&DictionaryField> { + self.fields.get(column_id) + } + + /// Insert or replace an entry (keyed by its own `column_id`). + pub fn set(&mut self, field: DictionaryField) { + self.fields.insert(field.column_id.clone(), field); + } + + /// Remove an entry; returns whether one was present. + pub fn remove(&mut self, column_id: &str) -> bool { + self.fields.remove(column_id).is_some() + } + + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } + + pub fn len(&self) -> usize { + self.fields.len() + } +} + +/// Reject an entry with no column ID before it reaches the model. +pub fn validate_field(field: &DictionaryField) -> AppResult<()> { + if field.column_id.trim().is_empty() { + return Err(AppError::invalid("dictionary entry has no columnId")); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Column context (technical name + inferred F31 type) +// --------------------------------------------------------------------------- + +/// The technical name + inferred F31 type of a documented column in the +/// CURRENT document (falling back to the entry's own name when the column has +/// been deleted). +struct ColumnContext { + /// Technical header when present, else the entry's display name / ID. + name: String, + logical_type: Option, +} + +fn column_context( + doc: &Document, + column_id: &str, + field: Option<&DictionaryField>, +) -> ColumnContext { + let position = doc.column_ids().iter().position(|id| id == column_id); + let name = match position { + Some(pos) => doc.headers()[pos].clone(), + None => field + .and_then(|f| f.display_name.clone()) + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| column_id.to_string()), + }; + let logical_type = position + .and_then(|_| doc.schema().column(column_id)) + .map(|s| s.logical_type); + ColumnContext { name, logical_type } +} + +// --------------------------------------------------------------------------- +// Editor view (every column, prefilled; plus orphans) +// --------------------------------------------------------------------------- + +/// One column in the dictionary editor: technical name + inferred type +/// prefilled, the stored entry when documented (an empty prefill otherwise). +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryEntryView { + pub column_id: String, + /// Current header — the technical name shown/prefilled in the editor. + pub column_name: String, + pub column_index: usize, + /// Declared/inferred logical type from F31, when a schema entry exists. + #[serde(skip_serializing_if = "Option::is_none")] + pub logical_type: Option, + pub field: DictionaryField, + /// Whether the user has actually documented this column. + pub documented: bool, +} + +/// A documented entry whose column no longer exists (reported after a delete; +/// kept until explicitly discarded, and re-attached if the column returns). +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OrphanEntry { + pub column_id: String, + /// Best-effort label (display name, else the column ID). + pub label: String, + pub field: DictionaryField, +} + +/// The full dictionary surface for the front end: the searchable editor rows +/// (one per current column) plus any orphaned entries. `dictionaryRevision` +/// is the metadata revision (moves on documentation edits only); `revision` +/// is the ordinary document revision (which those edits never move). +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryView { + pub dictionary_revision: u64, + pub revision: u64, + pub entries: Vec, + pub orphans: Vec, +} + +/// Snapshot the dictionary for the editor: one row per current column (stored +/// entry or an empty prefill), plus orphaned entries. +pub fn view(doc: &Document) -> DictionaryView { + let entries = doc + .column_ids() + .iter() + .enumerate() + .map(|(idx, id)| { + let stored = doc.dictionary().field(id); + let ctx = column_context(doc, id, stored); + DictionaryEntryView { + column_id: id.clone(), + column_name: ctx.name, + column_index: idx, + logical_type: ctx.logical_type, + field: stored + .cloned() + .unwrap_or_else(|| DictionaryField::empty(id.clone())), + documented: stored.is_some_and(DictionaryField::is_documented), + } + }) + .collect(); + + DictionaryView { + dictionary_revision: doc.dictionary_revision(), + revision: doc.revision(), + entries, + orphans: orphans(doc), + } +} + +/// Documented entries whose column ID is no longer present in the document. +pub fn orphans(doc: &Document) -> Vec { + let live: HashSet<&str> = doc.column_ids().iter().map(String::as_str).collect(); + doc.dictionary() + .fields + .iter() + .filter(|(id, _)| !live.contains(id.as_str())) + .map(|(id, field)| OrphanEntry { + column_id: id.clone(), + label: field + .display_name + .clone() + .filter(|s| !s.trim().is_empty()) + .unwrap_or_else(|| id.clone()), + field: field.clone(), + }) + .collect() +} + +// --------------------------------------------------------------------------- +// Exports: versioned JSON, Markdown, CSV +// --------------------------------------------------------------------------- + +/// One entry in the export envelope: the documentation plus the technical +/// column name captured at export time, so a later import can remap by name +/// when the target document's IDs differ. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryExportEntry { + /// Technical header at export time (the name-remap key on import). + pub column_name: String, + #[serde(flatten)] + pub field: DictionaryField, +} + +/// Versioned import/export envelope: `{ "version": 1, "entries": [...] }`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryExport { + pub version: u32, + pub entries: Vec, +} + +/// An export row: the entry with its resolved current context. +struct EntryRow { + column_id: String, + field: DictionaryField, + name: String, + logical_type: Option, + orphan: bool, +} + +/// Documented entries in current-column order, followed by orphans (sorted by +/// ID). Only actually-documented entries are emitted. +fn entry_rows(doc: &Document) -> Vec { + let mut rows = Vec::new(); + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); + for id in doc.column_ids() { + if let Some(field) = doc.dictionary().field(id) { + seen.insert(id.as_str()); + let ctx = column_context(doc, id, Some(field)); + rows.push(EntryRow { + column_id: id.clone(), + field: field.clone(), + name: ctx.name, + logical_type: ctx.logical_type, + orphan: false, + }); + } + } + for (id, field) in &doc.dictionary().fields { + if !seen.contains(id.as_str()) { + let ctx = column_context(doc, id, Some(field)); + rows.push(EntryRow { + column_id: id.clone(), + field: field.clone(), + name: ctx.name, + logical_type: ctx.logical_type, + orphan: true, + }); + } + } + rows +} + +/// Build the export envelope in current-column order (names refreshed from the +/// live headers; orphans last). +pub fn build_export(doc: &Document) -> DictionaryExport { + let entries = entry_rows(doc) + .into_iter() + .map(|row| DictionaryExportEntry { + column_name: row.name, + field: row.field, + }) + .collect(); + DictionaryExport { + version: DICTIONARY_VERSION, + entries, + } +} + +/// The three documentation export formats. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DictionaryFormat { + Json, + Markdown, + Csv, +} + +/// Render the dictionary in the requested format. +pub fn export_as(doc: &Document, format: DictionaryFormat) -> AppResult { + match format { + DictionaryFormat::Json => export_json(doc), + DictionaryFormat::Markdown => Ok(export_markdown(doc)), + DictionaryFormat::Csv => export_csv(doc), + } +} + +/// Serialize the dictionary to pretty, versioned JSON. +pub fn export_json(doc: &Document) -> AppResult { + serde_json::to_string_pretty(&build_export(doc)) + .map_err(|e| AppError::invalid(format!("could not serialize dictionary: {e}"))) +} + +fn logical_type_label(lt: LogicalType) -> &'static str { + match lt { + LogicalType::Text => "text", + LogicalType::Integer => "integer", + LogicalType::Decimal => "decimal", + LogicalType::Float => "float", + LogicalType::Boolean => "boolean", + LogicalType::Date => "date", + LogicalType::Datetime => "datetime", + LogicalType::Uuid => "uuid", + LogicalType::Json => "json", + } +} + +/// Markdown documentation: one section per documented column, every field +/// present (blank fields shown as an em dash so the section is complete). +pub fn export_markdown(doc: &Document) -> String { + let mut out = String::new(); + out.push_str("# Data dictionary\n\n"); + let file = doc.meta().file_name; + out.push_str(&format!("Source: {file}\n\n")); + + let rows = entry_rows(doc); + if rows.is_empty() { + out.push_str("_No columns documented yet._\n"); + return out; + } + + for row in rows { + let heading = if row.orphan { + format!("## {} (`{}`) — orphaned\n\n", row.name, row.column_id) + } else { + format!("## {} (`{}`)\n\n", row.name, row.column_id) + }; + out.push_str(&heading); + out.push_str(&format!("- **Column ID:** {}\n", row.column_id)); + out.push_str(&format!("- **Technical name:** {}\n", md_cell(&row.name))); + let ty = row + .logical_type + .map(logical_type_label) + .unwrap_or("—") + .to_string(); + out.push_str(&format!("- **Logical type (F31):** {ty}\n")); + for key in ALL_FIELD_KEYS { + let value = value_of(&row.field, key).unwrap_or_else(|| "—".to_string()); + out.push_str(&format!( + "- **{}:** {}\n", + capitalize(key.label()), + md_cell(&value) + )); + } + out.push('\n'); + } + out +} + +/// Escape the characters that would break a Markdown list line. +fn md_cell(value: &str) -> String { + value.replace('\n', " ").replace('|', "\\|") +} + +fn capitalize(s: &str) -> String { + let mut chars = s.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } +} + +/// CSV documentation: one row per documented column, every field a column. +pub fn export_csv(doc: &Document) -> AppResult { + let mut wtr = csv::Writer::from_writer(Vec::new()); + wtr.write_record([ + "columnId", + "columnName", + "logicalType", + "displayName", + "description", + "role", + "unit", + "source", + "sensitivity", + "allowedValues", + "example", + "owner", + "notes", + "orphaned", + ])?; + for row in entry_rows(doc) { + let cell = |key: DictionaryFieldKey| value_of(&row.field, key).unwrap_or_default(); + wtr.write_record([ + row.column_id.clone(), + row.name.clone(), + row.logical_type + .map(logical_type_label) + .unwrap_or("") + .to_string(), + cell(DictionaryFieldKey::DisplayName), + cell(DictionaryFieldKey::Description), + cell(DictionaryFieldKey::Role), + cell(DictionaryFieldKey::Unit), + cell(DictionaryFieldKey::Source), + cell(DictionaryFieldKey::Sensitivity), + // Allowed values as a semicolon list so a comma value stays intact. + row.field + .allowed_values + .iter() + .map(|v| v.trim()) + .filter(|v| !v.is_empty()) + .collect::>() + .join("; "), + cell(DictionaryFieldKey::Example), + cell(DictionaryFieldKey::Owner), + cell(DictionaryFieldKey::Notes), + if row.orphan { "true" } else { "false" }.to_string(), + ])?; + } + let bytes = wtr + .into_inner() + .map_err(|e| AppError::invalid(format!("could not serialize dictionary CSV: {e}")))?; + String::from_utf8(bytes) + .map_err(|e| AppError::invalid(format!("dictionary CSV was not valid UTF-8: {e}"))) +} + +// --------------------------------------------------------------------------- +// Import merge engine +// --------------------------------------------------------------------------- + +/// How incoming entries are matched to current columns. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MergeMatchBy { + /// Only by stable column ID. + ColumnId, + /// Only by (case-insensitive) technical column name. + ColumnName, + /// Prefer the column ID; fall back to the name when the ID is absent here. + #[default] + Auto, +} + +/// Parse a versioned dictionary JSON file (version probed first so an +/// incompatible future format fails with the version message, not a shape one). +pub fn parse_import(json: &str) -> AppResult { + #[derive(Deserialize)] + struct VersionProbe { + version: u32, + } + let probe: VersionProbe = serde_json::from_str(json) + .map_err(|e| AppError::invalid(format!("invalid dictionary JSON: {e}")))?; + if probe.version != DICTIONARY_VERSION { + return Err(AppError::invalid(format!( + "unsupported dictionary version {} (this build reads version {DICTIONARY_VERSION})", + probe.version + ))); + } + serde_json::from_str(json) + .map_err(|e| AppError::invalid(format!("invalid dictionary JSON: {e}"))) +} + +/// A single field-level disagreement between an existing entry and an incoming +/// one, requiring explicit resolution before the import can replace anything. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FieldConflict { + pub column_id: String, + /// Current technical name, for display. + pub column_name: String, + pub field: DictionaryFieldKey, + /// Existing value (display form). + pub existing: String, + /// Incoming value (display form). + pub incoming: String, +} + +/// The plan a `preview_dictionary_import` produces: what a merge would do, and +/// which conflicts block it. Computed against `dictionaryRevision`, which the +/// apply echoes back and is rejected if it has since moved. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MergePlan { + pub dictionary_revision: u64, + pub match_by: MergeMatchBy, + /// Number of imported entries matched to a current column. + pub matched_columns: usize, + /// Column IDs that would gain a brand-new entry. + pub new_entries: Vec, + /// Field additions that apply with no conflict (existing value was blank). + pub clean_additions: usize, + /// Disagreements needing explicit resolution. + pub conflicts: Vec, + /// Imported entries (by name/ID label) that matched no current column. + pub unmatched: Vec, +} + +/// Which side of a conflict wins. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ConflictChoice { + KeepExisting, + TakeIncoming, +} + +/// One explicit per-field resolution. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FieldResolution { + pub column_id: String, + pub field: DictionaryFieldKey, + pub choice: ConflictChoice, +} + +/// How the import resolves conflicts. `PerField` MUST cover every reported +/// conflict; a missing one fails the apply (conflicts are never silently +/// dropped). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum MergeResolution { + KeepAllExisting, + TakeAllIncoming, + PerField { resolutions: Vec }, +} + +impl MergeResolution { + /// The choice for one conflict, or `None` when unresolved (only possible + /// under `PerField`). + fn choice_for(&self, column_id: &str, field: DictionaryFieldKey) -> Option { + match self { + MergeResolution::KeepAllExisting => Some(ConflictChoice::KeepExisting), + MergeResolution::TakeAllIncoming => Some(ConflictChoice::TakeIncoming), + MergeResolution::PerField { resolutions } => resolutions + .iter() + .find(|r| r.column_id == column_id && r.field == field) + .map(|r| r.choice), + } + } +} + +enum MergeMode<'a> { + /// Report conflicts; make no choices (dry run). + Plan, + /// Apply choices from a resolution. + Apply(&'a MergeResolution), +} + +/// Accumulated results of a merge pass. +struct MergeStats { + matched: usize, + new_entries: Vec, + updated_entries: usize, + fields_added: usize, + conflicts: Vec, + resolved: usize, + unresolved: Vec, + unmatched: Vec, +} + +/// Outcome of resolving one imported entry to a current column. +enum Target { + /// Resolved to exactly one current column ID. + Matched(String), + /// No current column matched. + Unmatched, + /// The name matched more than one current column. Documents do not enforce + /// unique headers (a source CSV or an in-app rename can duplicate one), so + /// rather than silently collapse every same-named entry onto the FIRST + /// column — misattributing documentation with no signal — the entry is + /// reported and left unmatched for the user to disambiguate (e.g. by ID). + Ambiguous { name: String, count: usize }, +} + +/// The technical name an import entry matches on: its captured column name, or +/// its display name as a fallback. +fn match_name(entry: &DictionaryExportEntry) -> &str { + if entry.column_name.trim().is_empty() { + entry.field.display_name.as_deref().unwrap_or("").trim() + } else { + entry.column_name.trim() + } +} + +/// Current column IDs whose header equals `name` (case-insensitive, trimmed). +fn columns_named(doc: &Document, name: &str) -> Vec { + let ids = doc.column_ids(); + doc.headers() + .iter() + .enumerate() + .filter(|(_, h)| h.trim().eq_ignore_ascii_case(name)) + .map(|(i, _)| ids[i].clone()) + .collect() +} + +/// Resolve one imported entry to a current column ID under `match_by`. +fn resolve_target(doc: &Document, entry: &DictionaryExportEntry, match_by: MergeMatchBy) -> Target { + let by_id = || { + let id = entry.field.column_id.trim(); + (!id.is_empty() && doc.column_ids().iter().any(|c| c == id)).then(|| id.to_string()) + }; + let by_name = || { + let name = match_name(entry); + if name.is_empty() { + return Target::Unmatched; + } + let mut ids = columns_named(doc, name); + match ids.len() { + 0 => Target::Unmatched, + 1 => Target::Matched(ids.pop().expect("len checked")), + n => Target::Ambiguous { + name: name.to_string(), + count: n, + }, + } + }; + match match_by { + MergeMatchBy::ColumnId => by_id().map_or(Target::Unmatched, Target::Matched), + MergeMatchBy::ColumnName => by_name(), + // Prefer an exact ID match (always unambiguous); fall back to the name + // only when the ID is absent here. + MergeMatchBy::Auto => match by_id() { + Some(id) => Target::Matched(id), + None => by_name(), + }, + } +} + +/// Best-effort label for an imported entry that matched nothing. +fn unmatched_label(entry: &DictionaryExportEntry) -> String { + if !entry.column_name.trim().is_empty() { + entry.column_name.clone() + } else if let Some(name) = entry + .field + .display_name + .as_deref() + .filter(|s| !s.trim().is_empty()) + { + name.to_string() + } else { + entry.field.column_id.clone() + } +} + +/// Core merge pass shared by plan and apply. `working` starts from the current +/// dictionary and each imported entry is merged into it cumulatively. +fn run_merge( + doc: &Document, + imported: &DictionaryExport, + match_by: MergeMatchBy, + mode: &MergeMode<'_>, +) -> (Dictionary, MergeStats) { + let mut working = doc.dictionary().clone(); + let mut stats = MergeStats { + matched: 0, + new_entries: Vec::new(), + updated_entries: 0, + fields_added: 0, + conflicts: Vec::new(), + resolved: 0, + unresolved: Vec::new(), + unmatched: Vec::new(), + }; + + for entry in &imported.entries { + let target_id = match resolve_target(doc, entry, match_by) { + Target::Matched(id) => id, + Target::Unmatched => { + stats.unmatched.push(unmatched_label(entry)); + continue; + } + Target::Ambiguous { name, count } => { + stats.unmatched.push(format!( + "{name} (ambiguous — {count} columns share this name; import by column ID)" + )); + continue; + } + }; + stats.matched += 1; + let column_name = column_context(doc, &target_id, working.field(&target_id)).name; + + let original = working.field(&target_id).cloned(); + let mut base = original + .clone() + .unwrap_or_else(|| DictionaryField::empty(target_id.clone())); + // The merged entry always keys on the TARGET column's ID. + base.column_id = target_id.clone(); + + for key in ALL_FIELD_KEYS { + let Some(incoming) = value_of(&entry.field, key) else { + continue; // incoming blank — keep existing + }; + match value_of(&base, key) { + None => { + // Clean addition into a previously-blank field. + copy_field(&mut base, &entry.field, key); + stats.fields_added += 1; + } + Some(existing) if existing == incoming => {} // identical — no-op + Some(existing) => { + let conflict = FieldConflict { + column_id: target_id.clone(), + column_name: column_name.clone(), + field: key, + existing, + incoming, + }; + stats.conflicts.push(conflict.clone()); + match mode { + MergeMode::Plan => stats.unresolved.push(conflict), + MergeMode::Apply(resolution) => { + match resolution.choice_for(&target_id, key) { + Some(ConflictChoice::TakeIncoming) => { + copy_field(&mut base, &entry.field, key); + stats.resolved += 1; + } + Some(ConflictChoice::KeepExisting) => stats.resolved += 1, + None => stats.unresolved.push(conflict), + } + } + } + } + } + } + + // Only store an entry that carries real documentation. + if base.is_documented() { + let changed = original.as_ref() != Some(&base); + working.set(base); + match original { + None => stats.new_entries.push(target_id), + Some(_) if changed => stats.updated_entries += 1, + Some(_) => {} + } + } + } + + (working, stats) +} + +/// Dry-run a merge: report every conflict and what would change, without +/// touching the document. +pub fn plan_merge( + doc: &Document, + imported: &DictionaryExport, + match_by: MergeMatchBy, +) -> MergePlan { + let (_working, stats) = run_merge(doc, imported, match_by, &MergeMode::Plan); + MergePlan { + dictionary_revision: doc.dictionary_revision(), + match_by, + matched_columns: stats.matched, + new_entries: stats.new_entries, + clean_additions: stats.fields_added, + conflicts: stats.conflicts, + unmatched: stats.unmatched, + } +} + +/// Result of applying an import merge (the new dictionary + a summary). +pub struct MergeApplied { + pub dictionary: Dictionary, + pub matched_columns: usize, + pub new_entries: usize, + pub updated_entries: usize, + pub fields_added: usize, + pub conflicts_resolved: usize, + pub unmatched: Vec, +} + +/// Apply an import merge under an explicit resolution. Fails (without changing +/// anything) when the resolution leaves any conflict unresolved. +pub fn apply_merge( + doc: &Document, + imported: &DictionaryExport, + match_by: MergeMatchBy, + resolution: &MergeResolution, +) -> AppResult { + let (working, stats) = run_merge(doc, imported, match_by, &MergeMode::Apply(resolution)); + if !stats.unresolved.is_empty() { + let first = &stats.unresolved[0]; + return Err(AppError::invalid(format!( + "{} documentation conflict(s) still need explicit resolution (e.g. column \"{}\" field \"{}\")", + stats.unresolved.len(), + first.column_name, + first.field.label() + ))); + } + Ok(MergeApplied { + dictionary: working, + matched_columns: stats.matched, + new_entries: stats.new_entries.len(), + updated_entries: stats.updated_entries, + fields_added: stats.fields_added, + conflicts_resolved: stats.resolved, + unmatched: stats.unmatched, + }) +} + +/// The serializable outcome returned to the front end after an import. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DictionaryImportOutcome { + pub matched_columns: usize, + pub new_entries: usize, + pub updated_entries: usize, + pub fields_added: usize, + pub conflicts_resolved: usize, + pub unmatched: Vec, + pub view: DictionaryView, +} + +// --------------------------------------------------------------------------- +// F08 profile hook: required-documentation rule + checker +// --------------------------------------------------------------------------- + +/// A file-profile (F08) rule requiring certain dictionary fields to be +/// populated. `columns` names the technical columns it applies to; empty means +/// every column in the document. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequiredDocumentation { + #[serde(default)] + pub columns: Vec, + pub fields: Vec, +} + +/// One column missing required documentation. +pub struct DocumentationGap { + pub column_id: String, + pub column_name: String, + pub missing: Vec, +} + +/// Evaluate the required-documentation rules against the document's dictionary. +/// Returns one gap per (column, rule) that leaves a required field blank. +/// Columns a rule names that are not present are skipped (other profile rules +/// report missing columns). +pub fn documentation_gaps( + doc: &Document, + rules: &[RequiredDocumentation], +) -> Vec { + let mut gaps = Vec::new(); + for rule in rules { + if rule.fields.is_empty() { + continue; + } + // Resolve target column positions. A required-doc rule applies to EVERY + // column sharing a named header, not just the first — headers are not + // unique — and duplicate positions are collapsed so a column is reported + // at most once per rule. + let targets: Vec = if rule.columns.is_empty() { + (0..doc.column_ids().len()).collect() + } else { + let mut positions: Vec = rule + .columns + .iter() + .flat_map(|name| { + let name = name.trim(); + doc.headers() + .iter() + .enumerate() + .filter(move |(_, h)| h.trim().eq_ignore_ascii_case(name)) + .map(|(i, _)| i) + }) + .collect(); + positions.sort_unstable(); + positions.dedup(); + positions + }; + for pos in targets { + let column_id = &doc.column_ids()[pos]; + let entry = doc.dictionary().field(column_id); + let missing: Vec = rule + .fields + .iter() + .copied() + .filter(|&key| match entry { + Some(f) => !field_present(f, key), + None => true, + }) + .collect(); + if !missing.is_empty() { + gaps.push(DocumentationGap { + column_id: column_id.clone(), + column_name: doc.headers()[pos].clone(), + missing, + }); + } + } + } + gaps +} + +// --------------------------------------------------------------------------- +// F28 PII hook: sensitive columns +// --------------------------------------------------------------------------- + +/// A column the dictionary classifies as confidential or restricted. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SensitiveColumn { + pub column: usize, + pub column_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, + pub sensitivity: Sensitivity, +} + +/// Columns whose declared sensitivity makes them PII-relevant regardless of +/// pattern hits. Consumed by the F28 scan preflight. Ordered by column index. +pub fn sensitive_columns(doc: &Document) -> Vec { + doc.column_ids() + .iter() + .enumerate() + .filter_map(|(idx, id)| { + let field = doc.dictionary().field(id)?; + let sensitivity = field.sensitivity?; + sensitivity.is_sensitive().then(|| SensitiveColumn { + column: idx, + column_id: id.clone(), + display_name: field.display_name.clone().filter(|s| !s.trim().is_empty()), + sensitivity, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parse::{parse, ParseSettings}; + + fn doc(csv: &str) -> Document { + let parsed = parse(csv.as_bytes(), &ParseSettings::default()).unwrap(); + Document::from_parsed(1, None, parsed, true) + } + + fn field(column_id: &str) -> DictionaryField { + DictionaryField::empty(column_id) + } + + fn export_of(entries: Vec) -> DictionaryExport { + DictionaryExport { + version: DICTIONARY_VERSION, + entries, + } + } + + fn export_entry( + column_id: &str, + column_name: &str, + f: impl FnOnce(&mut DictionaryField), + ) -> DictionaryExportEntry { + let mut fld = field(column_id); + f(&mut fld); + DictionaryExportEntry { + column_name: column_name.to_string(), + field: fld, + } + } + + // ----- storage: no-dirty, rename survival, orphans --------------------- + + #[test] + fn dictionary_edits_never_dirty_or_move_the_document() { + let mut d = doc("a,b\n1,2\n"); + let rev = d.revision(); + let dict_rev = d.dictionary_revision(); + assert!(!d.is_dirty()); + + let mut f = field(&d.column_ids()[0].clone()); + f.description = Some("the primary key".into()); + d.set_dictionary_field(f); + + assert_eq!(d.revision(), rev, "document revision must not move"); + assert!(!d.is_dirty(), "documentation edits never dirty the source"); + assert_eq!( + d.dictionary_revision(), + dict_rev + 1, + "the metadata revision moves instead" + ); + } + + #[test] + fn entry_survives_a_rename_by_stable_id() { + let mut d = doc("email,amount\nx,1\n"); + let id = d.column_ids()[0].clone(); + let mut f = field(&id); + f.description = Some("customer email".into()); + d.set_dictionary_field(f); + + d.rename_column(0, "contact_email".into()).unwrap(); + + let entry = d.dictionary().field(&id).expect("entry preserved by ID"); + assert_eq!(entry.description.as_deref(), Some("customer email")); + // The view now shows the NEW technical name against the same entry. + let v = view(&d); + assert_eq!(v.entries[0].column_name, "contact_email"); + assert!(v.entries[0].documented); + assert!(orphans(&d).is_empty()); + } + + #[test] + fn deleting_a_column_reports_an_orphan_and_keeps_the_entry() { + let mut d = doc("a,b,c\n1,2,3\n"); + let id_b = d.column_ids()[1].clone(); + let mut f = field(&id_b); + f.description = Some("the middle column".into()); + d.set_dictionary_field(f); + + d.delete_columns(vec![1]).unwrap(); + + let orphs = orphans(&d); + assert_eq!(orphs.len(), 1); + assert_eq!(orphs[0].column_id, id_b); + assert_eq!( + orphs[0].field.description.as_deref(), + Some("the middle column") + ); + // The editor view no longer lists it as a live column. + assert!(view(&d).entries.iter().all(|e| e.column_id != id_b)); + + // Undo restores the column: the entry re-attaches (no longer orphaned). + d.undo().unwrap(); + assert!(orphans(&d).is_empty()); + assert!(d.dictionary().field(&id_b).is_some()); + + // Redo re-orphans it; discarding removes it explicitly. + d.redo().unwrap(); + assert_eq!(orphans(&d).len(), 1); + assert!(d.remove_dictionary_field(&id_b)); + assert!(orphans(&d).is_empty()); + } + + // ----- merge matrix: id / name / conflict ------------------------------ + + #[test] + fn merge_by_column_id_adds_and_flags_conflicts() { + let mut d = doc("a,b\n1,2\n"); + let id_a = d.column_ids()[0].clone(); + // Existing docs on column a: description set, owner blank. + let mut existing = field(&id_a); + existing.description = Some("existing description".into()); + d.set_dictionary_field(existing); + + // Incoming: same column ID, a CONFLICTING description + a NEW owner. + let incoming = export_of(vec![export_entry(&id_a, "a", |f| { + f.description = Some("incoming description".into()); + f.owner = Some("data-team".into()); + })]); + + let plan = plan_merge(&d, &incoming, MergeMatchBy::ColumnId); + assert_eq!(plan.matched_columns, 1); + assert_eq!(plan.clean_additions, 1, "owner is a clean addition"); + assert_eq!(plan.conflicts.len(), 1, "description conflicts"); + assert_eq!(plan.conflicts[0].field, DictionaryFieldKey::Description); + assert_eq!(plan.conflicts[0].existing, "existing description"); + assert_eq!(plan.conflicts[0].incoming, "incoming description"); + assert!(plan.unmatched.is_empty()); + + // Applying without resolving the conflict is impossible under PerField. + let unresolved = apply_merge( + &d, + &incoming, + MergeMatchBy::ColumnId, + &MergeResolution::PerField { + resolutions: vec![], + }, + ); + assert!(unresolved.is_err(), "unresolved conflicts block the apply"); + + // Resolve by taking the incoming value; owner still merges cleanly. + let applied = apply_merge( + &d, + &incoming, + MergeMatchBy::ColumnId, + &MergeResolution::PerField { + resolutions: vec![FieldResolution { + column_id: id_a.clone(), + field: DictionaryFieldKey::Description, + choice: ConflictChoice::TakeIncoming, + }], + }, + ) + .unwrap(); + assert_eq!(applied.conflicts_resolved, 1); + assert_eq!(applied.fields_added, 1); + let merged = applied.dictionary.field(&id_a).unwrap(); + assert_eq!(merged.description.as_deref(), Some("incoming description")); + assert_eq!(merged.owner.as_deref(), Some("data-team")); + } + + #[test] + fn keep_existing_resolution_preserves_current_values() { + let mut d = doc("a\n1\n"); + let id_a = d.column_ids()[0].clone(); + let mut existing = field(&id_a); + existing.description = Some("keep me".into()); + d.set_dictionary_field(existing); + + let incoming = export_of(vec![export_entry(&id_a, "a", |f| { + f.description = Some("overwrite me".into()); + })]); + let applied = apply_merge( + &d, + &incoming, + MergeMatchBy::ColumnId, + &MergeResolution::KeepAllExisting, + ) + .unwrap(); + assert_eq!( + applied + .dictionary + .field(&id_a) + .unwrap() + .description + .as_deref(), + Some("keep me") + ); + } + + #[test] + fn merge_by_mapped_name_when_ids_differ() { + // The document's IDs are c0/c1; the import carries foreign IDs but + // matching column NAMES. + let d = doc("email,amount\nx,1\n"); + let incoming = export_of(vec![ + export_entry("foreign-99", "email", |f| { + f.description = Some("the email".into()); + }), + export_entry("foreign-100", "amount", |f| { + f.owner = Some("finance".into()); + }), + ]); + + // By ID nothing matches; by name (or Auto) both do. + let by_id = plan_merge(&d, &incoming, MergeMatchBy::ColumnId); + assert_eq!(by_id.matched_columns, 0); + assert_eq!(by_id.unmatched.len(), 2); + + let applied = apply_merge( + &d, + &incoming, + MergeMatchBy::ColumnName, + &MergeResolution::KeepAllExisting, + ) + .unwrap(); + assert_eq!(applied.matched_columns, 2); + assert_eq!(applied.new_entries, 2); + // Entries land under the DOCUMENT's IDs, not the foreign ones. + assert_eq!( + applied + .dictionary + .field(&d.column_ids()[0]) + .unwrap() + .description + .as_deref(), + Some("the email") + ); + assert!(applied.dictionary.field("foreign-99").is_none()); + } + + #[test] + fn auto_prefers_id_then_falls_back_to_name() { + let d = doc("a,b\n1,2\n"); + let id_a = d.column_ids()[0].clone(); + let incoming = export_of(vec![ + // Matches column a by its real ID. + export_entry(&id_a, "renamed-header", |f| f.unit = Some("USD".into())), + // No such ID; matches column b by name. + export_entry("nope", "b", |f| f.unit = Some("kg".into())), + ]); + let plan = plan_merge(&d, &incoming, MergeMatchBy::Auto); + assert_eq!(plan.matched_columns, 2); + assert!(plan.unmatched.is_empty()); + } + + #[test] + fn merge_by_name_reports_ambiguous_duplicate_headers() { + // Headers are not unique — two columns share the name "email". An + // import entry that matches by name cannot be safely attributed to + // either, so it is reported as unmatched, NOT silently collapsed onto + // the first column. + let d = doc("email,email\n1,2\n"); + let incoming = export_of(vec![export_entry("foreign-1", "email", |f| { + f.description = Some("the email".into()); + })]); + + let plan = plan_merge(&d, &incoming, MergeMatchBy::ColumnName); + assert_eq!(plan.matched_columns, 0, "ambiguous name matches nothing"); + assert!(plan.new_entries.is_empty()); + assert_eq!(plan.unmatched.len(), 1); + assert!( + plan.unmatched[0].contains("ambiguous"), + "unmatched entry carries a reason: {:?}", + plan.unmatched[0] + ); + + // Applying attributes documentation to NEITHER column. + let applied = apply_merge( + &d, + &incoming, + MergeMatchBy::ColumnName, + &MergeResolution::KeepAllExisting, + ) + .unwrap(); + assert_eq!(applied.matched_columns, 0); + assert_eq!(applied.new_entries, 0); + assert!(applied.dictionary.field(&d.column_ids()[0]).is_none()); + assert!(applied.dictionary.field(&d.column_ids()[1]).is_none()); + assert_eq!(applied.unmatched.len(), 1); + } + + #[test] + fn case_variant_headers_are_also_ambiguous_by_name() { + // "Email" and "email" collide under case-insensitive name matching. + let d = doc("Email,email\n1,2\n"); + let incoming = export_of(vec![export_entry("foreign-1", "EMAIL", |f| { + f.owner = Some("data".into()); + })]); + let plan = plan_merge(&d, &incoming, MergeMatchBy::ColumnName); + assert_eq!(plan.matched_columns, 0); + assert_eq!(plan.unmatched.len(), 1); + } + + #[test] + fn auto_uses_id_even_when_the_name_is_ambiguous() { + // Duplicate headers, but the import carries the real ID of the SECOND + // column — Auto prefers the (unambiguous) ID and lands there exactly. + let d = doc("email,email\n1,2\n"); + let id1 = d.column_ids()[1].clone(); + let incoming = export_of(vec![export_entry(&id1, "email", |f| { + f.owner = Some("finance".into()); + })]); + let applied = apply_merge( + &d, + &incoming, + MergeMatchBy::Auto, + &MergeResolution::KeepAllExisting, + ) + .unwrap(); + assert_eq!(applied.matched_columns, 1); + assert_eq!( + applied.dictionary.field(&id1).unwrap().owner.as_deref(), + Some("finance") + ); + // The first same-named column is untouched. + assert!(applied.dictionary.field(&d.column_ids()[0]).is_none()); + } + + #[test] + fn identical_incoming_value_is_not_a_conflict() { + let mut d = doc("a\n1\n"); + let id_a = d.column_ids()[0].clone(); + let mut existing = field(&id_a); + existing.description = Some("same".into()); + d.set_dictionary_field(existing); + let incoming = export_of(vec![export_entry(&id_a, "a", |f| { + f.description = Some("same".into()); + })]); + let plan = plan_merge(&d, &incoming, MergeMatchBy::ColumnId); + assert!(plan.conflicts.is_empty()); + assert_eq!(plan.clean_additions, 0); + } + + // ----- export completeness: JSON round-trip, MD, CSV ------------------- + + fn fully_documented(d: &mut Document) -> String { + let id = d.column_ids()[0].clone(); + let mut f = field(&id); + f.display_name = Some("Customer Email".into()); + f.description = Some("primary contact email".into()); + f.role = Some(FieldRole::Dimension); + f.unit = Some("n/a".into()); + f.source = Some("CRM".into()); + f.sensitivity = Some(Sensitivity::Confidential); + f.allowed_values = vec!["a@x.com".into(), "b@x.com".into()]; + f.example = Some("a@x.com".into()); + f.owner = Some("data-team".into()); + f.notes = Some("deduplicated nightly".into()); + d.set_dictionary_field(f); + id + } + + #[test] + fn markdown_export_contains_every_documented_field() { + let mut d = doc("email,amount\nx,1\n"); + fully_documented(&mut d); + let md = export_markdown(&d); + for needle in [ + "Customer Email", + "primary contact email", + "dimension", + "CRM", + "confidential", + "a@x.com, b@x.com", + "data-team", + "deduplicated nightly", + ] { + assert!(md.contains(needle), "markdown missing {needle:?}:\n{md}"); + } + } + + #[test] + fn csv_export_contains_every_documented_field() { + let mut d = doc("email,amount\nx,1\n"); + fully_documented(&mut d); + let csv = export_csv(&d).unwrap(); + // Header row + one data row. + let mut reader = csv::ReaderBuilder::new().from_reader(csv.as_bytes()); + let headers = reader.headers().unwrap().clone(); + assert!(headers.iter().any(|h| h == "sensitivity")); + let row = reader.records().next().unwrap().unwrap(); + let get = |name: &str| { + let i = headers.iter().position(|h| h == name).unwrap(); + row.get(i).unwrap().to_string() + }; + assert_eq!(get("displayName"), "Customer Email"); + assert_eq!(get("description"), "primary contact email"); + assert_eq!(get("role"), "dimension"); + assert_eq!(get("source"), "CRM"); + assert_eq!(get("sensitivity"), "confidential"); + assert_eq!(get("allowedValues"), "a@x.com; b@x.com"); + assert_eq!(get("owner"), "data-team"); + assert_eq!(get("notes"), "deduplicated nightly"); + } + + #[test] + fn json_export_round_trips_and_reimports_by_id() { + let mut d = doc("email,amount\nx,1\n"); + let id = fully_documented(&mut d); + let json = export_json(&d).unwrap(); + let parsed = parse_import(&json).unwrap(); + assert_eq!(parsed.version, DICTIONARY_VERSION); + assert_eq!(parsed.entries.len(), 1); + assert_eq!(parsed.entries[0].column_name, "email"); + assert_eq!(parsed.entries[0].field.column_id, id); + assert_eq!( + parsed.entries[0].field.sensitivity, + Some(Sensitivity::Confidential) + ); + + // Re-importing onto a blank document reproduces the entry. + let blank = doc("email,amount\nx,1\n"); + let applied = apply_merge( + &blank, + &parsed, + MergeMatchBy::Auto, + &MergeResolution::KeepAllExisting, + ) + .unwrap(); + assert_eq!(applied.new_entries, 1); + assert_eq!( + applied + .dictionary + .field(&blank.column_ids()[0]) + .unwrap() + .description + .as_deref(), + Some("primary contact email") + ); + } + + #[test] + fn parse_import_rejects_unknown_version() { + let json = format!( + r#"{{"version": {}, "entries": []}}"#, + DICTIONARY_VERSION + 1 + ); + assert!(parse_import(&json).is_err()); + } + + // ----- profile hook ---------------------------------------------------- + + #[test] + fn documentation_gaps_flag_missing_required_fields() { + let mut d = doc("a,b\n1,2\n"); + // Column a documented with a description but no owner; b undocumented. + let mut f = field(&d.column_ids()[0].clone()); + f.description = Some("has a description".into()); + d.set_dictionary_field(f); + + let rules = vec![RequiredDocumentation { + columns: vec![], // all columns + fields: vec![DictionaryFieldKey::Description, DictionaryFieldKey::Owner], + }]; + let gaps = documentation_gaps(&d, &rules); + // a is missing owner; b is missing both. + assert_eq!(gaps.len(), 2); + let a = gaps.iter().find(|g| g.column_name == "a").unwrap(); + assert_eq!(a.missing, vec![DictionaryFieldKey::Owner]); + let b = gaps.iter().find(|g| g.column_name == "b").unwrap(); + assert_eq!( + b.missing, + vec![DictionaryFieldKey::Description, DictionaryFieldKey::Owner] + ); + + // Scoping the rule to a specific column limits the gaps. + let scoped = vec![RequiredDocumentation { + columns: vec!["a".into()], + fields: vec![DictionaryFieldKey::Description], + }]; + assert!( + documentation_gaps(&d, &scoped).is_empty(), + "a has a description" + ); + } + + #[test] + fn documentation_gaps_cover_every_column_sharing_a_name() { + // Two columns share the header "email"; a rule naming "email" must flag + // BOTH (not just the first) and report each at most once. + let d = doc("email,email\n1,2\n"); + let rules = vec![RequiredDocumentation { + columns: vec!["email".into()], + fields: vec![DictionaryFieldKey::Description], + }]; + let gaps = documentation_gaps(&d, &rules); + assert_eq!(gaps.len(), 2, "both same-named columns are flagged"); + let ids: HashSet<&str> = gaps.iter().map(|g| g.column_id.as_str()).collect(); + assert!(ids.contains(d.column_ids()[0].as_str())); + assert!(ids.contains(d.column_ids()[1].as_str())); + } + + // ----- PII hook -------------------------------------------------------- + + #[test] + fn sensitive_columns_selects_confidential_and_restricted() { + let mut d = doc("public_col,secret_col,top_col\n1,2,3\n"); + let ids: Vec = d.column_ids().to_vec(); + let mut p = field(&ids[0]); + p.sensitivity = Some(Sensitivity::Public); + d.set_dictionary_field(p); + let mut s = field(&ids[1]); + s.sensitivity = Some(Sensitivity::Confidential); + s.display_name = Some("Secret".into()); + d.set_dictionary_field(s); + let mut t = field(&ids[2]); + t.sensitivity = Some(Sensitivity::Restricted); + d.set_dictionary_field(t); + + let sensitive = sensitive_columns(&d); + assert_eq!(sensitive.len(), 2, "public is not flagged"); + assert_eq!(sensitive[0].column, 1); + assert_eq!(sensitive[0].sensitivity, Sensitivity::Confidential); + assert_eq!(sensitive[0].display_name.as_deref(), Some("Secret")); + assert_eq!(sensitive[1].column, 2); + assert_eq!(sensitive[1].sensitivity, Sensitivity::Restricted); + } + + #[test] + fn empty_field_is_not_stored_as_documented() { + let d = doc("a\n1\n"); + let id = d.column_ids()[0].clone(); + assert!(!field(&id).is_documented()); + let v = view(&d); + assert!(!v.entries[0].documented); + assert_eq!(v.entries[0].field.column_id, id); + } +} diff --git a/src-tauri/src/document.rs b/src-tauri/src/document.rs index f9e2004..71dd184 100644 --- a/src-tauri/src/document.rs +++ b/src-tauri/src/document.rs @@ -340,6 +340,16 @@ pub struct Document { /// their column's declared type. Bounded /// ([`crate::schema::MAX_SCHEMA_ISSUES`]), newest kept. schema_issues: Vec, + /// F38 data dictionary: human documentation per column keyed by stable + /// column ID. Like the schema it lives OUTSIDE the undo stack and is pure + /// metadata — editing it never rewrites cells and never makes the document + /// dirty. Entries survive renames/reorders (keyed by ID) and are kept when + /// a column is deleted (reported as orphaned until explicitly discarded). + dictionary: crate::dictionary::Dictionary, + /// Bumped on every dictionary mutation, independent of both the document + /// `revision` and `schema_revision`, so documentation edits never move the + /// dirty flag or invalidate data/schema previews. + dictionary_revision: u64, /// F17: where the document came from when opened out of an archive. /// Archive-backed documents have no `path` (no in-place saving); Save As /// clears this and turns them into ordinary file documents. @@ -418,6 +428,8 @@ impl Document { schema: crate::schema::DocumentSchema::default(), schema_revision: 0, schema_issues: Vec::new(), + dictionary: crate::dictionary::Dictionary::default(), + dictionary_revision: 0, backing: Backing::Memory, archive: None, archive_guard: None, @@ -466,6 +478,8 @@ impl Document { schema: crate::schema::DocumentSchema::default(), schema_revision: 0, schema_issues: Vec::new(), + dictionary: crate::dictionary::Dictionary::default(), + dictionary_revision: 0, backing: Backing::Memory, archive: None, archive_guard: None, @@ -521,6 +535,8 @@ impl Document { schema: crate::schema::DocumentSchema::default(), schema_revision: 0, schema_issues: Vec::new(), + dictionary: crate::dictionary::Dictionary::default(), + dictionary_revision: 0, backing: Backing::Indexed(handle), archive: None, archive_guard: None, @@ -632,6 +648,64 @@ impl Document { self.schema_issues.clear(); } + // ----- data dictionary (F38) ------------------------------------------- + + /// The document's data dictionary (per-column documentation keyed by + /// stable column ID). Reading it never touches cells. + pub fn dictionary(&self) -> &crate::dictionary::Dictionary { + &self.dictionary + } + + /// Bumped on every dictionary mutation; independent of [`Document::revision`] + /// and [`Document::schema_revision`] so documentation edits never dirty the + /// document or invalidate data/schema previews. + pub fn dictionary_revision(&self) -> u64 { + self.dictionary_revision + } + + /// Insert or replace one column's dictionary entry. + pub fn set_dictionary_field(&mut self, field: crate::dictionary::DictionaryField) { + self.dictionary.set(field); + self.dictionary_revision += 1; + } + + /// Remove one column's dictionary entry (e.g. discarding an orphan). + /// Returns whether an entry was present. + pub fn remove_dictionary_field(&mut self, column_id: &str) -> bool { + let removed = self.dictionary.remove(column_id); + self.dictionary_revision += 1; + removed + } + + /// Replace the whole dictionary (import / merge apply / bulk discard). + pub fn set_dictionary(&mut self, dictionary: crate::dictionary::Dictionary) { + self.dictionary = dictionary; + self.dictionary_revision += 1; + } + + /// Carry the dictionary across a whole-`Document` replacement (reparse / + /// reindex): entries are keyed by stable IDs, which restart positionally, + /// so they re-attach to the same columns the original open assigned. + pub fn inherit_dictionary(&mut self, prev: &Document) { + self.dictionary = prev.dictionary.clone(); + self.dictionary_revision = prev.dictionary_revision + 1; + } + + /// Guard a dictionary-dependent deferred operation (F38: a merge/import + /// resolution prepared against a snapshot): fail with + /// [`AppError::StaleDictionaryRevision`] when the dictionary moved since + /// `expected` was captured. Independent of the data and schema revisions. + pub fn check_dictionary_revision(&self, expected: u64) -> AppResult<()> { + if self.dictionary_revision == expected { + Ok(()) + } else { + Err(AppError::StaleDictionaryRevision { + expected, + actual: self.dictionary_revision, + }) + } + } + /// The in-memory row slice. EDITABLE backing only: for indexed documents /// this is always empty — mutation paths must gate with /// [`Document::ensure_editable`] first, and read paths must go through diff --git a/src-tauri/src/error.rs b/src-tauri/src/error.rs index df879cd..7788cec 100644 --- a/src-tauri/src/error.rs +++ b/src-tauri/src/error.rs @@ -42,6 +42,13 @@ pub enum AppError { #[error("stale schema: the column schema changed since this operation was prepared (expected schema revision {expected}, document is at {actual})")] StaleSchemaRevision { expected: u64, actual: u64 }, + /// A dictionary-dependent deferred operation (F38 import/merge resolution) + /// was prepared against an older dictionary revision. The imported + /// documentation must be re-planned against the current dictionary before + /// it can be applied. + #[error("stale dictionary: the data dictionary changed since this operation was prepared (expected dictionary revision {expected}, document is at {actual})")] + StaleDictionaryRevision { expected: u64, actual: u64 }, + /// A mutation was attempted on a document opened in indexed read-only /// mode (F10). Convert it to editable first. #[error( diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index ea59443..fa9ad68 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -14,6 +14,11 @@ mod delimiter; mod derived; mod diagnostics; mod dialect; +/// Public like [`job`]: the F38 data-dictionary model (per-column +/// documentation keyed by stable column ID, versioned import/export, the merge +/// engine and the profile/PII integration hooks) is a stable internal API +/// consumed by the profile and PII modules and the test harness. +pub mod dictionary; mod document; mod dto; mod encoding; @@ -275,6 +280,13 @@ pub fn run() { commands::start_pii_scan, commands::preview_redaction, commands::apply_redaction, + commands::get_dictionary, + commands::set_dictionary_field, + commands::remove_dictionary_field, + commands::discard_dictionary_orphans, + commands::export_dictionary, + commands::preview_dictionary_import, + commands::apply_dictionary_import, commands::get_changes, commands::revert_change, commands::revert_change_cells, diff --git a/src-tauri/src/pii.rs b/src-tauri/src/pii.rs index 82e85f2..b17441b 100644 --- a/src-tauri/src/pii.rs +++ b/src-tauri/src/pii.rs @@ -8,7 +8,7 @@ //! undo step, and nothing leaves the device (the audit log stores counts, //! never values). -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex, OnceLock}; use hmac::{Hmac, Mac}; @@ -196,6 +196,23 @@ pub struct PiiFinding { pub samples: Vec, } +/// A column the data dictionary (F38) classifies as confidential or +/// restricted. Surfaced by the scan preflight EVEN WITHOUT a pattern hit, so a +/// declared-sensitive column that no detector matched is still flagged for the +/// package/export preflight to act on. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SensitivityFlag { + pub column: usize, + pub column_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub display_name: Option, + /// "confidential" or "restricted". + pub sensitivity: String, + /// Whether a detector also matched this column in this scan. + pub has_pattern_hit: bool, +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct PiiReport { @@ -203,6 +220,9 @@ pub struct PiiReport { pub scanned_rows: usize, pub total_matches: usize, pub findings: Vec, + /// F38: columns the dictionary declares confidential/restricted, flagged + /// regardless of pattern hits so the preflight cannot miss them. + pub sensitivity_flags: Vec, } /// Run a PII scan. Read-only; never dirties the document. @@ -256,11 +276,27 @@ pub fn scan(doc: &Document, spec: &PiiSpec, ctx: &JobCtx) -> AppResult = findings.iter().map(|f| f.column).collect(); + let sensitivity_flags = crate::dictionary::sensitive_columns(doc) + .into_iter() + .map(|s| SensitivityFlag { + column: s.column, + column_id: s.column_id, + display_name: s.display_name, + sensitivity: s.sensitivity.label().to_string(), + has_pattern_hit: hit_columns.contains(&s.column), + }) + .collect(); + Ok(PiiReport { revision: doc.revision(), scanned_rows: rows.len(), total_matches: findings.iter().map(|f| f.count).sum(), findings, + sensitivity_flags, }) } @@ -636,6 +672,46 @@ mod tests { assert_eq!(d.rows()[0][0], "user@example.com"); } + #[test] + fn dictionary_sensitivity_is_folded_into_the_report() { + use crate::dictionary::{DictionaryField, Sensitivity}; + // Column 0 has emails (a pattern hit); column 1 is declared restricted + // but holds nothing a detector matches; column 2 is plain. + let mut d = doc("email,secret,plain\nuser@example.com,alpha,x\n"); + let ids: Vec = d.column_ids().to_vec(); + let mut a = DictionaryField::empty(ids[0].clone()); + a.sensitivity = Some(Sensitivity::Confidential); + d.set_dictionary_field(a); + let mut b = DictionaryField::empty(ids[1].clone()); + b.sensitivity = Some(Sensitivity::Restricted); + b.display_name = Some("Secret".into()); + d.set_dictionary_field(b); + + let report = run(&d, &spec(vec![PiiDetector::Email])); + // The restricted column is flagged even though no detector matched it. + assert_eq!(report.sensitivity_flags.len(), 2); + let secret = report + .sensitivity_flags + .iter() + .find(|f| f.column == 1) + .unwrap(); + assert_eq!(secret.sensitivity, "restricted"); + assert!( + !secret.has_pattern_hit, + "no detector matched the secret col" + ); + assert_eq!(secret.display_name.as_deref(), Some("Secret")); + // The confidential email column both matched a detector AND is flagged. + let email = report + .sensitivity_flags + .iter() + .find(|f| f.column == 0) + .unwrap(); + assert!(email.has_pattern_hit); + // Setting the dictionary never dirtied the document. + assert!(!d.is_dirty()); + } + #[test] fn invalid_configs_are_rejected() { let d = doc("a\n1\n"); diff --git a/src-tauri/src/project.rs b/src-tauri/src/project.rs index 9f958e2..01221cb 100644 --- a/src-tauri/src/project.rs +++ b/src-tauri/src/project.rs @@ -5,10 +5,10 @@ //! is a versioned envelope `{ formatVersion, appVersion, sections }` whose //! sections are typed and registered in [`SECTION_REGISTRY`]. Future features //! extend the registry by adding a typed field to [`ProjectSections`], a row -//! to the registry, and a match arm in [`set_section_typed`] — the reserved -//! `annotations` (F40), `dictionary` (F38) and `queries` (F36) sections are -//! already named, default-empty, and rejected for writes until their owning -//! feature lands. +//! to the registry, and a match arm in [`set_section_typed`] — as the F38 +//! `dictionary` section does. The still-reserved `annotations` (F40) and +//! `queries` (F36) sections are already named, default-empty, and rejected for +//! writes until their owning feature lands. //! //! Hard rules enforced here: //! @@ -29,8 +29,8 @@ //! section-map, source, tabs and per-source open-settings levels are //! preserved on round-trip via serde flatten catch-alls, alongside the //! version string. Typed sub-sections owned by other features (views, -//! schemas, comparisons, row keys) round-trip through those features' own -//! versioned payloads rather than through a catch-all here. +//! schemas, dictionaries, comparisons, row keys) round-trip through those +//! features' own versioned payloads rather than through a catch-all here. //! - **Nothing runs on open.** Opening a project yields a [`ProjectOpenPlan`] //! describing what to open and which named views are safe to reapply //! (fingerprint + column compatibility gated — warn, never break). Recipes, @@ -46,6 +46,7 @@ use serde_json::Value; use tauri::State; use crate::compare::CompareSpec; +use crate::dictionary::DictionaryExport; use crate::dto::{BackupPolicy, FileFingerprint}; use crate::error::{AppError, AppResult}; use crate::row_identity::KeySpec; @@ -171,10 +172,11 @@ pub struct ProjectSections { /// `row_identity::RowIdentity` (keys/record numbers), never by content. #[serde(default)] pub annotations: Vec, - /// RESERVED for F38 data dictionaries: per-column descriptions and - /// constraints (configuration only). + /// F38 data dictionaries, per source: per-column documentation and + /// constraints in the versioned dictionary export envelope (configuration + /// only — column IDs and metadata, never cell values). #[serde(default)] - pub dictionary: Vec, + pub dictionary: Vec, /// RESERVED for F36 saved queries: definitions only, never results. #[serde(default)] pub queries: Vec, @@ -253,7 +255,7 @@ pub const SECTION_REGISTRY: &[SectionSpec] = &[ }, SectionSpec { name: "dictionary", - reserved: true, + reserved: false, owner: "F38", }, SectionSpec { @@ -341,6 +343,17 @@ pub struct SourceSchema { pub schema: SchemaExport, } +/// F38 data dictionary for one source, in the versioned export envelope. The +/// envelope keys columns by their stable IDs and carries only documentation +/// (display name, description, role, unit, allowed values, …) — never a cell +/// value — so it passes the no-cell-data scan like every other section. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SourceDictionary { + pub source_id: String, + pub dictionary: DictionaryExport, +} + /// Row-identity key definition for one source. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -926,6 +939,7 @@ fn remove_source(sections: &mut ProjectSections, id: &str) { sections.views.retain(|v| v.source_id != id); sections.schemas.retain(|s| s.source_id != id); sections.row_keys.retain(|k| k.source_id != id); + sections.dictionary.retain(|d| d.source_id != id); sections .join_mappings .retain(|j| j.left_source_id != id && j.right_source_id != id); @@ -1236,6 +1250,7 @@ fn set_section_typed(sections: &mut ProjectSections, name: &str, value: Value) - "joinMappings" => sections.join_mappings = serde_json::from_value(value).map_err(shape)?, "comparisons" => sections.comparisons = serde_json::from_value(value).map_err(shape)?, "rowKeys" => sections.row_keys = serde_json::from_value(value).map_err(shape)?, + "dictionary" => sections.dictionary = serde_json::from_value(value).map_err(shape)?, _ => unreachable!("registry check above covers every arm"), } Ok(()) @@ -1297,6 +1312,7 @@ fn strip_sources(sections: &mut ProjectSections) { sections.views.clear(); sections.schemas.clear(); sections.row_keys.clear(); + sections.dictionary.clear(); sections.join_mappings.clear(); sections.comparisons.clear(); for profile in &mut sections.profiles { @@ -1479,6 +1495,9 @@ pub fn project_open_apply( mod tests { use super::*; use crate::compare::CompareMode; + use crate::dictionary::{ + DictionaryExportEntry, DictionaryField, FieldRole, Sensitivity, DICTIONARY_VERSION, + }; use crate::row_identity::KeyNormalization; use crate::schema::{ColumnSchema, LogicalType, SCHEMA_VERSION}; use crate::settings::ProfileMatch; @@ -1541,6 +1560,26 @@ mod tests { } } + /// A one-entry dictionary export documenting the `amount` column (keyed by + /// its stable ID `c1`), exercising text, enum and list fields. + fn a_dictionary() -> DictionaryExport { + DictionaryExport { + version: DICTIONARY_VERSION, + entries: vec![DictionaryExportEntry { + column_name: "amount".into(), + field: DictionaryField { + display_name: Some("Order amount".into()), + description: Some("Total charged for the order".into()), + role: Some(FieldRole::Measure), + unit: Some("USD".into()), + sensitivity: Some(Sensitivity::Confidential), + allowed_values: vec!["low".into(), "high".into()], + ..DictionaryField::empty("c1") + }, + }], + } + } + fn a_profile() -> FileProfile { FileProfile { id: "p1".into(), @@ -1564,6 +1603,7 @@ mod tests { cross_rules: Vec::new(), named_views: Vec::new(), last_view_id: None, + required_documentation: Vec::new(), } } @@ -1633,6 +1673,10 @@ mod tests { }, }, }]; + s.dictionary = vec![SourceDictionary { + source_id: "srcA".into(), + dictionary: a_dictionary(), + }]; (file, a, b) } @@ -1697,6 +1741,41 @@ mod tests { assert!(Path::new(&loaded.sections.sources[0].path).is_absolute()); } + #[test] + fn dictionary_section_registers_and_round_trips_per_source() { + // The F38 dictionary is a real registered section now, not reserved: + // a well-formed per-source payload (the JSON the front end sends via + // `project_set_section`) is accepted rather than rejected. + let mut sections = ProjectSections::default(); + let payload = serde_json::to_value(vec![SourceDictionary { + source_id: "srcA".into(), + dictionary: a_dictionary(), + }]) + .unwrap(); + set_section_typed(&mut sections, "dictionary", payload).unwrap(); + assert_eq!(sections.dictionary.len(), 1); + assert_eq!(sections.dictionary[0].source_id, "srcA"); + + // It survives an atomic save + reload, keyed by the stable column ID + // and carrying every documentation field verbatim. + let dir = tempfile::tempdir().unwrap(); + let (file, _a, _b) = full_project(dir.path()); + let path = dir.path().join(format!("dict.{PROJECT_EXTENSION}")); + write_project_file(&path, &file).unwrap(); + let loaded = load_project_file(&path).unwrap(); + assert_eq!(loaded.sections.dictionary, file.sections.dictionary); + let field = &loaded.sections.dictionary[0].dictionary.entries[0].field; + assert_eq!(field.column_id, "c1"); + assert_eq!(field.role, Some(FieldRole::Measure)); + assert_eq!(field.sensitivity, Some(Sensitivity::Confidential)); + assert_eq!(field.allowed_values, vec!["low", "high"]); + + // Documentation is configuration, not data: the whole serialized file + // still passes the no-cell-data scan with the dictionary populated. + let value: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + scan_for_data_keys("whole file", &value).expect("dictionary carries no cell data"); + } + #[test] fn unknown_fields_and_sections_survive_a_round_trip() { let dir = tempfile::tempdir().unwrap(); @@ -1866,7 +1945,7 @@ mod tests { #[test] fn reserved_and_unknown_section_writes_are_rejected() { let mut sections = ProjectSections::default(); - for reserved in ["annotations", "dictionary", "queries"] { + for reserved in ["annotations", "queries"] { let err = set_section_typed(&mut sections, reserved, serde_json::json!([])) .unwrap_err() .to_string(); @@ -2082,6 +2161,37 @@ mod tests { assert!(sections.sources.iter().all(|s| s.id != "srcB")); } + #[test] + fn removing_a_source_prunes_its_dictionary_section() { + // A per-source dictionary is source-specific metadata (keyed by + // `source_id`), so removing a source must drop its dictionary entry the + // same way views/schemas/row-keys are cascaded — otherwise a later save + // or template would keep documentation for a source that no longer + // exists. + let dir = tempfile::tempdir().unwrap(); + let (file, _, _) = full_project(dir.path()); + let mut sections = file.sections; + // full_project documents srcA; document srcB too so the removal has a + // dictionary entry to prune while srcA's is left intact. + sections.dictionary.push(SourceDictionary { + source_id: "srcB".into(), + dictionary: a_dictionary(), + }); + assert_eq!(sections.dictionary.len(), 2); + + remove_source(&mut sections, "srcB"); + + assert_eq!( + sections.dictionary.len(), + 1, + "srcB's dictionary section is dropped with the source" + ); + assert_eq!( + sections.dictionary[0].source_id, "srcA", + "srcA's dictionary survives" + ); + } + #[test] fn a_missing_source_without_a_resolution_cancels_the_whole_open() { let dir = tempfile::tempdir().unwrap(); @@ -2158,6 +2268,7 @@ mod tests { assert_eq!(sections["views"], serde_json::json!([])); assert_eq!(sections["schemas"], serde_json::json!([])); assert_eq!(sections["rowKeys"], serde_json::json!([])); + assert_eq!(sections["dictionary"], serde_json::json!([])); assert_eq!(sections["joinMappings"], serde_json::json!([])); assert_eq!(sections["comparisons"], serde_json::json!([])); assert_eq!(sections["tabs"]["open"], serde_json::json!([])); @@ -2267,6 +2378,7 @@ mod tests { let fresh = new_project(Some(&path)).unwrap(); assert!(fresh.file.sections.sources.is_empty()); assert!(fresh.file.sections.comparisons.is_empty()); + assert!(fresh.file.sections.dictionary.is_empty()); assert_eq!(fresh.file.sections.profiles.len(), 1); } @@ -2287,7 +2399,7 @@ mod tests { } // layout serializes as null; everything else defaults to empty lists // or objects. Reserved sections are present-but-empty by design. - for reserved in ["annotations", "dictionary", "queries"] { + for reserved in ["annotations", "queries"] { assert_eq!(map[reserved], serde_json::json!([]), "{reserved}"); } assert_eq!( diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs index 6f74da0..8acfde1 100644 --- a/src-tauri/src/settings.rs +++ b/src-tauri/src/settings.rs @@ -158,6 +158,13 @@ pub struct FileProfile { /// F12: the view last applied to a matching file, restored on reopen. #[serde(default)] pub last_view_id: Option, + + /// F38: required data-dictionary documentation. Each rule names the columns + /// it covers (empty = every column) and the dictionary fields that must be + /// populated; a gap surfaces as an ordinary `missingDocumentation` + /// validation issue. + #[serde(default)] + pub required_documentation: Vec, } /// The persisted settings document. @@ -451,6 +458,22 @@ pub fn validate_profile(doc: &Document, profile: &FileProfile) -> AppResult = gap.missing.iter().map(|k| k.label()).collect(); + issues.push(ProfileIssue { + kind: "missingDocumentation".into(), + column: Some(gap.column_name.clone()), + detail: format!( + "“{}” is missing required documentation: {}", + gap.column_name, + missing.join(", ") + ), + affected_count: gap.missing.len(), + }); + } + Ok(ProfileValidation { profile_id: profile.id.clone(), ok: issues.is_empty(), @@ -498,6 +521,7 @@ mod tests { cross_rules: Vec::new(), named_views: Vec::new(), last_view_id: None, + required_documentation: Vec::new(), } } @@ -627,6 +651,62 @@ mod tests { assert_eq!(kind("outOfRange").unwrap().affected_count, 2); } + #[test] + fn required_documentation_surfaces_as_validation_issues() { + use crate::dictionary::{DictionaryField, DictionaryFieldKey, RequiredDocumentation}; + let mut d = doc_from("id,amount,email\n1,10,a@b.c"); + // Document only the id column, and only with a description. + let mut f = DictionaryField::empty(d.column_ids()[0].clone()); + f.description = Some("primary key".into()); + d.set_dictionary_field(f); + + let mut p = profile(); + // Trim the noisier data rules so the documentation issues stand alone. + p.regex_rules.clear(); + p.range_rules.clear(); + p.expected_types.clear(); + p.required_documentation = vec![RequiredDocumentation { + columns: vec![], + fields: vec![DictionaryFieldKey::Description, DictionaryFieldKey::Owner], + }]; + let v = validate_profile(&d, &p).unwrap(); + let doc_issues: Vec<&ProfileIssue> = v + .issues + .iter() + .filter(|i| i.kind == "missingDocumentation") + .collect(); + // id: missing owner; amount + email: missing both. + assert_eq!(doc_issues.len(), 3); + let id_issue = doc_issues + .iter() + .find(|i| i.column.as_deref() == Some("id")) + .unwrap(); + assert!(id_issue.detail.contains("owner")); + assert!( + !id_issue.detail.contains("description"), + "description is set" + ); + assert!(!v.ok); + + // Fully documenting id + owner-only rule clears id's issue. + let mut full = DictionaryField::empty(d.column_ids()[0].clone()); + full.description = Some("primary key".into()); + full.owner = Some("data-team".into()); + d.set_dictionary_field(full); + let scoped = FileProfile { + required_documentation: vec![RequiredDocumentation { + columns: vec!["id".into()], + fields: vec![DictionaryFieldKey::Description, DictionaryFieldKey::Owner], + }], + ..profile() + }; + let v2 = validate_profile(&d, &scoped).unwrap(); + assert!( + v2.issues.iter().all(|i| i.kind != "missingDocumentation"), + "id is fully documented" + ); + } + #[test] fn shortcut_overrides_round_trip_and_default_empty() { let dir = tempfile::tempdir().unwrap(); diff --git a/src/App.tsx b/src/App.tsx index bb5d178..8aa2452 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -47,6 +47,7 @@ import { ReopenDialog } from "./components/ReopenDialog"; import { RepairDialog } from "./components/RepairDialog"; import { ReshapeDialog } from "./components/ReshapeDialog"; import { SchemaDialog } from "./components/SchemaDialog"; +import { DictionaryDialog } from "./components/DictionaryDialog"; import { SemanticDialog } from "./components/SemanticDialog"; import { ShortcutsDialog } from "./components/ShortcutsDialog"; import { SortDialog } from "./components/SortDialog"; @@ -319,6 +320,7 @@ export default function App() { {activeModal === "cluster" && setModal(null)} />} {activeModal === "semantic" && setModal(null)} />} {activeModal === "schema" && setModal(null)} />} + {activeModal === "dictionary" && setModal(null)} />} {activeModal === "crossval" && setModal(null)} />} {activeModal === "repair" && setModal(null)} />} {activeModal === "outlier" && setModal(null)} />} diff --git a/src/components/ColumnMenu.tsx b/src/components/ColumnMenu.tsx index 5a9169c..7f7a73d 100644 --- a/src/components/ColumnMenu.tsx +++ b/src/components/ColumnMenu.tsx @@ -37,6 +37,7 @@ export function ColumnMenu({ state, headers, columnIds, readOnly, onClose }: Col const pinColumn = useStore((s) => s.pinColumn); const requestAutoFit = useStore((s) => s.requestAutoFit); const openSchemaDialog = useStore((s) => s.openSchemaDialog); + const openDictionaryDialog = useStore((s) => s.openDictionaryDialog); const columnLayout = useStore((s) => s.columnLayout); const columnId = columnIds[col]; const isPinned = columnId !== undefined && !!columnLayout?.pinnedColumnIds.includes(columnId); @@ -128,6 +129,7 @@ export function ColumnMenu({ state, headers, columnIds, readOnly, onClose }: Col {/* F31: declaring a logical type is metadata — allowed read-only too. */} run(() => openSchemaDialog(col))}>Edit schema… + run(() => openDictionaryDialog(col))}>Document column… run(() => setColumnHidden(col, true))}>Hide column run(() => pinColumn(col, !isPinned))}> diff --git a/src/components/DictionaryDialog.tsx b/src/components/DictionaryDialog.tsx new file mode 100644 index 0000000..3c31d9a --- /dev/null +++ b/src/components/DictionaryDialog.tsx @@ -0,0 +1,827 @@ +import { useEffect, useMemo, useRef, useState } from "react"; + +import { + FIELD_KEY_LABELS, + MATCH_BY_OPTIONS, + ROLE_LABELS, + ROLE_OPTIONS, + SENSITIVITY_LABELS, + SENSITIVITY_OPTIONS, + allConflictsResolved, + applyMatchBy, + bulkChoices, + completeness, + conflictKey, + buildPerFieldResolution, + isDocumented, + isSensitive, + normalizeField, + unresolvedCount, + type ConflictChoices, +} from "../lib/dictionary"; +import { LOGICAL_TYPE_LABELS } from "../lib/schema"; +import { useActiveMeta, useStore } from "../store/useStore"; +import type { + DictionaryEntryView, + DictionaryField, + DictionaryFormat, + FieldRole, + MergeMatchBy, + MergePlan, + MergeResolution, + Sensitivity, +} from "../types"; +import { Modal } from "./Modal"; + +/** + * Data dictionary (F38): a searchable, per-column documentation editor. Each + * column's technical name and inferred F31 type are prefilled; every + * DictionaryField is editable (display name, description, role, unit, source, + * sensitivity, allowed values, example, owner, notes). Documentation is + * metadata — edits never dirty the source document. From here the user can + * export the dictionary (JSON / Markdown / CSV), import and MERGE a dictionary + * (matching by column ID or name, resolving field-level conflicts explicitly), + * and clean up orphaned entries left when a documented column is deleted. + */ +export function DictionaryDialog({ onClose }: { onClose: () => void }) { + const meta = useActiveMeta(); + const view = useStore((s) => s.dictionaryView); + const focusColumn = useStore((s) => s.dictionaryDialogColumn); + const loadDictionary = useStore((s) => s.loadDictionary); + const setField = useStore((s) => s.setDictionaryField); + const removeField = useStore((s) => s.removeDictionaryField); + const discardOrphans = useStore((s) => s.discardDictionaryOrphans); + const exportToFile = useStore((s) => s.exportDictionaryToFile); + const pickImportFile = useStore((s) => s.pickDictionaryImportFile); + const previewImport = useStore((s) => s.previewDictionaryImport); + const applyImport = useStore((s) => s.applyDictionaryImport); + + const entries = useMemo(() => view?.entries ?? [], [view]); + const orphans = view?.orphans ?? []; + + const [selectedId, setSelectedId] = useState(null); + const [search, setSearch] = useState(""); + const [draft, setDraft] = useState(null); + const [allowedInput, setAllowedInput] = useState(""); + const [notice, setNotice] = useState(null); + const [working, setWorking] = useState(false); + + // Import sub-flow state. + const [importPath, setImportPath] = useState(null); + const [matchBy, setMatchBy] = useState("auto"); + const [plan, setPlan] = useState(null); + const [importBusy, setImportBusy] = useState(false); + const [conflictOpen, setConflictOpen] = useState(false); + const [importSummary, setImportSummary] = useState(null); + // Monotonic id of the most recently requested preview. A preview that + // resolves after a newer request (a changed Match by selector) or after the + // panel was cancelled is stale and must not populate the displayed plan. + const previewSeq = useRef(0); + + useEffect(() => { + void loadDictionary(); + }, [loadDictionary]); + + // Pick an initial column: the one the caller focused, else the first. + useEffect(() => { + if (selectedId !== null || entries.length === 0) return; + const focused = + focusColumn != null ? entries.find((e) => e.columnIndex === focusColumn) : undefined; + setSelectedId((focused ?? entries[0]).columnId); + }, [entries, focusColumn, selectedId]); + + const selectedEntry: DictionaryEntryView | undefined = useMemo( + () => entries.find((e) => e.columnId === selectedId), + [entries, selectedId], + ); + const storedKey = selectedEntry ? JSON.stringify(selectedEntry.field) : ""; + + // Seed the editable draft from the stored field whenever the selected column + // (or its stored documentation) changes. Typing does not persist until Save. + useEffect(() => { + if (!selectedEntry) { + setDraft(null); + return; + } + setDraft({ + ...selectedEntry.field, + allowedValues: [...(selectedEntry.field.allowedValues ?? [])], + }); + setAllowedInput(""); + setNotice(null); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedId, storedKey]); + + const filtered = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!q) return entries; + return entries.filter( + (e) => + e.columnName.toLowerCase().includes(q) || + (e.field.displayName ?? "").toLowerCase().includes(q) || + (e.field.description ?? "").toLowerCase().includes(q), + ); + }, [entries, search]); + + if (!meta) return null; + + const isDirty = + draft !== null && + selectedEntry !== undefined && + JSON.stringify(normalizeField(draft)) !== JSON.stringify(normalizeField(selectedEntry.field)); + + const patch = (p: Partial) => setDraft((d) => (d ? { ...d, ...p } : d)); + + const save = async () => { + if (!draft) return; + setWorking(true); + setNotice(null); + const ok = await setField(normalizeField(draft)); + setWorking(false); + if (ok) setNotice("Documentation saved."); + }; + + const clearEntry = async () => { + if (!selectedId) return; + setWorking(true); + setNotice(null); + const ok = await removeField(selectedId); + setWorking(false); + if (ok) setNotice("Documentation cleared."); + }; + + const runExport = async (format: DictionaryFormat) => { + setNotice(null); + await exportToFile(format); + }; + + const startImport = async () => { + const path = await pickImportFile(); + if (!path) return; + setImportPath(path); + setImportSummary(null); + await runPreview(path, matchBy); + }; + + const runPreview = async (path: string, mb: MergeMatchBy) => { + const seq = ++previewSeq.current; + setImportBusy(true); + const p = await previewImport(path, mb); + // Drop a stale result: a newer preview or a cancel superseded this one, so + // the plan it produced no longer matches the current Match by selection. + if (seq !== previewSeq.current) return; + setImportBusy(false); + setPlan(p); + }; + + const changeMatchBy = async (mb: MergeMatchBy) => { + setMatchBy(mb); + if (importPath) await runPreview(importPath, mb); + }; + + const cancelImport = () => { + // Invalidate any in-flight preview so its late result cannot repopulate the + // panel after it has been dismissed. + previewSeq.current++; + setImportBusy(false); + setImportPath(null); + setPlan(null); + setConflictOpen(false); + }; + + const finishImport = async (resolution: MergeResolution) => { + if (!importPath || !plan) return; + setImportBusy(true); + // Apply under exactly what the reviewed plan was computed with: its own + // match mode and revision, NOT the live dialog state. The Match by selector + // (or a stale preview) can have moved on since this plan was displayed, and + // merging under a different mode would touch a different set of columns than + // the conflicts/counts the user reviewed. The revision guard likewise + // rejects a now-stale apply if documentation was edited after the preview. + const outcome = await applyImport( + importPath, + applyMatchBy(plan), + resolution, + plan.dictionaryRevision, + ); + setImportBusy(false); + if (!outcome) return; // error surfaced globally; keep the panel open to retry + setConflictOpen(false); + setImportPath(null); + setPlan(null); + const bits = [ + `${outcome.newEntries} new`, + `${outcome.fieldsAdded} field${outcome.fieldsAdded === 1 ? "" : "s"} added`, + `${outcome.conflictsResolved} conflict${outcome.conflictsResolved === 1 ? "" : "s"} resolved`, + ]; + if (outcome.unmatched.length > 0) bits.push(`${outcome.unmatched.length} unmatched`); + setImportSummary(`Imported: ${bits.join(", ")}.`); + }; + + // No-conflict imports still need an explicit apply; keepAllExisting is a safe + // resolution because there is nothing to resolve. + const applyClean = () => void finishImport({ type: "keepAllExisting" }); + + const draftComplete = draft ? completeness(draft) : null; + + return ( + <> + + + Documentation is metadata — it never changes cell text or marks the document dirty. + + + + } + > +
+ {/* toolbar */} +
+ +
+ Export + + + +
+ + {documentedCount(entries)} / {entries.length} column + {entries.length === 1 ? "" : "s"} documented + +
+ + {notice && ( +

+ {notice} +

+ )} + + {/* import panel */} + {importPath && ( +
+
+ Import dictionary + + {fileNameOf(importPath)} + + +
+ {importBusy &&

Analyzing…

} + {plan && !importBusy && ( + <> +
+ + {plan.matchedColumns} matched + + + {plan.newEntries.length} new + + + {plan.cleanAdditions} clean additions + + 0 + ? "bg-amber-100 text-amber-800 dark:bg-amber-500/15 dark:text-amber-300" + : "bg-zinc-100 text-zinc-600 dark:bg-zinc-800 dark:text-zinc-300" + } + > + {plan.conflicts.length} conflict{plan.conflicts.length === 1 ? "" : "s"} + + {plan.unmatched.length > 0 && ( + + {plan.unmatched.length} unmatched + + )} +
+ {plan.unmatched.length > 0 && ( +

+ No current column for: {plan.unmatched.join(", ")} +

+ )} +
+ {plan.conflicts.length > 0 ? ( + + ) : ( + + )} + +
+ + )} +
+ )} + + {importSummary && ( +

+ {importSummary} +

+ )} + +
+ {/* ----- column list ----- */} +
+ setSearch(e.target.value)} + placeholder="Search columns…" + className={`${inputCls} mb-2`} + /> +
+ {filtered.map((e) => { + const c = completeness(e.field); + return ( + + ); + })} + {filtered.length === 0 && ( +

No columns match.

+ )} +
+
+ + {/* ----- editor ----- */} + {draft && selectedEntry && ( +
+
+
+

+ {selectedEntry.columnName} +

+

+ ID {selectedEntry.columnId} + {selectedEntry.logicalType && ( + <> · type {LOGICAL_TYPE_LABELS[selectedEntry.logicalType]} + )} +

+
+ {draftComplete && ( + + {draftComplete.filled}/{draftComplete.total} fields + + )} +
+ +
+ + +
+ +