Ship the ADOP plugin, and fix nine engine defects found by running it

.claude-plugin/marketplace.json

@@ -0,0 +1,16 @@
+{
+  "$schema": "https://json.schemastore.org/claude-code-marketplace.json",
+  "name": "adop-marketplace",
+  "description": "Agentic Data Operations Platform (ADOP) plugins for Claude Code — turn a natural-language data source description into a tested Bronze→Silver→Gold pipeline with inline compliance controls.",
+  "owner": {
+    "name": "AWS Samples",
+    "url": "https://github.com/aws-samples/sample-Agentic-Ai-Data-Operations"
+  },
+  "plugins": [
+    {
+      "name": "adop",
+      "source": "./plugins/claude-code",
+      "description": "Agentic Data Operations Platform — natural language to a tested Bronze→Silver→Gold data pipeline with inline compliance controls."
+    }
+  ]
+}

.gitignore

@@ -14,8 +14,8 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
+/lib/
+/lib64/
 parts/
 sdist/
 var/
@@ -278,3 +278,8 @@ cleanup_*.sh
 
 # Local README preview (generated by /tmp/render_readme.py)
 README-preview.html
+
+# `claude plugin eval` writes timestamped run output here — reports, aggregate JSON and
+# scaffold temp dirs. Transient artifacts of running the suite, not plugin content.
+# Anchored, because an unanchored `results/` once swallowed 27 vendored files.
+/plugins/claude-code/evals/results/

.mcp.json

@@ -107,7 +107,7 @@
     },
     "pii-detection": {
       "command": "uv",
-      "args": ["run", "--no-project", "--with", "boto3", "--with", "mcp", "--python", "3.13", "mcp-servers/pii-detection-server/server.py"],
+      "args": ["run", "--no-project", "--with", "boto3", "--with", "fastmcp", "--python", "3.13", "mcp-servers/pii-detection-server/server.py"],
       "env": {
         "AWS_REGION": "us-east-1",
         "AWS_PROFILE": "default"

contracts/v1/bronze_spec.schema.json

@@ -70,6 +70,12 @@
       "enum": ["full_load", "incremental", "cdc"],
       "default": "full_load"
     },
+    "ingest_sequence_column": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9_]*$",
+      "description": "Name of a column Bronze STAMPS on every ingested row with the run's logical date, supplying a monotonic ordering across runs that the source itself does not provide. Bronze stays append-only: this ADDS a column, it never updates an existing row, so Bronze immutability holds. Silver then resolves same-primary-key conflicts by taking the highest value, which is the only way to apply an upstream restatement — a claim moving PENDING to PAID, say — when no source column changes on update. Point silver_spec.dedup_order_by at this column. Omit it when the source supplies its own reliable last-updated timestamp and order by that instead.",
+      "$comment": "Needed because ingestion_mode: cdc was unimplementable on its own: a merge has to know which of two rows is newer, and a source whose update does not touch any column cannot say."
+    },
     "file_pattern": {
       "type": "string",
       "description": "Glob pattern to match source files (e.g., *.csv)"

contracts/v1/dag_spec.schema.json

@@ -4,7 +4,12 @@
   "title": "DAG Schedule Spec",
   "description": "Defines Airflow DAG scheduling, retries, failure handling, and task dependencies.",
   "type": "object",
-  "required": ["dataset_name", "schema_version", "schedule", "tasks"],
+  "required": [
+    "dataset_name",
+    "schema_version",
+    "schedule",
+    "tasks"
+  ],
   "additionalProperties": false,
   "properties": {
     "dataset_name": {
@@ -22,13 +27,19 @@
     },
     "schedule": {
       "type": "object",
-      "required": ["cron", "timezone"],
+      "required": [
+        "cron",
+        "timezone"
+      ],
       "additionalProperties": false,
       "properties": {
         "cron": {
-          "type": "string",
+          "type": [
+            "string",
+            "null"
+          ],
           "minLength": 9,
-          "description": "Cron expression (e.g., '0 6 * * *')"
+          "description": "Cron expression (e.g., '0 6 * * *'), or null for a manual / on-demand DAG. minLength applies only to the string form, so null is accepted. Was string-only, which made schedule_interval=None inexpressible even though it is ordinary Airflow."
         },
         "timezone": {
           "type": "string",
@@ -81,7 +92,10 @@
       "type": "array",
       "items": {
         "type": "object",
-        "required": ["task_id", "type"],
+        "required": [
+          "task_id",
+          "type"
+        ],
         "additionalProperties": false,
         "properties": {
           "task_id": {
@@ -90,15 +104,24 @@
           },
           "type": {
             "type": "string",
-            "enum": ["glue_job", "quality_check", "sensor", "python_callable", "dummy"]
+            "enum": [
+              "glue_job",
+              "quality_check",
+              "sensor",
+              "python_callable",
+              "dummy"
+            ]
           },
           "script_path": {
             "type": "string",
             "description": "Path to the script this task executes"
           },
           "depends_on": {
             "type": "array",
-            "items": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" },
+            "items": {
+              "type": "string",
+              "pattern": "^[a-z][a-z0-9_]*$"
+            },
             "default": []
           },
           "timeout_minutes": {
@@ -120,7 +143,13 @@
       "properties": {
         "on_failure_callback": {
           "type": "string",
-          "enum": ["sns_alert", "slack_notify", "pagerduty", "email", "none"],
+          "enum": [
+            "sns_alert",
+            "slack_notify",
+            "pagerduty",
+            "email",
+            "none"
+          ],
           "default": "sns_alert"
         },
         "notification_channel": {
@@ -143,14 +172,19 @@
         },
         "critical_tasks": {
           "type": "array",
-          "items": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" },
+          "items": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9_]*$"
+          },
           "description": "Tasks that trigger SLA breach alerts"
         }
       }
     },
     "tags": {
       "type": "array",
-      "items": { "type": "string" },
+      "items": {
+        "type": "string"
+      },
       "default": []
     }
   }

contracts/v1/gold_spec.schema.json

@@ -104,6 +104,11 @@
       "type": "string",
       "pattern": "^[a-z][a-z0-9_]*$"
     },
+    "publish_via_gate": {
+      "type": "boolean",
+      "default": false,
+      "description": "When true this template writes to {iceberg_table}_staging instead of the real table, leaving publication to the quality gate. iceberg_table stays the real name so semantic.yaml, the LF-Tag grants and gold_spec.source_table keep pointing at it. Requires gate_before_publish on the workload's quality_spec and PROMOTE_TO on the quality task -- without both, the data is staged and never published."
+    },
     "iceberg_partition_spec": {
       "type": "array",
       "items": { "type": "string" },
@@ -121,10 +126,21 @@
           "columns": {
             "type": "array",
             "items": { "type": "string" }
+          },
+          "atomic": {
+            "type": "boolean",
+            "default": false,
+            "description": "For a `fact` table only: write one row per SOURCE row, straight from the pre-aggregation dataframe, instead of at the spec's `grain`. Use it to publish an atomic fact alongside the aggregate when a consumer needs a dimension that the aggregate grain does not carry — provider-level slicing off a daily plan/claim-type aggregate, for instance, which is otherwise impossible. An atomic fact republishes row-level detail into Gold, so under HIPAA it inherits the source zone's sensitivity and needs its own column-level controls; it is not a cheaper aggregate. Ignored for dimension and flat tables."
+          },
+          "business_key": {
+            "type": "array",
+            "minItems": 1,
+            "items": { "type": "string" },
+            "description": "Natural key identifying one entity in this dimension. Used as the MERGE predicate when the table is written as SCD Type 2. REQUIRED when any dimensions[] attribute whose source_column appears in this table's columns declares scd_type 2: a Type 2 merge must know which rows are successive versions of the same entity, and because dimensions[] names attributes rather than tables there is nothing to infer it from. Every listed column must also appear in columns. Ignored for Type 1 dimensions and for fact/flat tables."
           }
         }
       },
-      "description": "For star_schema: list of fact + dimension tables"
+      "description": "For star_schema: list of fact + dimension tables. A dimension table is written as SCD Type 2 when any dimensions[] attribute whose source_column is among its columns declares scd_type 2; otherwise Type 1 (overwrite)."
     },
     "freshness_sla_hours": {
       "type": "integer",
@@ -160,6 +176,11 @@
       },
       "default": [],
       "description": "Columns derived from aggregated measures (booleans, ratios, masking expressions over Gold dimensions)."
+    },
+    "post_aggregation_filter": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Spark SQL predicate applied to the aggregated rows immediately before the write. Rows not satisfying it are dropped, not labelled. This is the only way to suppress a published cell: post_aggregation_columns can add a column but cannot remove a row, so a k-anonymity flag computed there is a boolean sitting next to the disclosed row rather than a suppression. Example for k=11: claim_count >= 11."
     }
   }
 }