feat: graph utilization improvements for description generation and text2sql

CHANGELOG.md

@@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.8] - 2026-08-11
+
+### Added
+
+- `DescriptionSource.FALLBACK` - rule-based placeholder descriptions are now
+  distinguishable from model output and retried on the next
+  `generate_all_descriptions()` run. Exports emit `"fallback"`; older clgraph
+  versions will not recognize this value when importing such exports.
+- `generate_all_descriptions(include_sources=True)` also describes
+  source-table columns from forward usage context. New public
+  `build_source_description_prompt()`.
+- Public `Pipeline.get_incoming_edges()`. Bulk description/metadata passes now
+  use the adjacency index instead of linear edge scans.
+- Text2sql prompts now include column lineage in the default direct strategy,
+  table role labels (source/intermediate/final) with a prefer-final-tables
+  instruction, and a `## Join Hints` section (observed equi-joins from
+  pipeline SQL plus identity-preserving candidate joins).
+- `ContextConfig` fields `max_lineage_columns_per_table`, `max_lineage_lines`,
+  `annotate_table_roles`, `lineage_expansion_depth`, `max_join_hints`.
+
+### Changed
+
+- `generate_all_descriptions()` now also describes computed columns of
+  queries without a destination table (terminal SELECTs) - reruns may issue
+  more LLM calls than before.
+- `expand_with_lineage()` walks ancestors transitively (default depth 2,
+  configurable); two-stage text2sql context may include more tables.
+- `build_schema_context()`/`resolve_context_tables()` - explicit table
+  selections now preserve caller order and truncate to `max_tables`
+  (previously oversized explicit lists were reordered and could exceed the
+  cap).
+
+### Fixed
+
+- Generated-SQL safety validation now parses with the pipeline's dialect.
+  Dialect-specific syntax (e.g. BigQuery backticked identifiers) previously
+  failed to parse and was passed through unvalidated.
+
 ## [0.0.7] - 2026-08-02
 
 ### Fixed

README.md

@@ -509,6 +509,21 @@ Generated descriptions for 8 columns:
     Order total amount in USD per customer from raw orders table.
 ```
 
+Options worth knowing:
+
+<!-- skip-test -->
+```python
+# Also describe source-table columns (from how they are used downstream),
+# so the first computed layer gets real source context in its prompts:
+pipeline.generate_all_descriptions(include_sources=True)
+
+# When the LLM fails or its output is rejected, a rule-based placeholder is
+# written with description_source == DescriptionSource.FALLBACK. Placeholders
+# are retried automatically on the next run, and are never fed into
+# downstream prompts as source context. Prefer a hard error instead:
+pipeline.generate_all_descriptions(on_error="raise")
+```
+
 ### Lineage Agent (Natural Language Interface)
 
 Query your lineage data using natural language. The agent automatically routes questions to appropriate tools. Most queries work without an LLM - only SQL generation requires one:

examples/README.md

@@ -50,7 +50,10 @@ Extracting and using metadata from SQL comments.
 ---
 
 ### `llm_description_generation.ipynb`
-Using LLMs to generate column descriptions.
+Using LLMs to generate column descriptions — including source-column descriptions from forward usage (`include_sources=True`) and fallback/retry semantics (`DescriptionSource.FALLBACK`).
+
+### `text_to_sql.ipynb`
+Schema-aware SQL generation from natural language — shows the lineage-derived prompt context (table roles, column lineage, observed and candidate join hints), direct vs two-stage strategies, and routing through `LineageAgent`.
 
 **Features demonstrated:**
 - LLM-powered description generation

examples/enterprise_demo_with_ollama.ipynb

@@ -48,7 +48,7 @@
     "OLLAMA_MODEL = \"gpt-oss:20b\"  # Or try: llama3.2, qwen2.5-coder:7b\n",
     "SKIP_DESCRIPTIONS = False  # Set True to skip LLM description generation\n",
     "SKIP_AGENT = False  # Set True to skip LineageAgent demo\n",
-    "SKIP_TEXT_TO_SQL = False  # Set True to skip text-to-SQL demo"
+    "# Text-to-SQL has its own dedicated example: examples/text_to_sql.ipynb"
    ]
   },
   {
@@ -247,7 +247,7 @@
     "\n",
     "# Setup LLM if needed\n",
     "llm = None\n",
-    "if not (SKIP_DESCRIPTIONS and SKIP_AGENT and SKIP_TEXT_TO_SQL):\n",
+    "if not (SKIP_DESCRIPTIONS and SKIP_AGENT):\n",
     "    llm = setup_ollama_llm(OLLAMA_MODEL)"
    ]
   },

examples/llm_description_generation.ipynb

@@ -15,7 +15,7 @@
     "\n",
     "Requirements:\n",
     "- Install: uv pip install -e .\n",
-    "- For Ollama: Install Ollama and run: ollama pull llama3.2\n",
+    "- For Ollama: Install Ollama and run: ollama pull gemma4:31b\n",
     "- For OpenAI: Set OPENAI_API_KEY environment variable"
    ]
   },
@@ -33,10 +33,10 @@
    "id": "5e2c8f5e",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2025-12-30T20:10:34.992992Z",
-     "iopub.status.busy": "2025-12-30T20:10:34.992706Z",
-     "iopub.status.idle": "2025-12-30T20:10:43.205024Z",
-     "shell.execute_reply": "2025-12-30T20:10:43.204611Z"
+     "iopub.execute_input": "2026-08-11T22:16:58.691095Z",
+     "iopub.status.busy": "2026-08-11T22:16:58.690929Z",
+     "iopub.status.idle": "2026-08-11T22:23:40.832893Z",
+     "shell.execute_reply": "2026-08-11T22:23:40.831601Z"
     }
    },
    "outputs": [
@@ -50,7 +50,7 @@
       "================================================================================\n",
       "\n",
       "================================================================================\n",
-      "Example 1: Using Ollama with qwen3-coder:30b (Local LLM)\n",
+      "Example 1: Using Ollama with gemma4:31b (Local LLM)\n",
       "================================================================================\n",
       "\n",
       "📊 Parsing SQL pipeline...\n",
@@ -65,38 +65,29 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "✅ Ollama configured (model: qwen3-coder:30b)\n",
+      "✅ Ollama configured (model: gemma4:31b)\n",
       "\n",
       "🔮 Generating descriptions using LLM...\n",
       "(This may take 10-30 seconds depending on your machine)\n",
-      "\n",
-      "📊 Generating descriptions for 10 columns...\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "   Processed 10/10 columns...\n"
+      "\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "✅ Done! Generated 10 descriptions\n",
       "\n",
       "================================================================================\n",
       "Generated Descriptions\n",
       "================================================================================\n",
       "\n",
       "📊 analytics.user_metrics\n",
       "--------------------------------------------------------------------------------\n",
-      "  avg_order_value      [🤖 LLM] Average order value per customer derived from raw orders data in USD.\n",
-      "  last_order_date      [🤖 LLM] Last order date per day aggregated from staging.user_orders sourced from raw.orders table.\n",
-      "  order_count          [🤖 LLM] Total number of orders per user aggregated from raw orders table.\n",
-      "  total_revenue        [🤖 LLM] Total revenue per user aggregated from order amounts in USD.\n",
-      "  user_id              [🤖 LLM] Unique user identifier per customer record from raw orders table, used for user-level analytics.\n",
+      "  avg_order_value      [🤖 LLM] Average amount in USD per order sourced from user order data.\n",
+      "  last_order_date      [🤖 LLM] Date of the most recent order per user from staging user orders.\n",
+      "  order_count          [🤖 LLM] Total number of orders per user from staging.user_orders.\n",
+      "  total_revenue        [🤖 LLM] Total revenue in USD per user derived from raw order amounts.\n",
+      "  user_id              [🤖 LLM] Unique identifier for the user, sourced from raw orders.\n",
       "\n",
       "📊 raw.orders\n",
       "--------------------------------------------------------------------------------\n",
@@ -108,11 +99,11 @@
       "\n",
       "📊 staging.user_orders\n",
       "--------------------------------------------------------------------------------\n",
-      "  amount               [🤖 LLM] Order amount in USD per customer from raw.orders table.\n",
-      "  order_date           [🤖 LLM] Order date when customers placed their purchases, sourced from raw.orders table per day aggregation.\n",
-      "  order_id             [🤖 LLM] Unique order identifier per customer from the raw orders table.\n",
-      "  status               [🤖 LLM] Order status indicator showing pending, completed, or cancelled states per raw.orders source.\n",
-      "  user_id              [🤖 LLM] Unique user identifier from raw orders table, per customer record.\n",
+      "  amount               [🤖 LLM] Order amount in USD sourced from raw orders.\n",
+      "  order_date           [🤖 LLM] The date the order was placed, sourced from raw orders.\n",
+      "  order_id             [🤖 LLM] Unique identifier for the order, sourced from raw orders.\n",
+      "  status               [🤖 LLM] The order status (pending, completed, or cancelled) sourced from raw orders.\n",
+      "  user_id              [🤖 LLM] Unique identifier for the user, sourced from raw orders.\n",
       "\n",
       "\n",
       "\n",
@@ -152,9 +143,9 @@
     "\n",
     "\n",
     "def example_with_ollama():\n",
-    "    \"\"\"Example using local Ollama with qwen3-coder:30b (free, no API key needed)\"\"\"\n",
+    "    \"\"\"Example using local Ollama with gemma4:31b (free, no API key needed)\"\"\"\n",
     "    print(\"=\" * 80)\n",
-    "    print(\"Example 1: Using Ollama with qwen3-coder:30b (Local LLM)\")\n",
+    "    print(\"Example 1: Using Ollama with gemma4:31b (Local LLM)\")\n",
     "    print(\"=\" * 80)\n",
     "    print()\n",
     "\n",
@@ -212,22 +203,22 @@
     "                col.set_source_description(\"Order status: pending, completed, cancelled\")\n",
     "    print()\n",
     "\n",
-    "    # Configure LLM (Ollama with qwen3-coder:30b)\n",
+    "    # Configure LLM (Ollama with gemma4:31b)\n",
     "    print(\"🤖 Configuring Ollama LLM...\")\n",
     "    try:\n",
     "        from langchain_ollama import ChatOllama\n",
     "\n",
     "        llm = ChatOllama(\n",
-    "            model=\"qwen3-coder:30b\",\n",
+    "            model=\"gemma4:31b\",\n",
     "            temperature=0.3,  # Lower temperature for more consistent descriptions\n",
     "        )\n",
     "        lineage_graph.llm = llm\n",
-    "        print(\"✅ Ollama configured (model: qwen3-coder:30b)\")\n",
+    "        print(\"✅ Ollama configured (model: gemma4:31b)\")\n",
     "    except Exception as e:\n",
     "        print(f\"❌ Failed to configure Ollama: {e}\")\n",
     "        print(\"💡 Make sure Ollama is installed and running:\")\n",
     "        print(\"   brew install ollama\")\n",
-    "        print(\"   ollama pull qwen3-coder:30b\")\n",
+    "        print(\"   ollama pull gemma4:31b\")\n",
     "        print(\"   ollama serve\")\n",
     "        return\n",
     "    print()\n",
@@ -386,6 +377,123 @@
     "    print(\"   - Fallback mode works without any LLM but produces simple descriptions\")\n",
     "    print()"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7bc86d68",
+   "metadata": {},
+   "source": [
+    "### Source-Column Descriptions and Fallback Semantics\n",
+    "\n",
+    "Two options extend `generate_all_descriptions()`:\n",
+    "\n",
+    "- **`include_sources=True`** first describes source-table columns from how they are used *downstream* (no manual `set_source_description` needed), so the first computed layer gets real source context in its prompts.\n",
+    "- **`DescriptionSource.FALLBACK`** marks the rule-based placeholder written when the LLM fails or its output is rejected. Placeholder columns are retried automatically on the next run, and their text is never used as source context for downstream prompts. Pass `on_error=\"raise\"` to get a hard error instead."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "48b20e68",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2026-08-11T22:23:40.839148Z",
+     "iopub.status.busy": "2026-08-11T22:23:40.838636Z",
+     "iopub.status.idle": "2026-08-11T22:28:39.370198Z",
+     "shell.execute_reply": "2026-08-11T22:28:39.368432Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "================================================================================\n",
+      "Example 4: Source-Column Descriptions and Fallback Semantics\n",
+      "================================================================================\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "  analytics.user_metrics.total_revenue          [🤖 LLM] Total revenue per user derived from raw order amounts.\n",
+      "  analytics.user_metrics.user_id                [🤖 LLM] Unique identifier for the user, sourced from raw order data.\n",
+      "  raw.orders.amount                             [🤖 LLM] The total monetary value of the order.\n",
+      "  raw.orders.order_id                           [🤖 LLM] Unique identifier for each customer order.\n",
+      "  raw.orders.user_id                            [🤖 LLM] Unique identifier for the user who placed the order.\n",
+      "  staging.user_orders.amount                    [🤖 LLM] Total monetary value of the order from raw.orders.amount.\n",
+      "  staging.user_orders.order_id                  [🤖 LLM] Unique identifier for each customer order from raw.orders.\n",
+      "  staging.user_orders.user_id                   [🤖 LLM] Unique identifier for the user who placed the order, sourced from raw orders.\n",
+      "\n",
+      "Columns holding retryable fallback placeholders: none\n"
+     ]
+    }
+   ],
+   "source": [
+    "def example_source_columns_and_fallback():\n",
+    "    \"\"\"Describe source columns from forward usage; inspect fallback state.\"\"\"\n",
+    "    from clgraph import Pipeline\n",
+    "    from clgraph.models import DescriptionSource\n",
+    "\n",
+    "    print(\"=\" * 80)\n",
+    "    print(\"Example 4: Source-Column Descriptions and Fallback Semantics\")\n",
+    "    print(\"=\" * 80)\n",
+    "    print()\n",
+    "\n",
+    "    pipeline = Pipeline.from_dict(\n",
+    "        {\n",
+    "            \"staging_user_orders\": \"\"\"\n",
+    "                CREATE OR REPLACE TABLE staging.user_orders AS\n",
+    "                SELECT user_id, order_id, amount FROM raw.orders\n",
+    "            \"\"\",\n",
+    "            \"user_metrics\": \"\"\"\n",
+    "                CREATE OR REPLACE TABLE analytics.user_metrics AS\n",
+    "                SELECT user_id, SUM(amount) AS total_revenue\n",
+    "                FROM staging.user_orders\n",
+    "                GROUP BY user_id\n",
+    "            \"\"\",\n",
+    "        },\n",
+    "        dialect=\"bigquery\",\n",
+    "    )\n",
+    "\n",
+    "    try:\n",
+    "        from langchain_ollama import ChatOllama\n",
+    "\n",
+    "        pipeline.llm = ChatOllama(model=\"gemma4:31b\", temperature=0.3)\n",
+    "    except Exception as e:\n",
+    "        print(f\"⚠️  Ollama not available ({e}) - skipping example\")\n",
+    "        return\n",
+    "\n",
+    "    # include_sources=True: raw.orders.* columns are described first (from\n",
+    "    # their downstream usage), then feed the computed columns' prompts.\n",
+    "    pipeline.generate_all_descriptions(verbose=True, include_sources=True)\n",
+    "    print()\n",
+    "\n",
+    "    markers = {\n",
+    "        DescriptionSource.SOURCE: \"👤 USER\",\n",
+    "        DescriptionSource.GENERATED: \"🤖 LLM\",\n",
+    "        DescriptionSource.FALLBACK: \"🧩 FALLBACK\",\n",
+    "    }\n",
+    "    for col in sorted(pipeline.columns.values(), key=lambda c: c.full_name):\n",
+    "        if col.description:\n",
+    "            marker = markers.get(col.description_source, \"?\")\n",
+    "            print(f\"  {col.full_name:45} [{marker}] {col.description}\")\n",
+    "\n",
+    "    # FALLBACK placeholders (written when the LLM failed) are retried on the\n",
+    "    # next generate_all_descriptions() call - no overwrite=True needed.\n",
+    "    retryable = [\n",
+    "        c.full_name\n",
+    "        for c in pipeline.columns.values()\n",
+    "        if c.description_source == DescriptionSource.FALLBACK\n",
+    "    ]\n",
+    "    print(f\"\\nColumns holding retryable fallback placeholders: {retryable or 'none'}\")\n",
+    "\n",
+    "\n",
+    "example_source_columns_and_fallback()"
+   ]
   }
  ],
  "metadata": {