feat: add DataMarket — self-service data catalog on Databricks

CODEOWNERS

@@ -9,6 +9,7 @@
 acceptance                 @nfx @alexott
 conversational-agent-app   @vivian-xie-db @yuanchaoma-db
 database-diagram-builder   @alexott
+datamarket                 @rautsamir
 downstreams                @nfx @alexott
 feature-registry-app       @yang-chengg @mparkhe @mingyangge-db @stephanielu5
 go-libs                    @nfx @alexott

datamarket/README.md

@@ -0,0 +1,89 @@
+---
+title: "DataMarket — Self-Service Data Catalog"
+language: JavaScript
+author: "Samir Raut"
+date: 2026-06-29
+
+tags:
+- databricks-apps
+- unity-catalog
+- lakebase
+- data-catalog
+- data-governance
+- access-management
+---
+
+# DataMarket — Self-Service Data Catalog on Databricks
+
+A production-ready data catalog built entirely on Databricks. Gives every team a single place to discover certified data products, understand what they mean, and request access — without filing a ticket or waiting for an engineer.
+
+Built on Databricks Apps, Unity Catalog, Lakebase, and Foundation Model APIs. No external dependencies.
+
+![DataMarket](../../../docs/datamarket-thumbnail.png)
+
+## What it does
+
+- **Discover** — Browse and search all certified data products across your organization with domain filters, classification tags, and per-user access status
+- **Request** — Submit access requests with business justification; data stewards review and approve
+- **Govern** — Approvals automatically execute Unity Catalog `GRANT` statements via SQL Warehouse — no manual permission management
+- **Ask AI** — Natural language data discovery backed by Databricks Foundation Model APIs (Llama 3.3-70B)
+- **Insights** — Usage analytics and access request trends for stewards and managers
+
+## Databricks Services Used
+
+| Component | Service |
+|---|---|
+| Application hosting | Databricks Apps (serverless) |
+| Data governance | Unity Catalog |
+| OLTP backend | Lakebase Autoscaling (managed Postgres) |
+| AI discovery | Foundation Model APIs (Llama 3.3-70B) |
+| UC grant execution | SQL Statement Execution API |
+
+## Deploy
+
+### One-step CLI deploy (recommended for development)
+
+```bash
+git clone https://github.com/databricks-field-eng/datamarket.git
+cd datamarket/src/app
+databricks auth login --host https://your-workspace.azuredatabricks.net --profile my-profile
+./deploy.sh --profile my-profile
+```
+
+The script handles everything: Lakebase project creation, schema init, SP grants, frontend build, workspace upload, and app deploy. Prints the app URL when done.
+
+### Databricks Marketplace (coming soon)
+
+A Marketplace listing is in progress. When published, no CLI or pre-build step will be required — the app builds its frontend on first startup via `manifest.yaml`.
+
+## After Deploying
+
+A 4-step onboarding wizard guides the admin through:
+1. SQL Warehouse ID (auto-detected)
+2. Branding (app name, logo, tagline)
+3. Unity Catalog access grants for the app service principal
+4. Creating the first data product
+
+Admin access is automatic — the deployer's email is promoted to admin on first SSO login.
+
+## Requirements
+
+- Databricks workspace with Unity Catalog enabled
+- Databricks CLI (`pip install databricks-cli`)
+- Node.js ≥ 18
+- Python 3
+
+## Architecture
+
+```
+Browser → Databricks Apps (Express.js + React/Vite)
+                ↓                    ↓
+          Lakebase (Postgres)   Unity Catalog REST API
+          access requests       catalog metadata
+          approvals             GRANT execution
+          audit log
+```
+
+## Source
+
+Full source, documentation, and deploy guide: [github.com/databricks-field-eng/datamarket](https://github.com/databricks-field-eng/datamarket)

datamarket/app.js

@@ -0,0 +1,97 @@
+import express from 'express';
+import cors from 'cors';
+import helmet from 'helmet';
+import compression from 'compression';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+import { getPool, runMigrations, closePool, LAKEBASE_HOST, LAKEBASE_DB, LAKEBASE_SCHEMA, DEMO_MODE, RFA_ENABLED, SQL_WAREHOUSE_ID } from './db.js';
+import { registerRoutes as registerConfig }          from './routes/config.js';
+import { registerRoutes as registerProducts, maybeAutoDiscover } from './routes/products.js';
+import { registerRoutes as registerRequests }        from './routes/requests.js';
+import { registerRoutes as registerUsers }           from './routes/users.js';
+import { registerRoutes as registerFeatureRequests } from './routes/feature-requests.js';
+import { registerRoutes as registerAskCatalog }      from './routes/ask-catalog.js';
+import { registerRoutes as registerInsights }        from './routes/insights.js';
+import { registerRoutes as registerDemo }            from './routes/demo.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname  = path.dirname(__filename);
+
+const app  = express();
+const PORT = process.env.DATABRICKS_APP_PORT || process.env.PORT || 3000;
+
+// ─── Middleware ───────────────────────────────────────────────────────────────
+app.use(helmet({
+  contentSecurityPolicy: {
+    directives: {
+      defaultSrc: ["'self'"],
+      scriptSrc:  ["'self'", "'unsafe-inline'"],
+      styleSrc:   ["'self'", "'unsafe-inline'", "https://fonts.googleapis.com"],
+      fontSrc:    ["'self'", "https://fonts.gstatic.com"],
+      connectSrc: ["'self'", "https://*.azuredatabricks.net"],
+      imgSrc:     ["'self'", "data:", "https:"],
+    },
+  },
+}));
+app.use(compression());
+app.use(cors());
+app.use(express.json());
+app.use(express.static(path.join(__dirname, 'dist')));
+app.use('/public', express.static(path.join(__dirname, 'public')));
+
+// ─── Routes ───────────────────────────────────────────────────────────────────
+registerConfig(app);
+registerProducts(app);
+registerRequests(app);
+registerUsers(app);
+registerFeatureRequests(app);
+registerAskCatalog(app);
+registerInsights(app);
+registerDemo(app);
+
+// ─── SPA fallback ─────────────────────────────────────────────────────────────
+app.get('*', (req, res) => {
+  if (req.path.startsWith('/api/')) return res.status(404).json({ error: 'Not found' });
+  res.sendFile(path.join(__dirname, 'dist', 'index.html'));
+});
+
+app.use((err, req, res, next) => {
+  console.error(err.stack);
+  res.status(500).json({ error: 'Something went wrong!' });
+});
+
+// ─── Startup ──────────────────────────────────────────────────────────────────
+const server = app.listen(PORT, '0.0.0.0', () => {
+  console.log(`🚀 DataMarket running on 0.0.0.0:${PORT}`);
+  console.log(`🗄️  Lakebase: ${LAKEBASE_HOST}/${LAKEBASE_DB}/${LAKEBASE_SCHEMA}`);
+  console.log(`🔧 Mode: ${DEMO_MODE ? 'DEMO (persona switcher, no real grants)' : 'PRODUCTION (SSO identity, UC grants enabled)'}`);
+  console.log(`📡 RFA: ${RFA_ENABLED ? 'ENABLED' : 'disabled'} | UC Grants: ${!DEMO_MODE && SQL_WAREHOUSE_ID ? 'ENABLED' : 'disabled'}`);
+  console.log(`📊 Health: http://localhost:${PORT}/api/health`);
+  getPool()
+    .then(() => {
+      console.log('✅ Lakebase pool initialized');
+      // Retry migrations — Autoscaling Lakebase may need a moment to wake from idle
+      const tryMigrate = (attempt) => runMigrations().catch(e => {
+        if (attempt < 5) {
+          console.warn(`⚠️  Migration attempt ${attempt} failed (${e.message}) — retrying in 8s...`);
+          setTimeout(() => tryMigrate(attempt + 1), 8000);
+        } else {
+          console.warn('⚠️  Migrations skipped after 5 attempts:', e.message);
+        }
+      });
+      tryMigrate(1);
+      // Run auto-discover after migrations settle (non-blocking)
+      setTimeout(() => maybeAutoDiscover(), 15000);
+    })
+    .catch(e => console.warn('⚠️  Lakebase init deferred:', e.message));
+});
+
+process.on('SIGTERM', () => {
+  console.log('Received SIGTERM, shutting down...');
+  server.close(async () => {
+    await closePool();
+    process.exit(0);
+  });
+  setTimeout(() => process.exit(1), 14000);
+});

datamarket/app.yaml

@@ -0,0 +1,28 @@
+command:
+  - "node"
+  - "app.js"
+env:
+  # ── Databricks Identity ────────────────────────────────────────────────────
+  - name: DATABRICKS_HOST
+    value: "https://dbc-7e8bd036-20cc.cloud.databricks.com"
+
+  # ── Admin bootstrap ────────────────────────────────────────────────────────
+  # On first SSO login the app auto-promotes this email to admin.
+  # Set this to your own admin email before deploying.
+  - name: ADMIN_EMAIL
+    value: "admin@example.com"
+
+  # ── Lakebase Connection ────────────────────────────────────────────────────
+  - name: LAKEBASE_HOST
+    value: "ep-spring-dust-d8wvin0s.database.us-east-2.cloud.databricks.com"
+  - name: LAKEBASE_DB
+    value: "databricks_postgres"
+  - name: LAKEBASE_SCHEMA
+    value: "datamarket"
+  - name: LAKEBASE_ENDPOINT
+    value: "projects/datamarket/branches/production/endpoints/primary"
+
+  # ── Mode ──────────────────────────────────────────────────────────────────
+  # false = real SSO identity + UC grants. true = persona switcher (demos only)
+  - name: DEMO_MODE
+    value: "false"

datamarket/app.yaml.example

@@ -0,0 +1,41 @@
+command:
+  - "node"
+  - "app.js"
+env:
+  # ── Databricks Identity ──────────────────────────────────────────────────────
+  # DATABRICKS_TOKEN is auto-injected by Databricks Apps at runtime.
+  # Set DATABRICKS_HOST to your workspace URL.
+  - name: DATABRICKS_HOST
+    value: "https://your-workspace.azuredatabricks.net"
+
+  # ── IMPORTANT: Admin bootstrap ───────────────────────────────────────────────
+  # Set this to the deployer's email address. On first SSO login the app will
+  # automatically promote this user to admin so they can access Manage → Settings.
+  # Without this, the deployer lands as a regular analyst with no admin access.
+  # Comma-separate multiple emails for co-admins: "alice@org.com,bob@org.com"
+  - name: ADMIN_EMAIL
+    value: "your-email@company.com"
+
+  # ── Lakebase Connection ───────────────────────────────────────────────────────
+  # Autoscaling Lakebase: Get the endpoint URL from Compute → Lakebase → your project
+  - name: LAKEBASE_HOST
+    value: "ep-your-project.database.region.azuredatabricks.net"
+  - name: LAKEBASE_DB
+    value: "databricks_postgres"
+  - name: LAKEBASE_SCHEMA
+    value: "datamarket"
+  - name: LAKEBASE_ENDPOINT
+    value: "projects/your-project-name/branches/production/endpoints/primary"
+
+  # ── Mode ─────────────────────────────────────────────────────────────────────
+  # "true"  = persona switcher visible (for demos and internal POCs)
+  # "false" = real SSO identity from Databricks Apps headers (for production)
+  # NOTE: When switching from true → false, make sure ADMIN_EMAIL is set first
+  #       or you will land as a regular analyst with no way to configure the app.
+  - name: DEMO_MODE
+    value: "false"
+
+  # ── Branding & integrations ───────────────────────────────────────────────────
+  # Everything else (portal name, tagline, logo, SQL Warehouse ID, search chips,
+  # About/FAQ/Contact content) is configured via Admin → Settings in the app UI.
+  # No redeploy needed for those changes.