diff --git a/backend/app.py b/backend/app.py
index 03e354a..0a394e8 100644
--- a/backend/app.py
+++ b/backend/app.py
@@ -6,6 +6,7 @@
from typing import Dict, List, Optional, Any
import json
+import lance
import lancedb
import pyarrow as pa
from packaging.version import parse as parse_version
@@ -65,6 +66,16 @@ def get_lance_connection():
raise HTTPException(status_code=500, detail="Data path not found")
return lancedb.connect(str(DATA_PATH))
+def open_remote_dataset(uri: str):
+ try:
+ return lance.dataset(uri.strip())
+ except Exception as error:
+ logger.warning("Unable to open remote dataset: %s", error)
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unable to open dataset URI: {error}",
+ )
+
def serialize_arrow_value(value):
try:
# Stop immediately if the Arrow scalar is null
@@ -405,6 +416,123 @@ async def get_vector_preview(
logger.error(f"Error getting vector preview for {dataset_name}.{column}: {e}")
raise HTTPException(status_code=500, detail="Failed to get vector preview")
+@app.get("/dataset/schema")
+async def get_remote_dataset_schema(uri: str = Query(min_length=1)):
+ schema = open_remote_dataset(uri).schema
+ fields = []
+ for field in schema:
+ field_info = {
+ "name": field.name,
+ "type": str(field.type),
+ "nullable": field.nullable,
+ }
+ if (
+ (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type))
+ and pa.types.is_floating(field.type.value_type)
+ ):
+ field_info["vector_dim"] = None
+ fields.append(field_info)
+
+ metadata = {
+ key.decode("utf-8", errors="replace"): value.decode("utf-8", errors="replace")
+ for key, value in (schema.metadata or {}).items()
+ }
+ return {"fields": fields, "metadata": metadata}
+
+
+@app.get("/dataset/columns")
+async def get_remote_dataset_columns(uri: str = Query(min_length=1)):
+ schema = open_remote_dataset(uri).schema
+ columns = []
+ for field in schema:
+ is_vector = (
+ (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type))
+ and pa.types.is_floating(field.type.value_type)
+ )
+ column = {
+ "name": field.name,
+ "type": str(field.type),
+ "nullable": field.nullable,
+ "is_vector": is_vector,
+ }
+ if is_vector:
+ column["dim"] = None
+ columns.append(column)
+ return {"columns": columns}
+
+
+@app.get("/dataset/rows")
+async def get_remote_dataset_rows(
+ uri: str = Query(min_length=1),
+ limit: int = Query(default=50, ge=1, le=MAX_LIMIT),
+ offset: int = Query(default=0, ge=0),
+ columns: Optional[str] = Query(default=None),
+):
+ dataset = open_remote_dataset(uri)
+ schema = dataset.schema
+
+ column_list = None
+ if columns:
+ column_list = [column.strip() for column in columns.split(",") if column.strip()]
+ invalid_columns = [column for column in column_list if column not in schema.names]
+ if invalid_columns:
+ raise HTTPException(status_code=400, detail=f"Invalid columns: {invalid_columns}")
+
+ try:
+ total_count = dataset.count_rows()
+ end = min(offset + limit, total_count)
+ if offset >= total_count:
+ selected_fields = [
+ schema.field(name) for name in (column_list or schema.names)
+ ]
+ result_table = pa.table({
+ field.name: pa.array([], type=field.type)
+ for field in selected_fields
+ })
+ else:
+ result_table = dataset.take(
+ list(range(offset, end)),
+ columns=column_list,
+ )
+ except (AttributeError, TypeError):
+ result_table = dataset.to_table(columns=column_list).slice(offset, limit)
+ total_count = dataset.count_rows()
+ except Exception as read_error:
+ logger.warning(
+ "Failed to read remote dataset, returning informational row: %s",
+ read_error,
+ )
+ result_table = pa.table({
+ "error": ["Unable to read dataset"],
+ "dataset": [uri],
+ "details": [f"Error: {str(read_error)[:200]}"],
+ })
+ total_count = 1
+
+ rows = []
+ for row_index in range(result_table.num_rows):
+ row = {}
+ for column_index, column_name in enumerate(result_table.column_names):
+ try:
+ value = result_table.column(column_index)[row_index]
+ row[column_name] = serialize_arrow_value(value)
+ except Exception as serialize_error:
+ logger.warning(
+ "Failed to serialize column %s at row %s: %s",
+ column_name,
+ row_index,
+ serialize_error,
+ )
+ row[column_name] = {"error": "Failed to read value"}
+ rows.append(row)
+
+ return {
+ "rows": rows,
+ "total": total_count,
+ "limit": limit,
+ "offset": offset,
+ }
+
# Mount static files - use vanilla version by default
# In production, Docker copies vanilla files to /web
# For local development, serve from web/vanilla
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 466bb7e..0ee54bb 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -9,6 +9,7 @@ dependencies = [
"fastapi==0.104.1",
"uvicorn[standard]==0.24.0",
"lancedb==0.3.4",
+ "pylance==0.8.17",
"pyarrow==14.0.1",
"python-multipart==0.0.6",
]
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 3cee90c..14a1f2a 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -2,5 +2,5 @@ fastapi==0.104.1
uvicorn[standard]==0.24.0
python-multipart==0.0.6
numpy
-lancedb
+lancedb[pylance]
pyarrow
\ No newline at end of file
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 84f5f69..b62e7b8 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -62,6 +62,11 @@ def data_dir(tmp_path_factory):
return path
+@pytest.fixture(scope="session")
+def sample_uri(data_dir):
+ return str(data_dir / "sample.lance")
+
+
@pytest.fixture(scope="session")
def vec_nulls_preserved(data_dir):
"""Lance format v1 (lancedb 0.3.x/0.5) stores a null list as an empty
diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py
index 1e253a9..334c21e 100644
--- a/backend/tests/test_api.py
+++ b/backend/tests/test_api.py
@@ -8,6 +8,7 @@
import base64
import lancedb
+import pyarrow as pa
import pytest
from packaging.version import parse as parse_version
@@ -43,6 +44,69 @@ def test_datasets_lists_created_tables(client):
assert "broken" in names
+# /dataset/*?uri=
+
+def test_remote_dataset_uri_is_required(client):
+ assert client.get("/dataset/schema").status_code == 422
+ assert client.get("/dataset/columns").status_code == 422
+ assert client.get("/dataset/rows").status_code == 422
+
+
+def test_remote_dataset_invalid_uri_returns_400(client):
+ response = client.get("/dataset/schema", params={"uri": "/does/not/exist"})
+ assert response.status_code == 400
+ assert response.json()["detail"].startswith("Unable to open dataset URI:")
+
+
+def test_remote_dataset_schema_and_columns(client, sample_uri):
+ schema = client.get("/dataset/schema", params={"uri": sample_uri})
+ assert schema.status_code == 200
+ assert {field["name"] for field in schema.json()["fields"]} == {
+ "id", "text", "score", "blob", "vec", "embedding"
+ }
+
+ columns = client.get("/dataset/columns", params={"uri": sample_uri})
+ assert columns.status_code == 200
+ by_name = {column["name"]: column for column in columns.json()["columns"]}
+ assert by_name["vec"]["is_vector"] is True
+ assert by_name["id"]["is_vector"] is False
+
+
+def test_remote_dataset_rows(client, sample_uri):
+ response = client.get(
+ "/dataset/rows",
+ params={"uri": sample_uri, "limit": 2, "offset": 1, "columns": "id,text"},
+ )
+ assert response.status_code == 200
+ body = response.json()
+ assert body["total"] == ROWS
+ assert body["limit"] == 2
+ assert body["offset"] == 1
+ assert body["rows"] == [
+ {"id": 1, "text": "row 1"},
+ {"id": 2, "text": "row 2"},
+ ]
+
+
+def test_remote_dataset_uri_is_not_rewritten(client, monkeypatch):
+ import app as app_module
+
+ requested = []
+
+ class EmptyDataset:
+ schema = pa.schema([])
+
+ def open_dataset(uri):
+ requested.append(uri)
+ return EmptyDataset()
+
+ monkeypatch.setattr(app_module.lance, "dataset", open_dataset)
+ uri = "s3://example-bucket/path/table.lance"
+ response = client.get("/dataset/schema", params={"uri": uri})
+ assert response.status_code == 200
+ assert requested == [uri]
+
+
# /datasets/{name}/schema
def test_schema_fields(client):
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 2d2c8fe..9e0c36b 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -1,19 +1,14 @@
#!/bin/bash
set -Eeuo pipefail
-if [ ! -d "$DATA_PATH" ]; then
- echo "ERROR: Data path $DATA_PATH does not exist or is not mounted"
- exit 1
-fi
-
-if [ ! -r "$DATA_PATH" ]; then
- echo "ERROR: Data path $DATA_PATH is not readable"
- exit 1
-fi
-
+DATA_PATH="${DATA_PATH:-/data}"
PORT="${PORT:-8080}"
echo "Starting Lance Viewer on port ${PORT}..."
-echo "Data path: $DATA_PATH"
+if [ -d "$DATA_PATH" ] && [ -r "$DATA_PATH" ]; then
+ echo "Data path: $DATA_PATH"
+else
+ echo "WARNING: Mounted datasets are unavailable; remote dataset URIs can still be opened"
+fi
exec python -m uvicorn app:app --host 0.0.0.0 --port "${PORT}"
\ No newline at end of file
diff --git a/web/vanilla/app.js b/web/vanilla/app.js
index d2ef982..02d154f 100644
--- a/web/vanilla/app.js
+++ b/web/vanilla/app.js
@@ -1,6 +1,6 @@
class LanceViewer {
constructor() {
- this.currentDataset = null;
+ this.currentDatasetUri = null;
this.currentPage = 0;
this.pageSize = 50;
this.totalRows = 0;
@@ -11,13 +11,14 @@ class LanceViewer {
this.initializeElements();
this.setupEventListeners();
this.checkHealth();
- this.loadDatasets();
}
initializeElements() {
this.elements = {
healthStatus: document.getElementById('healthStatus'),
- datasetList: document.getElementById('datasetList'),
+ remoteDatasetForm: document.getElementById('remoteDatasetForm'),
+ remoteDatasetUri: document.getElementById('remoteDatasetUri'),
+ remoteDatasetStatus: document.getElementById('remoteDatasetStatus'),
datasetHeader: document.getElementById('datasetHeader'),
datasetTitle: document.getElementById('datasetTitle'),
columnSection: document.getElementById('columnSection'),
@@ -43,6 +44,10 @@ class LanceViewer {
}
setupEventListeners() {
+ this.elements.remoteDatasetForm.addEventListener('submit', (event) => {
+ event.preventDefault();
+ this.connectRemoteDataset();
+ });
this.elements.prevPage.addEventListener('click', () => this.previousPage());
this.elements.nextPage.addEventListener('click', () => this.nextPage());
this.elements.pageSize.addEventListener('change', (e) => {
@@ -93,53 +98,39 @@ class LanceViewer {
}
}
- async loadDatasets() {
- try {
- const response = await fetch(`${this.apiBase}/datasets`);
- if (!response.ok) {
- throw new Error(`API error: ${response.status} ${response.statusText}`);
- }
- const data = await response.json();
-
- this.elements.datasetList.innerHTML = '';
-
- if (data.datasets.length === 0) {
- this.elements.datasetList.innerHTML = 'No datasets found
';
- return;
- }
-
- data.datasets.forEach(dataset => {
- const item = document.createElement('div');
- item.className = 'dataset-item';
- item.textContent = dataset;
- item.addEventListener('click', () => this.selectDataset(dataset));
- this.elements.datasetList.appendChild(item);
- });
- } catch (error) {
- this.elements.datasetList.innerHTML = 'Failed to load datasets
';
- }
- }
+ async connectRemoteDataset() {
+ const uri = this.elements.remoteDatasetUri.value.trim();
+ if (!uri) return;
- async selectDataset(datasetName) {
document.querySelectorAll('.dataset-item').forEach(item => {
item.classList.remove('active');
});
+ this.currentDatasetUri = uri;
+ this.currentPage = 0;
+ this.elements.remoteDatasetStatus.textContent = 'Connecting...';
- event.target.classList.add('active');
+ if (!await this.loadSchema()) {
+ this.currentDatasetUri = null;
+ this.elements.remoteDatasetStatus.textContent = 'Unable to open dataset';
+ return;
+ }
- this.currentDataset = datasetName;
- this.currentPage = 0;
- this.elements.datasetTitle.textContent = datasetName;
+ this.elements.datasetTitle.textContent = uri.split('/').filter(Boolean).pop() || uri;
+ this.elements.datasetTitle.title = uri;
this.elements.datasetHeader.style.display = 'block';
-
- await this.loadSchema();
+ this.elements.remoteDatasetStatus.textContent = 'Connected';
await this.loadColumns();
await this.loadData();
}
+ datasetUrl(resource, params = new URLSearchParams()) {
+ params.set('uri', this.currentDatasetUri);
+ return `${this.apiBase}/dataset/${resource}?${params}`;
+ }
+
async loadSchema() {
try {
- const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/schema`);
+ const response = await fetch(this.datasetUrl('schema'));
if (!response.ok) {
throw new Error(`API error: ${response.status} ${response.statusText}`);
}
@@ -168,14 +159,16 @@ class LanceViewer {
});
this.elements.schemaSection.style.display = 'block';
+ return true;
} catch (error) {
this.showError('Failed to load schema');
+ return false;
}
}
async loadColumns() {
try {
- const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/columns`);
+ const response = await fetch(this.datasetUrl('columns'));
if (!response.ok) {
throw new Error(`API error: ${response.status} ${response.statusText}`);
}
@@ -222,7 +215,7 @@ class LanceViewer {
}
async loadData() {
- if (!this.currentDataset) return;
+ if (!this.currentDatasetUri) return;
this.showLoading();
@@ -236,7 +229,7 @@ class LanceViewer {
params.append('columns', this.selectedColumns.join(','));
}
- const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/rows?${params}`);
+ const response = await fetch(this.datasetUrl('rows', params));
if (!response.ok) {
throw new Error(`API error: ${response.status} ${response.statusText}`);
}
diff --git a/web/vanilla/index.html b/web/vanilla/index.html
index 85cd52d..06a7a77 100644
--- a/web/vanilla/index.html
+++ b/web/vanilla/index.html
@@ -18,10 +18,18 @@ Lance Data Viewer