Allow remote dataset access

backend/app.py

@@ -6,6 +6,7 @@
 from typing import Dict, List, Optional, Any
 import json
 
+import lance
 import lancedb
 import pyarrow as pa
 from packaging.version import parse as parse_version
@@ -65,6 +66,16 @@ def get_lance_connection():
         raise HTTPException(status_code=500, detail="Data path not found")
     return lancedb.connect(str(DATA_PATH))
 
+def open_remote_dataset(uri: str):
+    try:
+        return lance.dataset(uri.strip())
+    except Exception as error:
+        logger.warning("Unable to open remote dataset: %s", error)
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unable to open dataset URI: {error}",
+        )
+
 def serialize_arrow_value(value):
     try:
         # Stop immediately if the Arrow scalar is null
@@ -405,6 +416,123 @@ async def get_vector_preview(
         logger.error(f"Error getting vector preview for {dataset_name}.{column}: {e}")
         raise HTTPException(status_code=500, detail="Failed to get vector preview")
 
+@app.get("/dataset/schema")
+async def get_remote_dataset_schema(uri: str = Query(min_length=1)):
+    schema = open_remote_dataset(uri).schema
+    fields = []
+    for field in schema:
+        field_info = {
+            "name": field.name,
+            "type": str(field.type),
+            "nullable": field.nullable,
+        }
+        if (
+            (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type))
+            and pa.types.is_floating(field.type.value_type)
+        ):
+            field_info["vector_dim"] = None
+        fields.append(field_info)
+
+    metadata = {
+        key.decode("utf-8", errors="replace"): value.decode("utf-8", errors="replace")
+        for key, value in (schema.metadata or {}).items()
+    }
+    return {"fields": fields, "metadata": metadata}
+
+
+@app.get("/dataset/columns")
+async def get_remote_dataset_columns(uri: str = Query(min_length=1)):
+    schema = open_remote_dataset(uri).schema
+    columns = []
+    for field in schema:
+        is_vector = (
+            (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type))
+            and pa.types.is_floating(field.type.value_type)
+        )
+        column = {
+            "name": field.name,
+            "type": str(field.type),
+            "nullable": field.nullable,
+            "is_vector": is_vector,
+        }
+        if is_vector:
+            column["dim"] = None
+        columns.append(column)
+    return {"columns": columns}
+
+
+@app.get("/dataset/rows")
+async def get_remote_dataset_rows(
+    uri: str = Query(min_length=1),
+    limit: int = Query(default=50, ge=1, le=MAX_LIMIT),
+    offset: int = Query(default=0, ge=0),
+    columns: Optional[str] = Query(default=None),
+):
+    dataset = open_remote_dataset(uri)
+    schema = dataset.schema
+
+    column_list = None
+    if columns:
+        column_list = [column.strip() for column in columns.split(",") if column.strip()]
+        invalid_columns = [column for column in column_list if column not in schema.names]
+        if invalid_columns:
+            raise HTTPException(status_code=400, detail=f"Invalid columns: {invalid_columns}")
+
+    try:
+        total_count = dataset.count_rows()
+        end = min(offset + limit, total_count)
+        if offset >= total_count:
+            selected_fields = [
+                schema.field(name) for name in (column_list or schema.names)
+            ]
+            result_table = pa.table({
+                field.name: pa.array([], type=field.type)
+                for field in selected_fields
+            })
+        else:
+            result_table = dataset.take(
+                list(range(offset, end)),
+                columns=column_list,
+            )
+    except (AttributeError, TypeError):
+        result_table = dataset.to_table(columns=column_list).slice(offset, limit)
+        total_count = dataset.count_rows()
+    except Exception as read_error:
+        logger.warning(
+            "Failed to read remote dataset, returning informational row: %s",
+            read_error,
+        )
+        result_table = pa.table({
+            "error": ["Unable to read dataset"],
+            "dataset": [uri],
+            "details": [f"Error: {str(read_error)[:200]}"],
+        })
+        total_count = 1
+
+    rows = []
+    for row_index in range(result_table.num_rows):
+        row = {}
+        for column_index, column_name in enumerate(result_table.column_names):
+            try:
+                value = result_table.column(column_index)[row_index]
+                row[column_name] = serialize_arrow_value(value)
+            except Exception as serialize_error:
+                logger.warning(
+                    "Failed to serialize column %s at row %s: %s",
+                    column_name,
+                    row_index,
+                    serialize_error,
+                )
+                row[column_name] = {"error": "Failed to read value"}
+        rows.append(row)
+
+    return {
+        "rows": rows,
+        "total": total_count,
+        "limit": limit,
+        "offset": offset,
+    }
+
 # Mount static files - use vanilla version by default
 # In production, Docker copies vanilla files to /web
 # For local development, serve from web/vanilla

backend/pyproject.toml

@@ -9,6 +9,7 @@ dependencies = [
     "fastapi==0.104.1",
     "uvicorn[standard]==0.24.0",
     "lancedb==0.3.4",
+    "pylance==0.8.17",
     "pyarrow==14.0.1",
     "python-multipart==0.0.6",
 ]

backend/requirements.txt

@@ -2,5 +2,5 @@ fastapi==0.104.1
 uvicorn[standard]==0.24.0
 python-multipart==0.0.6
 numpy
-lancedb
+lancedb[pylance]
 pyarrow

backend/tests/conftest.py

@@ -62,6 +62,11 @@ def data_dir(tmp_path_factory):
     return path
 
 
+@pytest.fixture(scope="session")
+def sample_uri(data_dir):
+    return str(data_dir / "sample.lance")
+
+
 @pytest.fixture(scope="session")
 def vec_nulls_preserved(data_dir):
     """Lance format v1 (lancedb 0.3.x/0.5) stores a null list as an empty

backend/tests/test_api.py

@@ -8,6 +8,7 @@
 import base64
 
 import lancedb
+import pyarrow as pa
 import pytest
 from packaging.version import parse as parse_version
 
@@ -43,6 +44,69 @@ def test_datasets_lists_created_tables(client):
     assert "broken" in names
 
 
+# /dataset/*?uri=
+
+def test_remote_dataset_uri_is_required(client):
+    assert client.get("/dataset/schema").status_code == 422
+    assert client.get("/dataset/columns").status_code == 422
+    assert client.get("/dataset/rows").status_code == 422
+
+
+def test_remote_dataset_invalid_uri_returns_400(client):
+    response = client.get("/dataset/schema", params={"uri": "/does/not/exist"})
+    assert response.status_code == 400
+    assert response.json()["detail"].startswith("Unable to open dataset URI:")
+
+
+def test_remote_dataset_schema_and_columns(client, sample_uri):
+    schema = client.get("/dataset/schema", params={"uri": sample_uri})
+    assert schema.status_code == 200
+    assert {field["name"] for field in schema.json()["fields"]} == {
+        "id", "text", "score", "blob", "vec", "embedding"
+    }
+
+    columns = client.get("/dataset/columns", params={"uri": sample_uri})
+    assert columns.status_code == 200
+    by_name = {column["name"]: column for column in columns.json()["columns"]}
+    assert by_name["vec"]["is_vector"] is True
+    assert by_name["id"]["is_vector"] is False
+
+
+def test_remote_dataset_rows(client, sample_uri):
+    response = client.get(
+        "/dataset/rows",
+        params={"uri": sample_uri, "limit": 2, "offset": 1, "columns": "id,text"},
+    )
+    assert response.status_code == 200
+    body = response.json()
+    assert body["total"] == ROWS
+    assert body["limit"] == 2
+    assert body["offset"] == 1
+    assert body["rows"] == [
+        {"id": 1, "text": "row 1"},
+        {"id": 2, "text": "row 2"},
+    ]
+
+
+def test_remote_dataset_uri_is_not_rewritten(client, monkeypatch):
+    import app as app_module
+
+    requested = []
+
+    class EmptyDataset:
+        schema = pa.schema([])
+
+    def open_dataset(uri):
+        requested.append(uri)
+        return EmptyDataset()
+
+    monkeypatch.setattr(app_module.lance, "dataset", open_dataset)
+    uri = "s3://example-bucket/path/table.lance"
+    response = client.get("/dataset/schema", params={"uri": uri})
+    assert response.status_code == 200
+    assert requested == [uri]
+
+
 # /datasets/{name}/schema
 
 def test_schema_fields(client):

docker/entrypoint.sh

@@ -1,19 +1,14 @@
 #!/bin/bash
 set -Eeuo pipefail
 
-if [ ! -d "$DATA_PATH" ]; then
-    echo "ERROR: Data path $DATA_PATH does not exist or is not mounted"
-    exit 1
-fi
-
-if [ ! -r "$DATA_PATH" ]; then
-    echo "ERROR: Data path $DATA_PATH is not readable"
-    exit 1
-fi
-
+DATA_PATH="${DATA_PATH:-/data}"
 PORT="${PORT:-8080}"
 
 echo "Starting Lance Viewer on port ${PORT}..."
-echo "Data path: $DATA_PATH"
+if [ -d "$DATA_PATH" ] && [ -r "$DATA_PATH" ]; then
+    echo "Data path: $DATA_PATH"
+else
+    echo "WARNING: Mounted datasets are unavailable; remote dataset URIs can still be opened"
+fi
 
 exec python -m uvicorn app:app --host 0.0.0.0 --port "${PORT}"