diff --git a/src/README.md b/DEV.md similarity index 100% rename from src/README.md rename to DEV.md diff --git a/README.md b/README.md index c696acde..b0f5864f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Become a Sponsor](https://img.shields.io/static/v1?label=Become%20a%20Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=1ABC9C)](https://github.com/sponsors/datalayer) -# ☰ 〇 Datalayer Core +# ☰ β­• Datalayer Core

Python and Typescript libraries for Datalayer @@ -18,7 +18,7 @@ ## Overview -Datalayer Core is the foundational package that powers the [Datalayer AI Platform](https://datalayer.app/). It provides a TypesScript and Python packages as a Command Line Interface (CLI) for AI engineers, data scientists, and researchers to seamlessly integrate scalable compute runtimes into their workflows. +Datalayer Core is the foundational package that powers the [Datalayer AI Platform](https://datalayer.ai). It provides a TypesScript and Python packages as a Command Line Interface (CLI) for AI engineers, data scientists, and researchers to seamlessly integrate scalable compute runtimes into their workflows. This package serves as the base foundation used by many other Datalayer packages, containing core application classes, configuration, and unified APIs for authentication, runtime management, and code execution in cloud-based environments. @@ -66,8 +66,6 @@ npm install ## Quick Start with Python -### 1. Authentication - Set your Datalayer token as an environment variable: ```bash @@ -89,31 +87,19 @@ if client.authenticate(): print("Successfully authenticated!") ``` -### 2. Runtime, Snapshots, and Evals - -Runtime execution, snapshot workflows, and evals CLI are now documented and maintained in: +## Architecture -- [agent-runtimes README](https://github.com/datalayer/agent-runtimes/blob/main/README.md) -- [agent-runtimes CLI docs](https://agent-runtimes.datalayer.tech/cli) +Datalayer Core serves as the foundation for the entire Datalayer ecosystem: -Use `agent-runtimes` for runtime workloads (`RuntimeClient`, `@datalayer`, snapshots, and evals), and use `datalayer-core` for account/platform operations (authentication, secrets, API keys, usage, profile). +- **Base Classes**: Core application classes inherited by other Datalayer packages +- **Configuration Management**: Centralized configuration system for all Datalayer components +- **Authentication Layer**: Unified authentication across all Datalayer services +- **Runtime Abstraction**: Common interface for different types of compute runtimes +- **Resource Management**: Automatic cleanup and lifecycle management ## Examples -Examples have moved to Agent Runtimes: - -- [Agent Runtimes examples README](https://github.com/datalayer/agent-runtimes/blob/main/examples/README.md) -- [Agent Runtimes examples directory](https://github.com/datalayer/agent-runtimes/tree/main/examples) - -## Platform Integration - -Datalayer adds AI capabilities and scalable compute runtimes to your development workflows. The platform is designed to seamlessly integrate into your existing processes and supercharge your computations with the processing power you need. - -Key platform features accessible through this Client and CLI: - -- **Remote Runtimes**: Execute code on powerful remote machines with CPU, RAM, and GPU resources -- **Multiple Interfaces**: Access and consume runtimes through Python Client, CLI, or other integrated tools -- **Scalable Compute**: Dynamically scale your computational resources based on workload requirements +- [OTEL example README](./examples/otel/README.md) ## Documentation @@ -193,16 +179,6 @@ This Client is designed to be simple and extensible. We welcome contributions! P For issues and enhancement requests, please use the [GitHub issue tracker](https://github.com/datalayer/core/issues). -## Architecture - -Datalayer Core serves as the foundation for the entire Datalayer ecosystem: - -- **Base Classes**: Core application classes inherited by other Datalayer packages -- **Configuration Management**: Centralized configuration system for all Datalayer components -- **Authentication Layer**: Unified authentication across all Datalayer services -- **Runtime Abstraction**: Common interface for different types of compute runtimes -- **Resource Management**: Automatic cleanup and lifecycle management - ## Use Cases - **AI/ML Development**: Scale your machine learning workflows with cloud compute using Client or CLI @@ -219,15 +195,16 @@ This project is licensed under the [BSD 3-Clause License](https://github.com/dat - **Documentation**: [Datalayer Platform Documentation](https://datalayer.ai/docs/) - **Issues**: [GitHub Issues](https://github.com/datalayer/core/issues) -- **Community**: [Datalayer Platform](https://datalayer.app/) +- **Community**: [Datalayer Platform](https://datalayer.ai) --- -

- Datalayer Logo -

+
-

- πŸš€ AI Agents for Data Analysis

- Get started with Datalayer today! -

+**If this project is helpful to you, please give us a ⭐️** + +Made with ❀️ by [Datalayer](https://datalayer.ai) + +Datalayer Logo + +
diff --git a/datalayer_core/authn/authn.py b/datalayer_core/authn/authn.py index 41760e59..51d94170 100644 --- a/datalayer_core/authn/authn.py +++ b/datalayer_core/authn/authn.py @@ -40,13 +40,14 @@ def __init__(self, iam_url: str, storage: Optional[TokenStorage] = None): """ self.iam_url = iam_url - # Extract datalayer_url from iam_url (remove /api/iam/v1 suffix if present) - datalayer_url = iam_url.replace("/api/iam/v1", "") + # The service URL the credentials are stored under: the IAM one, + # without the path of its API. + service_url = iam_url.replace("/api/iam/v1", "") - # CRITICAL: Pass datalayer_url as service_name to KeyringStorage for backwards compatibility + # CRITICAL: Pass the service URL as service_name to KeyringStorage for backwards compatibility self.storage: TokenStorage if storage is None: - keyring_storage = KeyringStorage(service_name=datalayer_url) + keyring_storage = KeyringStorage(service_name=service_url) if keyring_storage.is_available(): self.storage = keyring_storage else: diff --git a/datalayer_core/authn/server/__main__.py b/datalayer_core/authn/server/__main__.py index cea0f606..5139ce1d 100644 --- a/datalayer_core/authn/server/__main__.py +++ b/datalayer_core/authn/server/__main__.py @@ -9,23 +9,23 @@ from typing import Optional from datalayer_core.authn.server.http_server import get_token -from datalayer_core.utils.urls import DEFAULT_DATALAYER_URL +from datalayer_core.utils.urls import DEFAULT_DATALAYER_IAM_URL logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -DATALAYER_URL = DEFAULT_DATALAYER_URL +IAM_URL = DEFAULT_DATALAYER_IAM_URL if __name__ == "__main__": from sys import argv if len(argv) == 2: - ans = get_token(DATALAYER_URL, port=int(argv[1])) + ans = get_token(IAM_URL, port=int(argv[1])) else: - ans = get_token(DATALAYER_URL) + ans = get_token(IAM_URL) handle: Optional[str] = None token: Optional[str] = None diff --git a/datalayer_core/authn/server/http_server.py b/datalayer_core/authn/server/http_server.py index f6b41b85..a779e046 100644 --- a/datalayer_core/authn/server/http_server.py +++ b/datalayer_core/authn/server/http_server.py @@ -152,7 +152,7 @@ def do_GET(self) -> None: elif path in {"/", "/datalayer/login/cli"}: config_json = json.dumps( { - "datalayerUrl": self.server.datalayer_url, # type: ignore + "iamUrl": self.server.iam_url, # type: ignore "iamUrl": self.server.iam_url, # type: ignore "whiteLabel": False, } @@ -229,8 +229,8 @@ class AuthHTTPServer(HTTPServer): The server address and port. RequestHandlerClass : Callable The request handler class. - datalayer_url : str - The runtime URL. + iam_url : str + The URL of the IAM service, which the login goes through. bind_and_activate : bool, default True Whether to bind and activate the server. """ @@ -239,7 +239,7 @@ def __init__( self, server_address: tuple[Union[str, bytes, bytearray], int], RequestHandlerClass: t.Callable[[t.Any, t.Any, t.Self], BaseRequestHandler], - datalayer_url: str, + iam_url: str, bind_and_activate: bool = True, ) -> None: """ @@ -251,14 +251,13 @@ def __init__( The server address and port. RequestHandlerClass : Callable The request handler class. - datalayer_url : str - The runtime URL. + iam_url : str + The URL of the IAM service, which the login goes through. bind_and_activate : bool, default True Whether to bind and activate the server. """ # Use DatalayerURLs for proper URL configuration - self._urls = DatalayerURLs.from_environment(datalayer_url=datalayer_url) - self.datalayer_url = self._urls.datalayer_url + self._urls = DatalayerURLs.from_environment(iam_url=iam_url) self.iam_url = self._urls.iam_url self.user_handle = None self.token = None @@ -299,15 +298,15 @@ def finish_request(self, request: t.Any, client_address: str) -> None: def get_token( - datalayer_url: str, port: Optional[int] = None, logger: logging.Logger = logger + iam_url: str, port: Optional[int] = None, logger: logging.Logger = logger ) -> Optional[tuple[str, str]]: """ Get the user handle and token. Parameters ---------- - datalayer_url : str - The runtime URL. + iam_url : str + The URL of the IAM service, which the login goes through. port : int or None, default None The port to use for the authentication server. logger : logging.Logger, default logger @@ -329,8 +328,8 @@ def get_token( ) sys.argv = [ "", - "--DatalayerExtensionApp.datalayer_url", - datalayer_url, + "--DatalayerExtensionApp.iam_url", + iam_url, "--ServerApp.disable_check_xsrf", "True", ] @@ -339,7 +338,7 @@ def get_token( # return None if httpd.token is None else (httpd.user_handle, httpd.token) return None else: - httpd = AuthHTTPServer(server_address, LoginRequestHandler, datalayer_url) + httpd = AuthHTTPServer(server_address, LoginRequestHandler, iam_url) logger.info( f"Waiting for user logging, open http://localhost:{port}. Press CTRL+C to abort.\n" ) diff --git a/datalayer_core/authn/storage.py b/datalayer_core/authn/storage.py index b9fc0060..ef5e6520 100644 --- a/datalayer_core/authn/storage.py +++ b/datalayer_core/authn/storage.py @@ -60,7 +60,7 @@ def __init__(self, service_name: str = DEFAULT_DATALAYER_IAM_URL): """Initialize keyring storage. Args: - service_name: Service name for keyring entries (MUST be datalayer_url for backwards compatibility) + service_name: Service name for keyring entries (MUST be the IAM URL for backwards compatibility) """ self.service_name = service_name self._keyring: Any = None diff --git a/datalayer_core/base/serverapplication.py b/datalayer_core/base/serverapplication.py index 7d540d9f..bbb0fb24 100644 --- a/datalayer_core/base/serverapplication.py +++ b/datalayer_core/base/serverapplication.py @@ -16,7 +16,7 @@ from datalayer_core.handlers.index.handler import IndexHandler from datalayer_core.handlers.login.handler import LoginHandler from datalayer_core.handlers.service_worker.handler import ServiceWorkerHandler -from datalayer_core.utils.urls import DEFAULT_DATALAYER_IAM_URL +from datalayer_core.utils.urls import DatalayerURLs _PACKAGE_ROOT = Path(__file__).resolve().parent.parent DEFAULT_STATIC_FILES_PATH = str(_PACKAGE_ROOT / "static") @@ -36,15 +36,110 @@ class DatalayerExtensionApp(ExtensionAppJinjaMixin, ExtensionApp): template_paths = [DEFAULT_TEMPLATE_FILES_PATH] - # datalayer_url can be set set and None or ' ' (empty string). - # In that case, the consumer of those settings are free to consider datalayer_url as null. - datalayer_url = Unicode( - DEFAULT_DATALAYER_IAM_URL, + # One URL per service: there is no single base any more. Each of them can + # be set and None or ' ' (empty string); the consumer of those settings is + # then free to consider it as null. What is not configured is resolved from + # the environment β€” `DATALAYER_IAM_URL` and friends β€” and falls back to the + # default of the service, see `DatalayerURLs`. + iam_url = Unicode( config=True, allow_none=True, - help="""URL to connect to the Datalayer RUN APIs.""", + help="""URL to connect to the Datalayer IAM API.""", ) + runtimes_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer Runtimes API.""", + ) + + spacer_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer Spacer API.""", + ) + + library_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer Library API.""", + ) + + manager_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer Manager API.""", + ) + + scheduler_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer Scheduler API.""", + ) + + ai_agents_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer AI Agents API.""", + ) + + ai_inference_url = Unicode( + config=True, + allow_none=True, + help="""URL to connect to the Datalayer AI Inference API.""", + ) + + @default("iam_url") + def _default_iam_url(self) -> str: + return self._urls.iam_url + + @default("runtimes_url") + def _default_runtimes_url(self) -> str: + return self._urls.runtimes_url + + @default("spacer_url") + def _default_spacer_url(self) -> str: + return self._urls.spacer_url + + @default("library_url") + def _default_library_url(self) -> str: + return self._urls.library_url + + @default("manager_url") + def _default_manager_url(self) -> str: + return self._urls.manager_url + + @default("scheduler_url") + def _default_scheduler_url(self) -> str: + return self._urls.scheduler_url + + @default("ai_agents_url") + def _default_ai_agents_url(self) -> str: + return self._urls.ai_agents_url + + @default("ai_inference_url") + def _default_ai_inference_url(self) -> str: + return self._urls.ai_inference_url + + @property + def _urls(self) -> DatalayerURLs: + """The URLs of the services, as the environment resolves them.""" + return DatalayerURLs.from_environment() + + @property + def service_urls(self) -> dict: + """The URL of every service, as the browser and the templates read them.""" + return { + "iam_url": self.iam_url, + "runtimes_url": self.runtimes_url, + "spacer_url": self.spacer_url, + "library_url": self.library_url, + "manager_url": self.manager_url, + "scheduler_url": self.scheduler_url, + "ai_agents_url": self.ai_agents_url, + "ai_inference_url": self.ai_inference_url, + } + white_label = Bool(False, config=True, help="""Display white label content.""") benchmarks = Bool(False, config=True, help="""Show the benchmarks page.""") @@ -63,7 +158,7 @@ class Launcher(Configurable): ) name = Unicode( - "Runtimes", + "Datalayer", config=True, help=("Application launcher card name."), ) @@ -189,7 +284,7 @@ def initialize_settings(self) -> None: self.serverapp.port = port settings = dict( - datalayer_url=self.datalayer_url, + **self.service_urls, launcher={ "category": self.launcher.category, "name": self.launcher.name, @@ -215,7 +310,7 @@ def initialize_templates(self) -> None: self.serverapp.jinja_template_vars.update( { "datalayer_version": __version__, - "datalayer_url": self.datalayer_url, + **self.service_urls, } ) diff --git a/datalayer_core/cli/__main__.py b/datalayer_core/cli/__main__.py index 5265d6e2..9ba0384a 100644 --- a/datalayer_core/cli/__main__.py +++ b/datalayer_core/cli/__main__.py @@ -4,6 +4,8 @@ """Command line interface for Datalayer based on Typer.""" import os +import shutil +import subprocess import sys import typer @@ -24,6 +26,7 @@ from datalayer_core.cli.commands.orgs import app as orgs_app from datalayer_core.cli.commands.orgs import orgs_ls from datalayer_core.cli.commands.otel import app as otel_app +from datalayer_core.cli.commands.sandboxes import app as sandboxes_app from datalayer_core.cli.commands.secrets import app as secrets_app from datalayer_core.cli.commands.secrets import secrets_ls from datalayer_core.cli.commands.subscription import app as subscription_app @@ -73,11 +76,6 @@ def main_callback( "omitted; otherwise built-in auth resolution is used." ), ), - datalayer_url: str | None = typer.Option( - None, - "--datalayer-url", - help="Override DATALAYER_URL for this CLI invocation.", - ), iam_url: str | None = typer.Option( None, "--iam-url", @@ -139,10 +137,10 @@ def main_callback( "--support-url", help="Override DATALAYER_SUPPORT_URL for this CLI invocation.", ), - mcp_server_url: str | None = typer.Option( + jupyter_mcp_server_url: str | None = typer.Option( None, - "--mcp-server-url", - help="Override DATALAYER_MCP_SERVER_URL for this CLI invocation.", + "--jupyter-mcp-server-url", + help="Override DATALAYER_JUPYTER_MCP_SERVER_URL for this CLI invocation.", ), scheduler_url: str | None = typer.Option( None, @@ -152,7 +150,6 @@ def main_callback( ) -> None: """Main callback to handle global options.""" overrides = { - "DATALAYER_URL": datalayer_url, "DATALAYER_IAM_URL": iam_url, "DATALAYER_RUNTIMES_URL": runtimes_url, "DATALAYER_SPACER_URL": spacer_url, @@ -165,7 +162,7 @@ def main_callback( "DATALAYER_SUCCESS_URL": success_url, "DATALAYER_STATUS_URL": status_url, "DATALAYER_SUPPORT_URL": support_url, - "DATALAYER_MCP_SERVER_URL": mcp_server_url, + "DATALAYER_JUPYTER_MCP_SERVER_URL": jupyter_mcp_server_url, "DATALAYER_SCHEDULER_URL": scheduler_url, } for env_name, value in overrides.items(): @@ -189,6 +186,7 @@ def main_callback( app.add_typer(orgs_app) app.add_typer(teams_app) app.add_typer(otel_app) +app.add_typer(sandboxes_app) app.add_typer(secrets_app) app.add_typer(subscription_app) app.add_typer(api_keys_app) @@ -214,7 +212,6 @@ def main_callback( _GLOBAL_OPTIONS_WITH_VALUES = { "--api-key", - "--datalayer-url", "--iam-url", "--runtimes-url", "--spacer-url", @@ -228,7 +225,7 @@ def main_callback( "--success-url", "--status-url", "--support-url", - "--mcp-server-url", + "--jupyter-mcp-server-url", "--scheduler-url", } @@ -236,6 +233,31 @@ def main_callback( "--version", } +_ROOT_COMMANDS = { + "about", + "auth", + "cluster", + "config", + "memberships", + "orgs", + "teams", + "otel", + "secrets", + "subscription", + "api-keys", + "users", + "usage", + "plans", + "web", + "login", + "logout", + "whoami", + "secrets-ls", + "api-keys-ls", + "orgs-ls", + "teams-ls", +} + def _normalize_global_options(argv: list[str]) -> list[str]: """Hoist supported global options so they work at any argument position.""" @@ -285,9 +307,59 @@ def _normalize_global_options(argv: list[str]) -> list[str]: return [argv[0], *extracted, *remaining] +def _find_root_command(args: list[str]) -> tuple[str | None, int | None]: + """Return the root command token and its index within normalized args.""" + i = 0 + while i < len(args): + token = args[i] + if token == "--": + i += 1 + break + if token in _GLOBAL_OPTIONS_NO_VALUES: + i += 1 + continue + if token in _GLOBAL_OPTIONS_WITH_VALUES: + i += 2 + continue + if any(token.startswith(f"{option}=") for option in _GLOBAL_OPTIONS_WITH_VALUES): + i += 1 + continue + if token.startswith("-"): + return None, None + return token, i + + if i < len(args): + token = args[i] + if token.startswith("-"): + return None, None + return token, i + + return None, None + + +def _try_external_command(args: list[str]) -> int | None: + """Run datalayer- when an unknown root command is invoked.""" + command, command_index = _find_root_command(args) + if command is None or command_index is None or command in _ROOT_COMMANDS: + return None + + executable = shutil.which(f"datalayer-{command}") + if executable is None: + return None + + forwarded_args = args[command_index + 1 :] + completed = subprocess.run([executable, *forwarded_args], check=False) + return completed.returncode + + def main() -> None: """Main entry point for the Datalayer Typer CLI.""" - app(args=_normalize_global_options(sys.argv)[1:]) + normalized_args = _normalize_global_options(sys.argv)[1:] + external_exit_code = _try_external_command(normalized_args) + if external_exit_code is not None: + raise SystemExit(external_exit_code) + + app(args=normalized_args) if __name__ == "__main__": diff --git a/datalayer_core/cli/commands/authn.py b/datalayer_core/cli/commands/authn.py index b803d9da..7045c864 100644 --- a/datalayer_core/cli/commands/authn.py +++ b/datalayer_core/cli/commands/authn.py @@ -183,12 +183,12 @@ def login( if access_token: # Token-based authentication console.print("πŸ”‘ Authenticating with provided token...") - asyncio.run(_login_with_token(auth, access_token, urls.datalayer_url)) + asyncio.run(_login_with_token(auth, access_token, urls.iam_url)) elif handle and password: # Credentials-based authentication console.print(f"πŸ‘€ Authenticating as {handle}...") - asyncio.run(_login_with_credentials(auth, handle, password, urls.datalayer_url)) + asyncio.run(_login_with_credentials(auth, handle, password, urls.iam_url)) else: # Try stored token first @@ -196,7 +196,7 @@ def login( if stored_token: console.print("πŸ”‘ Found stored token, validating...") try: - asyncio.run(_login_with_token(auth, stored_token, urls.datalayer_url)) + asyncio.run(_login_with_token(auth, stored_token, urls.iam_url)) return except Exception: console.print( @@ -213,7 +213,7 @@ def login( if credentials.get("credentials_type") == "api_key": asyncio.run( _login_with_api_key( - auth, credentials["api_key"], urls.datalayer_url + auth, credentials["api_key"], urls.iam_url ) ) else: @@ -222,7 +222,7 @@ def login( auth, credentials["handle"], credentials["password"], - urls.datalayer_url, + urls.iam_url, ) ) else: @@ -230,7 +230,7 @@ def login( console.print( "[yellow]No API key found. Starting browser-based authentication...[/yellow]" ) - _authenticate_with_browser(auth, urls.datalayer_url) + _authenticate_with_browser(auth, urls.iam_url) except typer.Exit: raise @@ -408,7 +408,7 @@ def logout( asyncio.run(auth.logout()) - console.print(f"πŸ‘‹ Logged out from [green]{urls.datalayer_url}[/green]") + console.print(f"πŸ‘‹ Logged out from [green]{urls.iam_url}[/green]") console.print("🧹 Stored API key cleared") except Exception as e: @@ -440,8 +440,7 @@ def whoami( if urls_only: url_items = [ - ("DATALAYER_URL", urls.datalayer_url), - ("DATALAYER_IAM_URL", urls.iam_url), + ("DATALAYER_IAM_URL", urls.iam_url), ("DATALAYER_RUNTIMES_URL", urls.runtimes_url), ("DATALAYER_SPACER_URL", urls.spacer_url), ("DATALAYER_LIBRARY_URL", urls.library_url), @@ -453,7 +452,10 @@ def whoami( ("DATALAYER_SUCCESS_URL", urls.success_url), ("DATALAYER_STATUS_URL", urls.status_url), ("DATALAYER_SUPPORT_URL", urls.support_url), - ("DATALAYER_MCP_SERVER_URL", urls.mcp_server_url), + ( + "DATALAYER_JUPYTER_MCP_SERVER_URL", + urls.jupyter_mcp_server_url, + ), ("DATALAYER_SCHEDULER_URL", urls.scheduler_url), ] console.print("[bold]Defined URLs:[/bold]") @@ -477,14 +479,13 @@ def whoami( console.print(f"πŸ‘€ User: [cyan]{handle}[/cyan]") if email: console.print(f"πŸ“§ Email: {email}") - console.print(f"🌐 Datalayer URL: [green]{urls.datalayer_url}[/green]") + console.print(f"🌐 Datalayer IAM URL: [green]{urls.iam_url}[/green]") if details: console.print("\n[bold]Detailed Information:[/bold]") url_items = [ - ("DATALAYER_URL", urls.datalayer_url), - ("DATALAYER_IAM_URL", urls.iam_url), + ("DATALAYER_IAM_URL", urls.iam_url), ("DATALAYER_RUNTIMES_URL", urls.runtimes_url), ("DATALAYER_SPACER_URL", urls.spacer_url), ("DATALAYER_LIBRARY_URL", urls.library_url), @@ -496,7 +497,10 @@ def whoami( ("DATALAYER_SUCCESS_URL", urls.success_url), ("DATALAYER_STATUS_URL", urls.status_url), ("DATALAYER_SUPPORT_URL", urls.support_url), - ("DATALAYER_MCP_SERVER_URL", urls.mcp_server_url), + ( + "DATALAYER_JUPYTER_MCP_SERVER_URL", + urls.jupyter_mcp_server_url, + ), ("DATALAYER_SCHEDULER_URL", urls.scheduler_url), ] diff --git a/datalayer_core/cli/commands/otel.py b/datalayer_core/cli/commands/otel.py index a8d165ef..1299b286 100644 --- a/datalayer_core/cli/commands/otel.py +++ b/datalayer_core/cli/commands/otel.py @@ -76,7 +76,7 @@ def _otel_base_url(url: str | None) -> str: return ( url or os.environ.get("DATALAYER_OTEL_RUN_URL") - or os.environ.get("DATALAYER_URL", "https://prod1.datalayer.run") + or os.environ.get("DATALAYER_OTEL_URL", "https://prod1.datalayer.run") ) diff --git a/datalayer_core/cli/commands/sandboxes.py b/datalayer_core/cli/commands/sandboxes.py new file mode 100644 index 00000000..58cdf262 --- /dev/null +++ b/datalayer_core/cli/commands/sandboxes.py @@ -0,0 +1,155 @@ +# Copyright (c) 2023-2025 Datalayer, Inc. +# Distributed under the terms of the Modified BSD License. + +"""Code sandbox commands for the Datalayer CLI. + +A code sandbox is a place code runs, and there is more than one kind of place: +the platform of Datalayer, a Jupyter Server, Kaggle, Modal, a container of this +machine. Each of those is a PROVIDER, each provider ships the environments it +offers β€” Datalayer ships `ai-agents-env`, Kaggle ships a CPU and a GPU session +β€” and each needs its own credentials before it can be used at all. + +What decides whether a provider is offered is therefore what this machine holds: +a token in the environment, a file its own CLI wrote. These commands report +that, so nothing is offered that would fail on the first call, and what is +missing is named rather than left to be guessed. + +The knowledge itself lives in `code_sandboxes.providers`, shared with everything +else that offers sandboxes β€” the web application and the JupyterLab extension +ask the same question and must get the same answer. +""" + +from typing import Optional + +import typer +from rich.console import Console +from rich.table import Table + +app = typer.Typer( + name="sandboxes", + help="Code sandbox providers and their environments.", + invoke_without_command=True, +) + +console = Console() + + +@app.callback() +def sandboxes_callback(ctx: typer.Context) -> None: + """Code sandbox commands.""" + if ctx.invoked_subcommand is None: + typer.echo(ctx.get_help()) + + +def _providers(): # type: ignore[no-untyped-def] + """The provider registry, or a clear failure when it is not installed.""" + try: + from code_sandboxes.providers import PROVIDERS + + return PROVIDERS + except ImportError as error: # pragma: no cover - depends on the install + console.print( + "[red]The code sandboxes are not available: " + f"{error}[/red]\nInstall them with: pip install code-sandboxes" + ) + raise typer.Exit(code=1) from error + + +@app.command(name="providers") +def list_providers( + all_providers: bool = typer.Option( + False, + "--all", + "-a", + help="Include the providers this machine has no credentials for.", + ), +) -> None: + """List the code sandbox providers, and whether they can be used here.""" + providers = _providers() + shown = [p for p in providers if all_providers or p.is_available()] + if not shown: + console.print( + "No sandbox provider is available. Run with --all to see what " + "each of them requires." + ) + return + + table = Table(title="Code Sandbox Providers") + table.add_column("Provider", style="bold") + table.add_column("Available") + table.add_column("Requires") + table.add_column("Description") + for provider in shown: + if provider.is_available(): + available = "[green]yes[/green]" + requires = "" if provider.needs_credentials else "nothing" + else: + available = "[yellow]no[/yellow]" + # Every way of satisfying it, since any one of them is enough. + requires = "\n".join( + requirement.hint for requirement in provider.missing() + ) + # Escaped: square brackets are markup to rich, and an extra written + # plainly came out as `pip install code-sandboxes` with the extra gone. + extra = ( + f" (pip install code-sandboxes\\[{provider.extra}])" + if provider.extra + else "" + ) + table.add_row( + provider.name, available, requires, f"{provider.description}{extra}" + ) + console.print(table) + + +@app.command(name="environments") +def list_environments( + provider: Optional[str] = typer.Argument( + None, + help="Only the environments of that provider; all of them by default.", + ), + all_providers: bool = typer.Option( + False, + "--all", + "-a", + help="Include providers this machine has no credentials for.", + ), +) -> None: + """List the environments the providers ship.""" + providers = _providers() + if provider: + from code_sandboxes.providers import get_provider + + found = get_provider(provider) + if found is None: + console.print(f"[red]No such sandbox provider: {provider}[/red]") + raise typer.Exit(code=1) + selection = [found] + else: + selection = [p for p in providers if all_providers or p.is_available()] + + table = Table(title="Code Sandbox Environments") + table.add_column("Provider", style="bold") + table.add_column("Environment") + table.add_column("Title") + table.add_column("Language") + rows = 0 + for entry in selection: + if not entry.is_available() and not all_providers and not provider: + continue + for environment in entry.environments(): + table.add_row( + entry.name, + environment.name, + environment.title, + environment.language, + ) + rows += 1 + if not rows: + console.print( + "No environment to show. A provider lists its environments only " + "once its credentials are in place β€” `datalayer sandboxes " + "providers --all` says what each one needs." + ) + return + console.print(table) diff --git a/datalayer_core/cli/commands/web.py b/datalayer_core/cli/commands/web.py index db8ea62b..13c824c2 100644 --- a/datalayer_core/cli/commands/web.py +++ b/datalayer_core/cli/commands/web.py @@ -29,10 +29,10 @@ def web_callback(ctx: typer.Context) -> None: @app.command(name="start") def web_start( - datalayer_url: Optional[str] = typer.Option( + iam_url: Optional[str] = typer.Option( None, - "--datalayer-url", - help="Datalayer URL", + "--iam-url", + help="Datalayer IAM URL", ), disable_xsrf: bool = typer.Option( True, @@ -43,18 +43,18 @@ def web_start( """Launch the Datalayer web application.""" try: # Get URLs configuration - urls = DatalayerURLs.from_environment(datalayer_url=datalayer_url) + urls = DatalayerURLs.from_environment(iam_url=iam_url) # Prepare arguments for Jupyter server sys.argv = [ "", f"--ServerApp.disable_check_xsrf={disable_xsrf}", "--DatalayerExtensionApp.webapp=True", - f"--DatalayerExtensionApp.datalayer_url={urls.datalayer_url}", + f"--DatalayerExtensionApp.iam_url={urls.iam_url}", ] console.print("[green]Starting Datalayer web application...[/green]") - console.print(f"Datalayer URL: {urls.datalayer_url}") + console.print(f"Datalayer IAM URL: {urls.iam_url}") console.print("[yellow]Press Ctrl+C to stop the server[/yellow]") # Launch the Jupyter server @@ -71,10 +71,10 @@ def web_start( @app.callback(invoke_without_command=True) def web_callback_default( ctx: typer.Context, - datalayer_url: Optional[str] = typer.Option( + iam_url: Optional[str] = typer.Option( None, - "--datalayer-url", - help="Datalayer Datalayer URL", + "--iam-url", + help="Datalayer IAM URL", ), disable_xsrf: bool = typer.Option( True, @@ -85,7 +85,7 @@ def web_callback_default( """Launch the Datalayer web application (default behavior).""" if ctx.invoked_subcommand is None: # Call web_start with the same parameters - web_start(datalayer_url=datalayer_url, disable_xsrf=disable_xsrf) + web_start(iam_url=iam_url, disable_xsrf=disable_xsrf) if __name__ == "__main__": diff --git a/datalayer_core/client/client.py b/datalayer_core/client/client.py index f18308aa..0b5b8d89 100644 --- a/datalayer_core/client/client.py +++ b/datalayer_core/client/client.py @@ -15,10 +15,12 @@ from datalayer_core.mixins.authn import AuthnMixin from datalayer_core.mixins.secrets import SecretsMixin from datalayer_core.mixins.api_keys import ApiKeysMixin +from datalayer_core.mixins.spaces import SpacesMixin from datalayer_core.mixins.usage import UsageMixin from datalayer_core.mixins.whoami import WhoamiAppMixin from datalayer_core.models import UserModel from datalayer_core.models.api_key import ApiKeyModel, ApiKeyType +from datalayer_core.models.space import ItemModel, SpaceModel from datalayer_core.models.secret import SecretModel, SecretVariant from datalayer_core.utils.urls import DatalayerURLs @@ -29,6 +31,7 @@ class DatalayerClient( AuthnMixin, SecretsMixin, ApiKeysMixin, + SpacesMixin, UsageMixin, WhoamiAppMixin, ): @@ -333,6 +336,40 @@ def list_api_keys(self) -> list[ApiKeyModel]: return api_key_objects return [] + def list_spaces(self) -> list[SpaceModel]: + """ + List the spaces of the authenticated user. + + The items of each space are included, so a caller wanting notebooks + does not have to ask again per space. + + Returns + ------- + list[SpaceModel] + The spaces this user can reach, empty when the call fails. + """ + response = self._list_spaces() + if response.get("success"): + return [SpaceModel.from_response(s) for s in response.get("spaces", [])] + return [] + + def list_notebooks(self) -> list[ItemModel]: + """ + List the notebooks of the authenticated user, across their spaces. + + "Which notebooks do I have" is one question, and answering it with + "first, which spaces do you have" is a round trip the caller should + not have to make. Each notebook carries the space it belongs to. + + Returns + ------- + list[ItemModel] + Every notebook this user can reach. + """ + return [ + notebook for space in self.list_spaces() for notebook in space.notebooks() + ] + def delete_api_key(self, api_key: Union[str, ApiKeyModel]) -> bool: """ Delete a specific API key. diff --git a/datalayer_core/displays/me.py b/datalayer_core/displays/me.py index 76566fd1..27376b39 100644 --- a/datalayer_core/displays/me.py +++ b/datalayer_core/displays/me.py @@ -31,7 +31,7 @@ def display_me(me: dict[str, str], infos: dict[str, str]) -> None: me["handle_s"], me["first_name_t"], me["last_name_t"], - infos.get("datalayer_url"), + infos.get("iam_url"), ) console = Console() console.print(table) diff --git a/datalayer_core/handlers/config/handler.py b/datalayer_core/handlers/config/handler.py index 8c29ab8c..0a923a22 100644 --- a/datalayer_core/handlers/config/handler.py +++ b/datalayer_core/handlers/config/handler.py @@ -22,7 +22,8 @@ def get(self) -> None: """Return the configuration of the server extension.""" settings = self.settings["datalayer"] configuration = dict( - datalayer_url=settings.datalayer_url, + # One URL per service; there is no single base any more. + **settings.service_urls, launcher={ "category": settings.launcher.category, "name": settings.launcher.name, diff --git a/datalayer_core/mixins/__init__.py b/datalayer_core/mixins/__init__.py index 4ca783c8..99597921 100644 --- a/datalayer_core/mixins/__init__.py +++ b/datalayer_core/mixins/__init__.py @@ -4,10 +4,12 @@ from .secrets import SecretsMixin from .api_keys import ApiKeysMixin from .usage import UsageMixin +from .spaces import SpacesMixin from .whoami import WhoamiAppMixin __all__ = [ "AuthnMixin", + "SpacesMixin", "SecretsMixin", "ApiKeysMixin", "UsageMixin", diff --git a/datalayer_core/mixins/authn.py b/datalayer_core/mixins/authn.py index 89f970e6..5356e80c 100644 --- a/datalayer_core/mixins/authn.py +++ b/datalayer_core/mixins/authn.py @@ -18,7 +18,7 @@ class AuthnMixin: Provide authentication methods for Datalayer client. This mixin expects the implementing class to provide: - - urls property: DatalayerURLs instance with datalayer_url and iam_url + - urls property: DatalayerURLs instance with iam_url and the other service URLs """ @property @@ -71,9 +71,9 @@ def _get_api_key(self) -> Optional[str]: try: import keyring - stored_api_key = keyring.get_password( - self.urls.datalayer_url, "access_token" - ) + # The credentials are stored under the IAM URL, which is what + # issued them. + stored_api_key = keyring.get_password(self.urls.iam_url, "access_token") if stored_api_key: self._api_key = stored_api_key return self._api_key diff --git a/datalayer_core/mixins/spaces.py b/datalayer_core/mixins/spaces.py new file mode 100644 index 00000000..747a1554 --- /dev/null +++ b/datalayer_core/mixins/spaces.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023-2025 Datalayer, Inc. +# Distributed under the terms of the Modified BSD License. + +"""Reading the spaces of a user, and the items they hold.""" + +from typing import Any + + +class SpacesListMixin: + """Mixin for listing spaces in Datalayer.""" + + def _list_spaces(self) -> dict[str, Any]: + """ + List the spaces of the authenticated user. + + The items of each space come back nested in the response, so this one + call answers both "which spaces" and "what is in them". + + Returns + ------- + dict + The platform response, carrying a ``spaces`` list on success. + """ + try: + response = self._fetch( # type: ignore + "{}/api/spacer/v1/spaces/users/me".format(self.urls.spacer_url), # type: ignore + method="GET", + ) + return response.json() + except RuntimeError as e: + return {"success": False, "message": str(e)} + + +class SpacesMixin(SpacesListMixin): + """Mixin bringing together the space operations.""" diff --git a/datalayer_core/models/space.py b/datalayer_core/models/space.py new file mode 100644 index 00000000..5bfebf9a --- /dev/null +++ b/datalayer_core/models/space.py @@ -0,0 +1,127 @@ +# Copyright (c) 2023-2025 Datalayer, Inc. +# Distributed under the terms of the Modified BSD License. + +"""Spaces and the items they hold.""" + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class ItemModel: + """An item in a space: a notebook, a cell, a document, a dataset.""" + + uid: str + name: str + kind: str + space_uid: str = "" + space_name: str = "" + description: str = "" + #: The file name of a notebook, which differs from its display name. + notebook_name: str = "" + + @classmethod + def from_response( + cls, data: dict[str, Any], space_uid: str = "", space_name: str = "" + ) -> "ItemModel": + """ + Build an item from the platform's response. + + The platform stores Solr documents, so fields arrive with type + suffixes β€” ``name_t``, ``type_s``. Both spellings are accepted so a + caller is not broken by a field being renamed underneath it. + + Parameters + ---------- + data : dict + One item, as the platform returned it. + space_uid : str + The space this item belongs to. + space_name : str + That space's display name, carried so a caller listing notebooks + across spaces can say where each one lives. + + Returns + ------- + ItemModel + The item. + """ + return cls( + uid=data.get("uid", ""), + name=data.get("name_t") or data.get("name") or "", + kind=data.get("type_s") or data.get("type") or "", + space_uid=space_uid, + space_name=space_name, + description=data.get("description_t") or data.get("description") or "", + notebook_name=data.get("notebook_name_s") or "", + ) + + def is_notebook(self) -> bool: + """ + Whether this item is a notebook. + + Returns + ------- + bool + True when the platform typed it as a notebook. + """ + return self.kind.lower() == "notebook" + + def __str__(self) -> str: + return f"{self.name} ({self.uid})" + + +@dataclass +class SpaceModel: + """A space, and the items it holds.""" + + uid: str + name: str + handle: str + description: str = "" + items: list[ItemModel] = field(default_factory=list) + + @classmethod + def from_response(cls, data: dict[str, Any]) -> "SpaceModel": + """ + Build a space, and the items nested in it, from the platform response. + + Parameters + ---------- + data : dict + One space, as the platform returned it. + + Returns + ------- + SpaceModel + The space, with its items. + """ + uid = data.get("uid", "") + name = data.get("name_t") or data.get("name") or "" + space = cls( + uid=uid, + name=name, + handle=data.get("handle_s") or data.get("handle") or "", + description=data.get("description_t") or data.get("description") or "", + ) + # Items come back nested in the space, so listing spaces already + # answers "what is in them" without a call per space. + space.items = [ + ItemModel.from_response(item, space_uid=uid, space_name=name) + for item in (data.get("items") or []) + ] + return space + + def notebooks(self) -> list[ItemModel]: + """ + The notebooks in this space. + + Returns + ------- + list[ItemModel] + Only the items the platform typed as notebooks. + """ + return [item for item in self.items if item.is_notebook()] + + def __str__(self) -> str: + return f"{self.name} ({self.uid})" diff --git a/datalayer_core/otel/emitter.py b/datalayer_core/otel/emitter.py index 8e5ff67a..05487a70 100644 --- a/datalayer_core/otel/emitter.py +++ b/datalayer_core/otel/emitter.py @@ -85,7 +85,7 @@ def __init__( otlp_base = ( ( os.environ.get("DATALAYER_OTEL_URL") - or os.environ.get("DATALAYER_URL") + or os.environ.get("DATALAYER_OTEL_URL") or "https://prod1.datalayer.run" ).rstrip("/") + "/api/otel/v1/otlp" diff --git a/datalayer_core/otel/logfire.py b/datalayer_core/otel/logfire.py index 93714eb9..bee4215a 100644 --- a/datalayer_core/otel/logfire.py +++ b/datalayer_core/otel/logfire.py @@ -22,7 +22,7 @@ 1. ``DATALAYER_OTLP_URL`` β€” explicit full base URL 2. ``DATALAYER_OTEL_RUN_URL`` β€” run URL, appends ``/api/otel/v1/otlp`` -3. ``DATALAYER_URL`` β€” fallback run URL, appends ``/api/otel/v1/otlp`` +3. ``DATALAYER_OTEL_URL`` β€” the OTEL service, appends ``/api/otel/v1/otlp`` 4. ``https://prod1.datalayer.run`` β€” production default Authentication reads ``DATALAYER_API_KEY`` as a Bearer token. The JWT payload is @@ -62,12 +62,12 @@ def otlp_endpoint() -> str: explicit = os.environ.get("DATALAYER_OTLP_URL") if explicit: return explicit.rstrip("/") - datalayer_url = ( + otel_url = ( os.environ.get("DATALAYER_OTEL_RUN_URL") - or os.environ.get("DATALAYER_URL") + or os.environ.get("DATALAYER_OTEL_URL") or "https://prod1.datalayer.run" ) - return datalayer_url.rstrip("/") + "/api/otel/v1/otlp" + return otel_url.rstrip("/") + "/api/otel/v1/otlp" def decode_user_uid(token: str) -> str | None: diff --git a/datalayer_core/templates/index.html b/datalayer_core/templates/index.html index 607c1c73..3c550079 100644 --- a/datalayer_core/templates/index.html +++ b/datalayer_core/templates/index.html @@ -11,9 +11,14 @@