diff --git a/CHANGELOG.md b/CHANGELOG.md index 270ced35c..2e4eb4c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,12 @@ - [Rust](https://github.com/moonrepo/plugins/blob/master/tools/rust/CHANGELOG.md) - [Schema (TOML, JSON, YAML)](https://github.com/moonrepo/plugins/blob/master/tools/internal-schema/CHANGELOG.md) +## Unreleased + +#### 🚀 Updates + +- Added support for base64 encoded `data://` locators for plugins. This is primarily for tools built around proto, like moon. + ## 0.55.4 #### 🚀 Updates diff --git a/Cargo.lock b/Cargo.lock index c16387c0f..93e00796b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3856,9 +3856,9 @@ dependencies = [ [[package]] name = "schematic" -version = "0.19.4" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc7d9aef1d4faaa9fb78e2976423d24b0961cb15dfd61ed7785fdbffd89e9fa" +checksum = "cc12ebacdd2b93bc506fffc0376c1d8b04b35bce54e5ed6f3228c6e41d346921" dependencies = [ "convert_case", "garde", @@ -3883,9 +3883,9 @@ dependencies = [ [[package]] name = "schematic_macros" -version = "0.19.3" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de6118658d45bba6f7eb5c050a4af41dca2760f79cf3543e276ba8e80624984" +checksum = "22ef30d52d6ce2b34b0dbedf49efaaec437c93aada037e575e9831d95de8f82a" dependencies = [ "convert_case", "darling 0.23.0", @@ -3896,9 +3896,9 @@ dependencies = [ [[package]] name = "schematic_types" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca3461a8dd2ba8439cbd64ae546f95b32c08cb6a8c81ef18986d125710d291" +checksum = "87f45f7fb23e4533d3a0a9f5b27a8bb897a7b3b410730f0c1ee99025bf8b06aa" dependencies = [ "indexmap", "regex", @@ -5243,6 +5243,7 @@ name = "warpgate" version = "0.29.3" dependencies = [ "async-trait", + "base64 0.22.1", "compact_str 0.9.0", "docker_credential", "extism", diff --git a/Cargo.toml b/Cargo.toml index ad43b6799..0feda107c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ rmcp = { version = "1.1.0", default-features = false, features = [ rustc-hash = "2.1.1" scc = "3.6.9" schemars = "1.2.1" -schematic = { version = "0.19.4", default-features = false } +schematic = { version = "0.19.6", default-features = false } semver = { version = "1.0.27", features = ["serde"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.49", features = ["preserve_order"] } diff --git a/crates/cli/src/components/locator.rs b/crates/cli/src/components/locator.rs index 23c5fcd4a..f7ec144c7 100644 --- a/crates/cli/src/components/locator.rs +++ b/crates/cli/src/components/locator.rs @@ -10,6 +10,17 @@ pub struct LocatorProps<'a> { #[component] pub fn Locator<'a>(props: &LocatorProps<'a>) -> impl Into> + use<'a> { match props.value.as_ref().expect("`value` prop is required!") { + PluginLocator::Data(_) => element! { + Entry( + name: "Source", + value: element! { + StyledText( + content: "(built-in plugin)", + ) + }.into_any() + ) + } + .into_any(), PluginLocator::File(file) => element! { Entry( name: "Source file", diff --git a/crates/cli/src/workflows/exec_workflow.rs b/crates/cli/src/workflows/exec_workflow.rs index 5dd174080..84c1f37b5 100644 --- a/crates/cli/src/workflows/exec_workflow.rs +++ b/crates/cli/src/workflows/exec_workflow.rs @@ -393,8 +393,8 @@ async fn prepare_tool( .globals_dir .as_ref() .map(|dir| tool.to_virtual_path(dir)), - globals_prefix: locations.globals_prefix, - passthrough_args: params.passthrough_args, + globals_prefix: locations.globals_prefix.as_deref(), + passthrough_args: params.passthrough_args.iter().map(|a| a.as_str()).collect(), }, ) .await?; diff --git a/crates/cli/src/workflows/install_workflow.rs b/crates/cli/src/workflows/install_workflow.rs index e667e80d3..722ca60f4 100644 --- a/crates/cli/src/workflows/install_workflow.rs +++ b/crates/cli/src/workflows/install_workflow.rs @@ -242,7 +242,11 @@ impl InstallWorkflow { InstallHook { context: tool.create_plugin_context(spec), forced: params.force, - passthrough_args: params.passthrough_args.clone(), + passthrough_args: params + .passthrough_args + .iter() + .map(|a| a.as_str()) + .collect(), pinned: params.pin_to.is_some(), quiet: params.quiet, }, @@ -373,7 +377,11 @@ impl InstallWorkflow { InstallHook { context: tool.create_plugin_context(spec), forced: params.force, - passthrough_args: params.passthrough_args.clone(), + passthrough_args: params + .passthrough_args + .iter() + .map(|a| a.as_str()) + .collect(), pinned: params.pin_to.is_some(), quiet: params.quiet, }, @@ -438,7 +446,7 @@ impl InstallWorkflow { PluginFunction::SyncShellProfile, SyncShellProfileInput { context: tool.create_plugin_context(spec), - passthrough_args: params.passthrough_args.clone(), + passthrough_args: params.passthrough_args.iter().map(|a| a.as_str()).collect(), }, ) .await?; diff --git a/crates/cli/tests/exec_test.rs b/crates/cli/tests/exec_test.rs index 9b6e713c6..d53eef1fb 100644 --- a/crates/cli/tests/exec_test.rs +++ b/crates/cli/tests/exec_test.rs @@ -72,7 +72,7 @@ mod exec { assert .success() - .stdout(predicate::str::contains("v20.20.1")); + .stdout(predicate::str::contains("v20.20.2")); } #[test] @@ -107,7 +107,7 @@ bun = "1.2" assert .inner - .stdout(predicate::str::contains("v20.20.1").and(predicate::str::contains("1.2.23"))); + .stdout(predicate::str::contains("v20.20.2").and(predicate::str::contains("1.2.23"))); } #[test] @@ -141,7 +141,7 @@ bun = "1.2" assert .inner - .stdout(predicate::str::contains("v20.20.1").and(predicate::str::contains("1.2.23"))); + .stdout(predicate::str::contains("v20.20.2").and(predicate::str::contains("1.2.23"))); } #[test] diff --git a/crates/core/src/flow/detect.rs b/crates/core/src/flow/detect.rs index f6a7c3584..32cd887f8 100644 --- a/crates/core/src/flow/detect.rs +++ b/crates/core/src/flow/detect.rs @@ -5,6 +5,7 @@ use crate::tool::Tool; use crate::tool_spec::ToolSpec; use proto_pdk_api::*; use starbase_utils::fs; +use std::borrow::Cow; use std::env; use std::path::{Path, PathBuf}; use tracing::{debug, instrument, trace}; @@ -151,9 +152,9 @@ impl<'tool> Detector<'tool> { .call_func_with( PluginFunction::ParseVersionFile, ParseVersionFileInput { - content, + content: Cow::Borrowed(&content), context: self.tool.create_plugin_unresolved_context(), - file: file.clone(), + file: Cow::Borrowed(&file), path: self.tool.to_virtual_path(&file_path), }, ) diff --git a/crates/pdk-api/src/api/build.rs b/crates/pdk-api/src/api/build.rs index e9b7d14d3..43e9a447c 100644 --- a/crates/pdk-api/src/api/build.rs +++ b/crates/pdk-api/src/api/build.rs @@ -5,9 +5,9 @@ use rustc_hash::FxHashMap; use semver::VersionReq; use std::path::PathBuf; use system_env::SystemDependency; -use warpgate_api::{Id, VirtualPath, api_enum, api_struct}; +use warpgate_api::{Id, VirtualPath, api_enum, api_input_struct, api_output_struct, api_struct}; -api_struct!( +api_input_struct!( /// Input passed to the `build_instructions` function. pub struct BuildInstructionsInput { /// Current tool context. @@ -141,7 +141,7 @@ api_enum!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `build_instructions` function. #[serde(default)] pub struct BuildInstructionsOutput { diff --git a/crates/pdk-api/src/api/mod.rs b/crates/pdk-api/src/api/mod.rs index f54014fc3..410a8088d 100644 --- a/crates/pdk-api/src/api/mod.rs +++ b/crates/pdk-api/src/api/mod.rs @@ -6,6 +6,7 @@ use crate::shapes::*; use derive_setters::Setters; use rustc_hash::FxHashMap; use schematic::Schema; +use std::borrow::Cow; use std::path::PathBuf; use version_spec::{CalVer, SemVer, SpecError, UnresolvedVersionSpec, VersionSpec}; use warpgate_api::*; @@ -213,7 +214,7 @@ pub(crate) fn is_default(value: &T) -> bool { value == &T::default() } -api_struct!( +api_input_struct!( /// Information about the current state of the plugin, /// after a version has been resolved. pub struct PluginContext { @@ -232,7 +233,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Information about the current state of the plugin, /// before a version has been resolved. pub struct PluginUnresolvedContext { @@ -259,19 +260,19 @@ api_struct!( api_unit_enum!( /// Supported types of plugins. pub enum PluginType { - #[serde(alias = "CLI", alias = "CommandLine")] // TEMP + #[serde(alias = "CLI", alias = "CommandLine")] CommandLine, #[default] #[serde(alias = "Language")] Language, - #[serde(alias = "PM", alias = "DependencyManager")] // TEMP + #[serde(alias = "PM", alias = "DependencyManager")] DependencyManager, - #[serde(alias = "VM", alias = "VersionManager")] // TEMP + #[serde(alias = "VM", alias = "VersionManager")] VersionManager, } ); -api_struct!( +api_input_struct!( /// Input passed to the `register_tool` function. pub struct RegisterToolInput { /// ID of the tool, as it was configured. @@ -279,9 +280,6 @@ api_struct!( } ); -#[deprecated(note = "Use `RegisterToolInput` instead.")] -pub type ToolMetadataInput = RegisterToolInput; - api_struct!( /// Controls aspects of the tool inventory. #[serde(default)] @@ -382,12 +380,9 @@ api_struct!( } ); -#[deprecated(note = "Use `RegisterToolOutput` instead.")] -pub type ToolMetadataOutput = RegisterToolOutput; - pub type ConfigSchema = Schema; -api_struct!( +api_output_struct!( /// Output returned from the `define_tool_config` function. pub struct DefineToolConfigOutput { /// Schema shape of the tool's configuration. @@ -397,7 +392,7 @@ api_struct!( // BACKEND -api_struct!( +api_input_struct!( /// Input passed to the `register_backend` function. pub struct RegisterBackendInput { /// Current tool context. @@ -408,7 +403,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `register_backend` function. pub struct RegisterBackendOutput { /// Unique identifier for this backend. Will be used as the folder name. @@ -425,7 +420,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned from the `define_backend_config` function. pub struct DefineBackendConfigOutput { /// Schema shape of the backend's configuration. @@ -435,7 +430,7 @@ api_struct!( // VERSION DETECTION/PINNING -api_struct!( +api_input_struct!( /// Input passed to the `detect_version_files` function. pub struct DetectVersionInput { /// Current tool context. @@ -443,7 +438,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `detect_version_files` function. #[serde(default)] pub struct DetectVersionOutput { @@ -457,24 +452,26 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `parse_version_file` function. - pub struct ParseVersionFileInput { + pub struct ParseVersionFileInput<'a> { /// File contents to parse/extract a version from. - pub content: String, + #[serde(borrow)] + pub content: Cow<'a, str>, /// Current tool context. pub context: PluginUnresolvedContext, /// Name of file that's being parsed. - pub file: String, + #[serde(borrow)] + pub file: Cow<'a, str>, /// Virtual path to the file being parsed. pub path: VirtualPath, } ); -api_struct!( +api_output_struct!( /// Output returned by the `parse_version_file` function. #[serde(default)] pub struct ParseVersionFileOutput { @@ -485,7 +482,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `pin_version` function. pub struct PinVersionInput { /// Current tool context. @@ -499,7 +496,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `pin_version` function. #[serde(default)] pub struct PinVersionOutput { @@ -516,7 +513,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `unpin_version` function. pub struct UnpinVersionInput { /// Current tool context. @@ -527,7 +524,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `unpin_version` function. #[serde(default)] pub struct UnpinVersionOutput { @@ -549,7 +546,7 @@ api_struct!( // DOWNLOAD, BUILD, INSTALL, VERIFY -api_struct!( +api_input_struct!( /// Input passed to the `native_install` function. pub struct NativeInstallInput { /// Current tool context. @@ -563,7 +560,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `native_install` function. pub struct NativeInstallOutput { /// A checksum/hash that was generated. @@ -583,7 +580,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `native_uninstall` function. pub struct NativeUninstallInput { /// Current tool context. @@ -594,7 +591,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `native_uninstall` function. pub struct NativeUninstallOutput { /// Error message if the uninstall failed. @@ -610,7 +607,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `download_prebuilt` function. pub struct DownloadPrebuiltInput { /// Current tool context. @@ -621,7 +618,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `download_prebuilt` function. pub struct DownloadPrebuiltOutput { /// Name of the direct folder within the archive that contains the tool, @@ -662,7 +659,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `unpack_archive` function. pub struct UnpackArchiveInput { /// Current tool context. @@ -676,8 +673,8 @@ api_struct!( } ); -api_struct!( - /// Output returned by the `verify_checksum` function. +api_input_struct!( + /// Input passed to the `verify_checksum` function. pub struct VerifyChecksumInput { /// Current tool context. pub context: PluginContext, @@ -696,7 +693,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `verify_checksum` function. pub struct VerifyChecksumOutput { /// Was the checksum correct? @@ -706,7 +703,7 @@ api_struct!( // EXECUTABLES, BINARYS, GLOBALS -api_struct!( +api_input_struct!( /// Input passed to the `locate_executables` function. pub struct LocateExecutablesInput { /// Current tool context. @@ -803,7 +800,7 @@ impl ExecutableConfig { } } -api_struct!( +api_output_struct!( /// Output returned by the `locate_executables` function. #[serde(default)] pub struct LocateExecutablesOutput { @@ -834,7 +831,7 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `activate_environment` function. pub struct ActivateEnvironmentInput { /// Current tool context. @@ -845,7 +842,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `activate_environment` function. #[serde(default)] pub struct ActivateEnvironmentOutput { @@ -861,7 +858,7 @@ api_struct!( // VERSION RESOLVING -api_struct!( +api_input_struct!( /// Input passed to the `load_versions` function. pub struct LoadVersionsInput { /// Current tool context. @@ -939,7 +936,7 @@ impl LoadVersionsOutput { } } -api_struct!( +api_input_struct!( /// Input passed to the `resolve_version` function. pub struct ResolveVersionInput { /// Current tool context. @@ -950,7 +947,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `resolve_version` function. #[serde(default)] pub struct ResolveVersionOutput { @@ -967,7 +964,7 @@ api_struct!( // MISCELLANEOUS -api_struct!( +api_input_struct!( /// Input passed to the `sync_manifest` function. pub struct SyncManifestInput { /// Current tool context. @@ -975,7 +972,7 @@ api_struct!( } ); -api_struct!( +api_output_struct!( /// Output returned by the `sync_manifest` function. #[serde(default)] pub struct SyncManifestOutput { @@ -989,18 +986,19 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `sync_shell_profile` function. - pub struct SyncShellProfileInput { + pub struct SyncShellProfileInput<'a> { /// Current tool context. pub context: PluginContext, /// Arguments passed after `--` that was directly passed to the tool's executable. - pub passthrough_args: Vec, + #[serde(borrow)] + pub passthrough_args: Vec<&'a str>, } ); -api_struct!( +api_output_struct!( /// Output returned by the `sync_shell_profile` function. pub struct SyncShellProfileOutput { /// An environment variable to check for in the shell profile. diff --git a/crates/pdk-api/src/hooks.rs b/crates/pdk-api/src/hooks.rs index 8b3b03d73..2f146da8e 100644 --- a/crates/pdk-api/src/hooks.rs +++ b/crates/pdk-api/src/hooks.rs @@ -54,10 +54,10 @@ impl AsRef for HookFunction { } } -api_struct!( +api_input_struct!( /// Input passed to the `pre_install` and `post_install` hooks, /// while a `proto install` command is running. - pub struct InstallHook { + pub struct InstallHook<'a> { /// Current tool context. pub context: PluginContext, @@ -65,7 +65,8 @@ api_struct!( pub forced: bool, /// Arguments passed after `--` that was directly passed to the tool's executable. - pub passthrough_args: Vec, + #[serde(borrow)] + pub passthrough_args: Vec<&'a str>, /// Whether the resolved version was pinned. pub pinned: bool, @@ -75,10 +76,10 @@ api_struct!( } ); -api_struct!( +api_input_struct!( /// Input passed to the `pre_run` hook, before a `proto run` command /// or language executable is ran. - pub struct RunHook { + pub struct RunHook<'a> { /// Current tool context. pub context: PluginContext, @@ -86,14 +87,14 @@ api_struct!( pub globals_dir: Option, /// A prefix applied to the file names of globally installed packages. - pub globals_prefix: Option, + pub globals_prefix: Option<&'a str>, /// Arguments passed after `--` that was directly passed to the tool's executable. - pub passthrough_args: Vec, + pub passthrough_args: Vec<&'a str>, } ); -api_struct!( +api_output_struct!( /// Output returned from the `pre_run` hook. #[serde(default)] pub struct RunHookResult { diff --git a/crates/pdk-test-utils/src/wrapper.rs b/crates/pdk-test-utils/src/wrapper.rs index 1655432c2..a3897ede3 100644 --- a/crates/pdk-test-utils/src/wrapper.rs +++ b/crates/pdk-test-utils/src/wrapper.rs @@ -92,7 +92,7 @@ impl WasmTestWrapper { pub async fn parse_version_file( &self, - mut input: ParseVersionFileInput, + mut input: ParseVersionFileInput<'_>, ) -> ParseVersionFileOutput { input.context = self.prepare_unresolved_context(input.context); input.path = self.tool.to_virtual_path(input.path); @@ -115,7 +115,7 @@ impl WasmTestWrapper { .unwrap() } - pub async fn pre_install(&self, mut input: InstallHook) { + pub async fn pre_install(&self, mut input: InstallHook<'_>) { input.context = self.prepare_context(input.context); self.tool @@ -125,7 +125,7 @@ impl WasmTestWrapper { .unwrap(); } - pub async fn pre_run(&self, mut input: RunHook) -> RunHookResult { + pub async fn pre_run(&self, mut input: RunHook<'_>) -> RunHookResult { input.context = self.prepare_context(input.context); self.tool @@ -135,7 +135,7 @@ impl WasmTestWrapper { .unwrap() } - pub async fn post_install(&self, mut input: InstallHook) { + pub async fn post_install(&self, mut input: InstallHook<'_>) { input.context = self.prepare_context(input.context); self.tool @@ -185,7 +185,7 @@ impl WasmTestWrapper { pub async fn sync_shell_profile( &self, - mut input: SyncShellProfileInput, + mut input: SyncShellProfileInput<'_>, ) -> SyncShellProfileOutput { input.context = self.prepare_context(input.context); diff --git a/crates/warpgate-api/src/lib.rs b/crates/warpgate-api/src/lib.rs index a34f3e001..50d95453d 100644 --- a/crates/warpgate-api/src/lib.rs +++ b/crates/warpgate-api/src/lib.rs @@ -22,6 +22,26 @@ macro_rules! api_struct { }; } +/// Wrap an input struct with common derives and serde required attributes. +#[macro_export] +macro_rules! api_input_struct { + ($struct:item) => { + #[derive(Clone, Debug, serde::Deserialize, PartialEq, serde::Serialize)] + #[cfg_attr(feature = "schematic", derive(schematic::Schematic))] + $struct + }; +} + +/// Wrap an output struct with common derives and serde required attributes. +#[macro_export] +macro_rules! api_output_struct { + ($struct:item) => { + #[derive(Clone, Debug, Default, serde::Deserialize, PartialEq, serde::Serialize)] + #[cfg_attr(feature = "schematic", derive(schematic::Schematic))] + $struct + }; +} + /// Wrap an enum with common derives and serde required attributes. #[macro_export] macro_rules! api_enum { diff --git a/crates/warpgate-api/src/locator.rs b/crates/warpgate-api/src/locator.rs index dc8bc63e9..c2fd7bdd8 100644 --- a/crates/warpgate-api/src/locator.rs +++ b/crates/warpgate-api/src/locator.rs @@ -4,6 +4,17 @@ use std::fmt::Display; use std::path::PathBuf; use std::str::FromStr; +/// An inline data locator. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct DataLocator { + /// Base64 encoded data (with data://). + pub data: String, + + /// The decoded bytes of the data. + /// This must be done manually on the host side. + pub bytes: Option>, +} + /// A file system locator. #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct FileLocator { @@ -77,6 +88,9 @@ pub struct RegistryLocator { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(untagged, into = "String", try_from = "String")] pub enum PluginLocator { + /// data://base64encodeddata + Data(Box), + /// file:///abs/path/to/file.wasm /// file://../rel/path/to/file.wasm File(Box), @@ -109,6 +123,13 @@ impl schematic::Schematic for PluginLocator { impl Display for PluginLocator { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + PluginLocator::Data(data) => { + if data.data.starts_with("data://") { + write!(f, "{}", data.data) + } else { + write!(f, "data://{}", data.data) + } + } PluginLocator::File(file) => { if file.file.starts_with("file://") { write!(f, "{}", file.file) @@ -192,6 +213,10 @@ impl TryFrom for PluginLocator { } match protocol { + "data" => Ok(PluginLocator::Data(Box::new(DataLocator { + data: value, + bytes: None, + }))), "file" => Ok(PluginLocator::File(Box::new(FileLocator { file: value, path: None, diff --git a/crates/warpgate/Cargo.toml b/crates/warpgate/Cargo.toml index 426d23ba0..83a0d99cd 100644 --- a/crates/warpgate/Cargo.toml +++ b/crates/warpgate/Cargo.toml @@ -11,6 +11,7 @@ rust-version = "1.89.0" system_env = { version = "0.9.1", path = "../system-env" } warpgate_api = { version = "0.17.3", path = "../warpgate-api" } async-trait = { workspace = true } +base64 = "0.22.1" compact_str = { workspace = true } docker_credential = { workspace = true } extism = { workspace = true, features = ["http"] } diff --git a/crates/warpgate/src/helpers.rs b/crates/warpgate/src/helpers.rs index 4a7808017..6f8a7e313 100644 --- a/crates/warpgate/src/helpers.rs +++ b/crates/warpgate/src/helpers.rs @@ -101,7 +101,7 @@ pub fn move_or_unpack_download( match temp_file.extension().and_then(|ext| ext.to_str()) { Some("wasm" | "toml" | "json" | "jsonc" | "yaml" | "yml") => { // Plugins can be downloaded in parallel, which means - // that his temp file can also be moved by another process. + // that this temp file can also be moved by another process. // Because of this, proto constantly runs into "Failed to rename" // errors when hitting this block, so let's avoid the failure // if the condition is met and assume all is good! diff --git a/crates/warpgate/src/lib.rs b/crates/warpgate/src/lib.rs index fc3df0f5c..a1da3cc0d 100644 --- a/crates/warpgate/src/lib.rs +++ b/crates/warpgate/src/lib.rs @@ -20,6 +20,6 @@ pub use registry::*; pub use extism::{Manifest as PluginManifest, Wasm}; pub use warpgate_api as api; pub use warpgate_api::{ - FileLocator, GitHubLocator, Id, IdError, PluginLocator, PluginLocatorError, RegistryLocator, - UrlLocator, VirtualPath, + DataLocator, FileLocator, GitHubLocator, Id, IdError, PluginLocator, PluginLocatorError, + RegistryLocator, UrlLocator, VirtualPath, }; diff --git a/crates/warpgate/src/loader.rs b/crates/warpgate/src/loader.rs index 7428c5925..b4a9877ee 100644 --- a/crates/warpgate/src/loader.rs +++ b/crates/warpgate/src/loader.rs @@ -3,7 +3,9 @@ use crate::helpers::{ create_cache_key, determine_cache_extension, download_from_url_to_file, move_or_unpack_download, }; use crate::loader_error::WarpgateLoaderError; -use crate::protocols::{FileLoader, GitHubLoader, HttpLoader, LoadFrom, LoaderProtocol, OciLoader}; +use crate::protocols::{ + DataLoader, FileLoader, GitHubLoader, HttpLoader, LoadFrom, LoaderProtocol, OciLoader, +}; use crate::registry::RegistryConfig; use once_cell::sync::OnceCell; use starbase_styles::color; @@ -24,6 +26,9 @@ pub struct PluginLoader { /// Duration in seconds in which to cache downloaded plugins. cache_duration: Duration, + /// Loader for referencing local plugins using byte streams. + data_loader: OnceCell, + /// Loader for referencing local plugins using file paths. file_loader: OnceCell, @@ -70,6 +75,7 @@ impl PluginLoader { Self { cache_duration: Duration::from_secs(86400 * 30), // 30 days + data_loader: OnceCell::new(), file_loader: OnceCell::new(), github_loader: OnceCell::new(), http_client: OnceCell::new(), @@ -97,6 +103,11 @@ impl PluginLoader { } } + /// Return a data loader for use with [`DataLocator`]s. + pub fn get_data_loader(&self) -> Result<&DataLoader, WarpgateLoaderError> { + self.data_loader.get_or_try_init(|| Ok(DataLoader {})) + } + /// Return a file loader for use with [`FileLocator`]s. pub fn get_file_loader(&self) -> Result<&FileLoader, WarpgateLoaderError> { self.file_loader.get_or_try_init(|| Ok(FileLoader {})) @@ -157,6 +168,11 @@ impl PluginLoader { // Determine the source location let (source, is_latest) = match locator { + PluginLocator::Data(data) => { + let loader = self.get_data_loader()?; + + (loader.load(id, data, &()).await?, loader.is_latest(data)) + } PluginLocator::File(file) => { let loader = self.get_file_loader()?; diff --git a/crates/warpgate/src/loader_error.rs b/crates/warpgate/src/loader_error.rs index 5c1fb90ee..578b7200d 100644 --- a/crates/warpgate/src/loader_error.rs +++ b/crates/warpgate/src/loader_error.rs @@ -144,6 +144,16 @@ pub enum WarpgateLoaderError { .location.style(Style::Path), )] OCIReferenceError { message: String, location: String }, + + #[cfg_attr( + feature = "miette", + diagnostic(code(plugin::loader::data::decode_failure)) + )] + #[error("Failed to decode base64 data for plugin.")] + Base64DecodeError { + #[source] + error: Box, + }, } impl From for WarpgateLoaderError { diff --git a/crates/warpgate/src/plugin.rs b/crates/warpgate/src/plugin.rs index 73ae13119..10c514f5c 100644 --- a/crates/warpgate/src/plugin.rs +++ b/crates/warpgate/src/plugin.rs @@ -1,6 +1,7 @@ use crate::helpers::{from_virtual_path, to_virtual_path}; use crate::plugin_error::WarpgatePluginError; use extism::{Error, Function, Manifest, Plugin}; +use scc::hash_map::Entry; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use starbase_styles::{apply_style_tags, color}; @@ -87,7 +88,7 @@ pub type OnCallFn = Arc, Option<&str>) + Send + Sync>; /// additional methods for easily working with WASI and virtual paths. pub struct PluginContainer { pub id: Id, - pub manifest: Manifest, + pub manifest: Arc, debug_call: bool, func_cache: Arc>>, @@ -123,7 +124,7 @@ impl PluginContainer { ); Ok(PluginContainer { - manifest, + manifest: Arc::new(manifest), plugin: Arc::new(RwLock::new(plugin)), id, func_cache: Arc::new(scc::HashMap::new()), @@ -168,8 +169,6 @@ impl PluginContainer { I: Debug + Serialize, O: Debug + DeserializeOwned, { - use scc::hash_map::Entry; - let func = func.as_ref(); let input = self.format_input(func, input)?; let cache_key = format!("{func}-{input}"); @@ -210,7 +209,7 @@ impl PluginContainer { self.parse_output( func, - &self.call(func, self.format_input(func, input)?).await?, + self.call(func, self.format_input(func, input)?).await?, ) } @@ -234,7 +233,24 @@ impl PluginContainer { /// Return true if the plugin has a function with the given id. pub async fn has_func(&self, func: impl AsRef) -> bool { - self.plugin.read().await.function_exists(func.as_ref()) + let func = func.as_ref(); + + match self.func_cache.entry_async(func.to_owned()).await { + Entry::Occupied(entry) => entry.get()[0] != 0, + Entry::Vacant(entry) => { + let has = self.plugin.read().await.function_exists(func); + + entry.insert_entry(vec![has as u8]); + + has + } + } + } + + /// Convert the provided absolute host path to a virtual guest path suitable + /// for WASI sandboxed runtimes. + pub fn from_real_path(&self, path: impl AsRef + Debug) -> VirtualPath { + self.to_virtual_path(path) } /// Convert the provided virtual guest path to an absolute host path. @@ -248,6 +264,11 @@ impl PluginContainer { from_virtual_path(&paths, path) } + /// Convert the provided virtual guest path to an absolute host path. + pub fn to_real_path(&self, path: impl AsRef + Debug) -> PathBuf { + self.from_virtual_path(path) + } + /// Convert the provided absolute host path to a virtual guest path suitable /// for WASI sandboxed runtimes. pub fn to_virtual_path(&self, path: impl AsRef + Debug) -> VirtualPath { @@ -261,10 +282,10 @@ impl PluginContainer { } /// Call a function on the plugin with the given raw input and return the raw output. - pub async fn call( + pub async fn call>( &self, func: &str, - input: impl AsRef<[u8]>, + input: I, ) -> Result, WarpgatePluginError> { let mut instance = self.plugin.write().await; let input = input.as_ref(); @@ -357,12 +378,12 @@ impl PluginContainer { }) } - fn parse_output( + fn parse_output>( &self, func: &str, - data: &[u8], + data: D, ) -> Result { - serde_json::from_slice(data).map_err(|error| WarpgatePluginError::InvalidOutput { + serde_json::from_slice(data.as_ref()).map_err(|error| WarpgatePluginError::InvalidOutput { id: self.id.clone(), func: func.to_owned(), error: Box::new(error), diff --git a/crates/warpgate/src/protocols/data.rs b/crates/warpgate/src/protocols/data.rs new file mode 100644 index 000000000..08fc7de90 --- /dev/null +++ b/crates/warpgate/src/protocols/data.rs @@ -0,0 +1,46 @@ +use super::{LoadFrom, LoaderProtocol}; +use crate::helpers::create_cache_key; +use crate::loader_error::WarpgateLoaderError; +use base64::prelude::*; +use tracing::trace; +use warpgate_api::{DataLocator, Id}; + +#[derive(Clone)] +pub struct DataLoader {} + +impl LoaderProtocol for DataLoader { + type Data = (); + + fn is_latest(&self, _locator: &DataLocator) -> bool { + false + } + + async fn load( + &self, + id: &Id, + locator: &DataLocator, + _: &Self::Data, + ) -> Result { + trace!(id = id.as_str(), "Linking plugin from explicit byte stream"); + + let encoded_data = locator + .data + .strip_prefix("data://") + .unwrap_or(&locator.data); + + let data = match &locator.bytes { + Some(bytes) => bytes.clone(), + None => BASE64_STANDARD.decode(encoded_data).map_err(|error| { + WarpgateLoaderError::Base64DecodeError { + error: Box::new(error), + } + })?, + }; + + Ok(LoadFrom::Blob { + hash: create_cache_key(encoded_data, None), + ext: ".wasm".into(), + data, + }) + } +} diff --git a/crates/warpgate/src/protocols/mod.rs b/crates/warpgate/src/protocols/mod.rs index dc55ab45f..1131625a5 100644 --- a/crates/warpgate/src/protocols/mod.rs +++ b/crates/warpgate/src/protocols/mod.rs @@ -1,8 +1,10 @@ +mod data; mod file; mod github; mod http; mod oci; +pub use data::*; pub use file::*; pub use github::*; pub use http::*; diff --git a/crates/warpgate/tests/loader_test.rs b/crates/warpgate/tests/loader_test.rs index 82b1eabcc..c18add14a 100644 --- a/crates/warpgate/tests/loader_test.rs +++ b/crates/warpgate/tests/loader_test.rs @@ -1,6 +1,11 @@ +use base64::Engine; +use base64::prelude::BASE64_STANDARD; use starbase_sandbox::{Sandbox, create_empty_sandbox, locate_fixture}; +use starbase_utils::fs; use std::path::PathBuf; -use warpgate::{FileLocator, GitHubLocator, Id, PluginLoader, PluginLocator, UrlLocator}; +use warpgate::{ + DataLocator, FileLocator, GitHubLocator, Id, PluginLoader, PluginLocator, UrlLocator, +}; fn create_loader() -> (Sandbox, PluginLoader) { let sandbox = create_empty_sandbox(); @@ -12,6 +17,30 @@ fn create_loader() -> (Sandbox, PluginLoader) { mod loader { use super::*; + mod data { + use super::*; + + #[tokio::test] + async fn decodes_bytes() { + let (sandbox, loader) = create_loader(); + let fixture = locate_fixture("loader"); + let wasm = fs::read_file_bytes(fixture.join("test.wasm")).unwrap(); + + let path = loader + .load_plugin( + Id::raw("test"), + PluginLocator::Data(Box::new(DataLocator { + data: format!("data://{}", BASE64_STANDARD.encode(wasm)), + bytes: None, + })), + ) + .await + .unwrap(); + + assert_eq!(path, sandbox.path().join("plugins/test-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.wasm")); + } + } + mod source_file { use super::*; diff --git a/package/src/api-types.ts b/package/src/api-types.ts index 671d524b8..d61c680b2 100644 --- a/package/src/api-types.ts +++ b/package/src/api-types.ts @@ -390,7 +390,7 @@ export interface UnpackArchiveInput { outputDir: VirtualPath; } -/** Output returned by the `verify_checksum` function. */ +/** Input passed to the `verify_checksum` function. */ export interface VerifyChecksumInput { /** Virtual path to the checksum file. */ checksumFile: VirtualPath; @@ -401,7 +401,7 @@ export interface VerifyChecksumInput { * is derived from the checksum file's extension, otherwise * it defaults to SHA256. */ - downloadChecksum?: Checksum | null; + downloadChecksum?: string | null; /** Virtual path to the downloaded file. */ downloadFile: VirtualPath; } @@ -491,7 +491,7 @@ export interface LoadVersionsInput { /** Current tool context. */ context: PluginUnresolvedContext; /** The alias or version currently being resolved. */ - initial: UnresolvedVersionSpec; + initial: string; } /** Output returned by the `load_versions` function. */ diff --git a/plugins/Cargo.lock b/plugins/Cargo.lock index b767d56af..71a3fba86 100644 --- a/plugins/Cargo.lock +++ b/plugins/Cargo.lock @@ -131,7 +131,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" dependencies = [ "aws-lc-sys", - "untrusted 0.7.1", "zeroize", ] @@ -2140,7 +2139,6 @@ version = "10.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" dependencies = [ - "aws-lc-rs", "base64 0.22.1", "getrandom 0.2.16", "js-sys", @@ -2529,9 +2527,9 @@ dependencies = [ [[package]] name = "oci-client" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "407c69ed8de3ceeef4662d8fc2d830769b66f0b31ac23fdc9e9f59ac5f995f62" +checksum = "1b7f8deaffcd3b0e3baf93dddcab3d18b91d46dc37d38a8b170089b234de5bb3" dependencies = [ "bytes", "chrono", @@ -2555,9 +2553,9 @@ dependencies = [ [[package]] name = "oci-spec" -version = "0.8.3" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb4684653aeaba48dea019caa17b2773e1212e281d50b6fa759f36fe032239d" +checksum = "e8445a2631507cec628a15fdd6154b54a3ab3f20ed4fe9d73a3b8b7a4e1ba03a" dependencies = [ "const_format", "derive_builder", @@ -2899,7 +2897,7 @@ dependencies = [ [[package]] name = "proto_core" -version = "0.55.4" +version = "0.55.5" dependencies = [ "convert_case", "docker_credential", @@ -2952,7 +2950,7 @@ dependencies = [ [[package]] name = "proto_pdk" -version = "0.32.6" +version = "0.32.7" dependencies = [ "extism-pdk", "proto_pdk_api", @@ -2963,7 +2961,7 @@ dependencies = [ [[package]] name = "proto_pdk_api" -version = "0.31.7" +version = "0.31.8" dependencies = [ "derive_setters", "rustc-hash", @@ -2979,7 +2977,7 @@ dependencies = [ [[package]] name = "proto_pdk_test_utils" -version = "0.43.6" +version = "0.43.7" dependencies = [ "extism", "proto_core", @@ -3335,7 +3333,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.16", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] @@ -3493,7 +3491,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -3544,9 +3542,9 @@ dependencies = [ [[package]] name = "schematic" -version = "0.19.4" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abc7d9aef1d4faaa9fb78e2976423d24b0961cb15dfd61ed7785fdbffd89e9fa" +checksum = "cc12ebacdd2b93bc506fffc0376c1d8b04b35bce54e5ed6f3228c6e41d346921" dependencies = [ "convert_case", "garde", @@ -3569,9 +3567,9 @@ dependencies = [ [[package]] name = "schematic_macros" -version = "0.19.3" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de6118658d45bba6f7eb5c050a4af41dca2760f79cf3543e276ba8e80624984" +checksum = "22ef30d52d6ce2b34b0dbedf49efaaec437c93aada037e575e9831d95de8f82a" dependencies = [ "convert_case", "darling 0.23.0", @@ -3582,9 +3580,9 @@ dependencies = [ [[package]] name = "schematic_types" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca3461a8dd2ba8439cbd64ae546f95b32c08cb6a8c81ef18986d125710d291" +checksum = "87f45f7fb23e4533d3a0a9f5b27a8bb897a7b3b410730f0c1ee99025bf8b06aa" dependencies = [ "indexmap", "regex", @@ -4160,7 +4158,7 @@ dependencies = [ [[package]] name = "system_env" -version = "0.9.0" +version = "0.9.1" dependencies = [ "regex", "schematic", @@ -4664,12 +4662,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -4803,9 +4795,10 @@ dependencies = [ [[package]] name = "warpgate" -version = "0.29.2" +version = "0.29.3" dependencies = [ "async-trait", + "base64 0.22.1", "compact_str 0.9.0", "docker_credential", "extism", @@ -4836,7 +4829,7 @@ dependencies = [ [[package]] name = "warpgate_api" -version = "0.17.2" +version = "0.17.3" dependencies = [ "anyhow", "derive_setters", diff --git a/plugins/mocked-tool/src/proto.rs b/plugins/mocked-tool/src/proto.rs index 7708ae6f8..9db293625 100644 --- a/plugins/mocked-tool/src/proto.rs +++ b/plugins/mocked-tool/src/proto.rs @@ -44,7 +44,7 @@ struct VersionJson { #[plugin_fn] pub fn parse_version_file( - Json(input): Json, + Json(input): Json>, ) -> FnResult> { let mut version = None; let id = get_plugin_id()?; diff --git a/registry/data/third-party.json b/registry/data/third-party.json index 0e37c5068..e591a9162 100644 --- a/registry/data/third-party.json +++ b/registry/data/third-party.json @@ -50,7 +50,7 @@ "name": "Archgate", "description": "Enforce Architecture Decision Records as executable rules.", "author": "archgate", - "homepageUrl": "https://cli.archgate.dev", + "homepageUrl": "https://cli.archgate.dev/", "repositoryUrl": "https://github.com/archgate/proto-plugin", "bins": [ "archgate"