From c981d722da61a5a4e514943117e191b877d096d6 Mon Sep 17 00:00:00 2001 From: Muhamad Awad Date: Fri, 10 Jul 2026 17:13:36 +0300 Subject: [PATCH 1/2] WIP: ingestion API protocol Summary: A seed for the ingestion protocol protobuf. --- .../ingress-http/protobuf/ingestion_svc.proto | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 crates/ingress-http/protobuf/ingestion_svc.proto diff --git a/crates/ingress-http/protobuf/ingestion_svc.proto b/crates/ingress-http/protobuf/ingestion_svc.proto new file mode 100644 index 0000000000..9aaf57f09a --- /dev/null +++ b/crates/ingress-http/protobuf/ingestion_svc.proto @@ -0,0 +1,113 @@ +// Copyright (c) 2026 - Restate Software, Inc., Restate GmbH +// +// This file is part of the Restate service protocol, which is +// released under the MIT license. +// +// You can find a copy of the license in file LICENSE in the root +// directory of this repository or package, or at +// https://github.com/restatedev/proto/blob/main/LICENSE + +syntax = "proto3"; + +package restate.ingestion; + + +// A Settings message defines the default values applied to all +// subsequent records in the stream. +// Each Settings message must specify every field it wants to set: +// its fields are not merged with those of previous Settings messages. +// Leaving an optional field unset clears any previously established +// default, so later records must supply the value explicitly if needed. +message Settings { + // required fields + + // Unique producer ID for the stream. Deduplication relies on this + // ID being unique per ingestion stream + // (think Kafka cluster + topic + partition). + // An explicitly empty producer ID is valid but disables + // deduplication for the entire stream. + string producer_id = 1; + + // optional fields + optional string scope = 2; + optional string limit_key = 3; + optional string service = 4; + optional string handler = 5; + map headers = 6; +} + +message Overrides { +} + +message Record { + uint64 offset = 1; + // Required when the target service is a virtual object (VO). + optional string key = 2; + optional string traceparent = 3; + optional string tracestate = 4; + + // Overrides + optional string scope = 5; + optional string limit_key = 6; + optional string service = 7; + optional string handler = 8; + // These headers are merged with the default headers + // from the Settings message. + map additional_headers = 9; + + bytes payload = 100; +} + +message Request { + oneof payload { + Settings settings = 1; + Record record = 2; + } +} + +// Application-layer flow control message. +// It tells the client how much more it can send +// before it must wait for the next window update. +// +// WindowUpdate also doubles as an explicit ack: the +// server can send an update with a 0 increment to +// acknowledge commits up to "Response::last_committed" +// without changing the window size. +message WindowUpdate { + uint64 increment = 1; +} + +enum ErrorKind { + ERROR_KIND_UNKNOWN = 0; + // Server is shutting down and + // can't accept more records + ERROR_KIND_SHUTTING_DOWN = 1; + ERROR_KIND_UNKNOWN_SERVICE = 2; + ERROR_KIND_UNKNOWN_HANDLER = 3; + // add new error kinds here + +} +message Error { + ErrorKind kind = 1; + string message = 2; +} + +message Response { + // Offsets are 0-based, so last_committed must be optional + // to distinguish "offset 0 committed" from "nothing committed yet". + // This matters when an error is returned on the first + // ingestion message. + optional uint64 last_committed = 1; + + oneof response { + WindowUpdate ack = 2; + Error error = 3; + } +} + +// Service that is only reachable on nodes that are alive. +service IngestionSvc { + // Opens a bidirectional node-to-node stream. + rpc Ingest(stream Request) + returns (stream Response); +} From 5bb247a9bc6b541dbb485276493f9f02865a019b Mon Sep 17 00:00:00 2001 From: Muhamad Awad Date: Fri, 10 Jul 2026 17:26:03 +0300 Subject: [PATCH 2/2] wip-poc: connect-rpc scafolding Summary: Experiment with connect-rpc since it support streaming over http2 and http1 (connect protocol) --- Cargo.lock | 123 +++++++++++++++++ crates/ingress-http/Cargo.toml | 11 +- crates/ingress-http/build.rs | 23 ++++ .../src/ingestion/ingestion_svc.rs | 124 ++++++++++++++++++ crates/ingress-http/src/ingestion/mod.rs | 106 +++++++++++++++ crates/ingress-http/src/lib.rs | 2 + crates/ingress-http/src/server.rs | 7 +- workspace-hack/Cargo.toml | 18 ++- 8 files changed, 410 insertions(+), 4 deletions(-) create mode 100644 crates/ingress-http/build.rs create mode 100644 crates/ingress-http/src/ingestion/ingestion_svc.rs create mode 100644 crates/ingress-http/src/ingestion/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 803118868a..0a2d7b16a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,6 +1329,51 @@ dependencies = [ "serde", ] +[[package]] +name = "buffa" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f29a40702df4b86ccd84211bfde8cee0bce6d0811450ade4a86a7d0958a23a" +dependencies = [ + "base64 0.22.1", + "bytes", + "foldhash 0.1.5", + "hashbrown 0.15.5", + "once_cell", + "rustversion", + "serde", + "serde_json", + "smoothutf8", + "thiserror 2.0.18", +] + +[[package]] +name = "buffa-codegen" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6681b562b18ea719622d0d12684e88ecbdca600d517557dc7bcef7704631e28" +dependencies = [ + "buffa", + "buffa-descriptor", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "thiserror 2.0.18", +] + +[[package]] +name = "buffa-descriptor" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ed423c4ecec86d1500879ce42e7e5f6def01bc632985ac756c1fd723fa21fc" +dependencies = [ + "buffa", + "rustversion", + "serde", + "serde_json", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -1800,6 +1845,64 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "connectrpc" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d1bbfc95ed1f4f31a027f5b97ff981be2617bc8a5c800fd6fcc701867bb97f" +dependencies = [ + "async-compression", + "async-trait", + "base64 0.22.1", + "buffa", + "bytes", + "flate2", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tower", + "tracing", + "wasm-bindgen-futures", + "zstd", +] + +[[package]] +name = "connectrpc-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cdec2dacacd00437898dad38f98105c82d51086a78baa62cee6dc66c031070" +dependencies = [ + "anyhow", + "buffa", + "buffa-codegen", + "connectrpc-codegen", + "tempfile", +] + +[[package]] +name = "connectrpc-codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c6e3bb62a4a77f41d5cd3d290a488bf529dc7e26b444cda67a3e5544c97542" +dependencies = [ + "anyhow", + "buffa", + "buffa-codegen", + "heck", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "console" version = "0.16.3" @@ -4012,6 +4115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "foldhash 0.1.5", + "serde", ] [[package]] @@ -7910,10 +8014,13 @@ name = "restate-ingress-http" version = "1.7.3-dev" dependencies = [ "anyhow", + "buffa", "bytes", "bytestring", "chrono", "codederror", + "connectrpc", + "connectrpc-build", "enumset", "futures", "http 1.4.2", @@ -9180,6 +9287,7 @@ dependencies = [ "arrow-cast", "arrow-ipc", "arrow-schema", + "async-compression", "aws-credential-types", "aws-lc-rs", "aws-lc-sys", @@ -9191,8 +9299,10 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "axum", + "base64 0.22.1", "bilrost", "bitflags 2.11.1", + "buffa", "bytes", "bytestring", "cc", @@ -9201,6 +9311,7 @@ dependencies = [ "clap_builder", "comfy-table", "compact_str", + "compression-codecs", "constant_time_eq", "criterion", "crossbeam-epoch", @@ -9215,6 +9326,7 @@ dependencies = [ "enum-map", "errno", "flate2", + "foldhash 0.1.5", "foldhash 0.2.0", "futures-channel", "futures-core", @@ -9226,6 +9338,7 @@ dependencies = [ "getrandom 0.4.2", "half", "hashbrown 0.14.5", + "hashbrown 0.15.5", "hashbrown 0.17.1", "hdrhistogram", "hex", @@ -9286,6 +9399,7 @@ dependencies = [ "sha2 0.10.9", "signal-hook", "simd-adler32", + "simdutf8", "slab", "smallvec", "stable_deref_trait", @@ -10210,6 +10324,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "smoothutf8" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36358427d32ecdb1624616deed99eccfef0a167fe5bf40ddb51efe6980bc1ec8" +dependencies = [ + "simdutf8", +] + [[package]] name = "snafu" version = "0.8.9" diff --git a/crates/ingress-http/Cargo.toml b/crates/ingress-http/Cargo.toml index e8973812ce..2f70c0bae4 100644 --- a/crates/ingress-http/Cargo.toml +++ b/crates/ingress-http/Cargo.toml @@ -8,8 +8,12 @@ license.workspace = true publish = false [features] -default = [] +default = ["ingestion"] options_schema = [] +# Experimental Connect (connectrpc) ingestion API served at +# `/restate/restate.ingestion.IngestionSvc/Ingest`. Off by default: enabling it +# pulls in the `connectrpc` runtime stack and generates the proto bindings. +ingestion = ["dep:connectrpc"] [dependencies] restate-workspace-hack = { workspace = true } @@ -22,9 +26,11 @@ restate-types = { workspace = true } restate-util-string = { workspace = true } anyhow = { workspace = true } +buffa = { version = "0.8.1", features = ["std", "fast-utf8"] } bytes = { workspace = true } bytestring = { workspace = true } chrono = { workspace = true } +connectrpc = { version = "0.8.1", optional = true } codederror = { workspace = true } enumset = { workspace = true } futures = { workspace = true } @@ -64,5 +70,8 @@ mockall = { workspace = true } tempfile = { workspace = true } tracing-test = { workspace = true } +[build-dependencies] +connectrpc-build = { version = "0.8" } + [lints] workspace = true diff --git a/crates/ingress-http/build.rs b/crates/ingress-http/build.rs new file mode 100644 index 0000000000..6afb418167 --- /dev/null +++ b/crates/ingress-http/build.rs @@ -0,0 +1,23 @@ +// Copyright (c) 2023 - 2026 Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +fn main() { + println!("cargo:rerun-if-changed=protobuf/ingestion_svc.proto"); + + // Only generate (and thus require the `connectrpc` runtime + protoc) when the + // `ingestion` feature is enabled. Build scripts cannot use `cfg(feature = ...)`, + // so we gate on the env var Cargo sets for the feature instead. + connectrpc_build::Config::new() + .files(&["protobuf/ingestion_svc.proto"]) + .includes(&["protobuf/"]) + .include_file("_connectrpc.rs") + .compile() + .expect("failed to compile protobuf/ingestion_svc.proto"); +} diff --git a/crates/ingress-http/src/ingestion/ingestion_svc.rs b/crates/ingress-http/src/ingestion/ingestion_svc.rs new file mode 100644 index 0000000000..e8b09bbfdc --- /dev/null +++ b/crates/ingress-http/src/ingestion/ingestion_svc.rs @@ -0,0 +1,124 @@ +// Copyright (c) 2023 - 2026 Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Experimental Connect (connectrpc) ingestion API. +//! +//! Exposes the bidirectional-streaming `restate.ingestion.IngestionSvc/Ingest` +//! RPC (see `protobuf/ingestion_svc.proto`). This is a **scaffold**: the handler +//! parses the incoming `Settings`/`Record` stream and replies with `Ack`/`Error`, +//! but does not yet write records to the log. Real ingestion (WAL `Envelope` + +//! producer/offset deduplication via `restate_ingestion_client::IngestionClient`) +//! is a follow-up. + +use std::sync::Arc; + +use connectrpc::{InboundStream, RequestContext, Response, ServiceResult, ServiceStream}; +use futures::StreamExt; + +use restate_types::live::Live; +use restate_types::schema::invocation_target::InvocationTargetResolver; + +/// Generated protobuf bindings for `restate.ingestion` (see `protobuf/ingestion_svc.proto`). +// Enum variant names mirror the proto (`ErrorKind_UNKNOWN`), which the generator does +// not already allow-list. +// #[allow(clippy::enum_variant_names)] +pub mod proto { + connectrpc::include_generated!(); +} + +use proto::restate::ingestion::__buffa::oneof::request::Payload; +use proto::restate::ingestion::__buffa::oneof::response as response_oneof; +use proto::restate::ingestion::{ + IngestionSvc, IngestionSvcExt, Request, Response as IngestResponse, Settings, WindowUpdate, +}; + +/// Build the Connect tower service for the ingestion API. +pub(crate) fn connect_service(schemas: Live) -> connectrpc::ConnectRpcService +where + Schemas: InvocationTargetResolver + Clone + Send + Sync + 'static, +{ + let router = Arc::new(IngestionService::new(schemas)).register(connectrpc::Router::new()); + connectrpc::ConnectRpcService::new(router) +} + +/// Scaffold implementation of the `IngestionSvc` Connect service. +struct IngestionService { + schemas: Live, +} + +impl IngestionService { + fn new(schemas: Live) -> Self { + Self { schemas } + } +} + +impl IngestionSvc for IngestionService +where + Schemas: InvocationTargetResolver + Clone + Send + Sync + 'static, +{ + async fn ingest( + &self, + _ctx: RequestContext, + requests: InboundStream, + ) -> ServiceResult< + ServiceStream + Send + use>, + > { + // Snapshot the schema once so the stream owns a `'static` resolver rather + // than borrowing `&self` across `.await` points. + let schemas = self.schemas.snapshot(); + + let responses = futures::stream::unfold( + State { + requests, + _schemas: schemas, + settings: None, + }, + |mut state| async move { + loop { + let message = match state.requests.next().await { + None => return None, + Some(Ok(message)) => message, + Some(Err(err)) => return Some((Err(err), state)), + }; + + match message.to_owned_message().payload { + // `Settings` establishes the defaults for subsequent records; + // its fields are replaced wholesale (not merged) and it emits no ack. + Some(Payload::Settings(settings)) => { + state.settings = Some(*settings); + } + Some(Payload::Record(record)) => { + return Some((Ok(ack(record.offset)), state)); + } + // Empty request payload: nothing to do. + None => {} + } + } + }, + ); + + Response::stream_ok(responses) + } +} + +/// Streaming state carried across the inbound `Request` stream. +struct State { + requests: InboundStream, + _schemas: Arc, + settings: Option, +} + +fn ack(offset: u64) -> IngestResponse { + IngestResponse { + last_committed: Some(offset), + response: Some(response_oneof::Response::from(WindowUpdate::default())), + ..Default::default() + } +} diff --git a/crates/ingress-http/src/ingestion/mod.rs b/crates/ingress-http/src/ingestion/mod.rs new file mode 100644 index 0000000000..e6909004ec --- /dev/null +++ b/crates/ingress-http/src/ingestion/mod.rs @@ -0,0 +1,106 @@ +// Copyright (c) 2023 - 2026 Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +mod ingestion_svc; +use hyper::body::Incoming; +use restate_types::live::Live; +use restate_types::schema::invocation_target::InvocationTargetResolver; + +use std::convert::Infallible; +use std::task::{Context, Poll}; + +use futures::future::BoxFuture; +use http::{Request, Response, Uri}; +use http_body_util::BodyExt; +use http_body_util::combinators::UnsyncBoxBody; +use tower::ServiceExt; + +use crate::ingestion; + +use super::*; + +/// The base prefix that is stripped before dispatching to the Connect service. +pub(crate) const INGEST_BASE: &str = "/restate"; + +/// The ingress-mounted path prefix under which the Connect service is served. +/// +/// Clients use the base URL `.../restate`; Connect appends the canonical RPC +/// path, yielding `/restate/restate.ingestion.IngestionSvc/Ingest`. The server +/// strips the leading `/restate` before handing the request to `ConnectRpcService`. +pub(crate) const INGEST_MOUNT_PREFIX: &str = "/restate/restate.ingestion.IngestionSvc/"; + +/// Unified response body for both branches. +type RoutedBody = UnsyncBoxBody; + +#[derive(Clone)] +pub(super) struct IngestRouter { + inner: S, + connect: connectrpc::ConnectRpcService, +} + +impl IngestRouter { + pub(crate) fn new(inner: S, schemas: Live) -> Self + where + Schemas: InvocationTargetResolver + Clone + Send + Sync + 'static, + { + Self { + inner, + connect: ingestion_svc::connect_service(schemas), + } + } +} + +impl tower::Service> for IngestRouter +where + S: tower::Service, Response = Response, Error = Infallible> + + Clone + + Send + + 'static, + S::Future: Send + 'static, + Body: http_body::Body + Send + 'static, +{ + type Response = Response; + type Error = Infallible; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + // Both inner services are always ready. + Poll::Ready(Ok(())) + } + + fn call(&mut self, mut req: Request) -> Self::Future { + if req.uri().path().starts_with(INGEST_MOUNT_PREFIX) { + strip_base_prefix(&mut req); + let connect = self.connect.clone(); + Box::pin(async move { + let response = connect.oneshot(req).await?; + // `ConnectRpcBody::Error` is `Infallible`. + Ok(response.map(|body| body.boxed_unsync())) + }) + } else { + let main = self.inner.clone(); + Box::pin(async move { + let response = main.oneshot(req).await?; + Ok(response.map(|body| body.boxed_unsync())) + }) + } + } +} + +/// Strip the leading `/restate` base so the remaining path matches the Connect +/// canonical `/restate.ingestion.IngestionSvc/Ingest`. +fn strip_base_prefix(req: &mut Request) { + let uri = req.uri(); + let path_and_query = uri.path_and_query().map(|pq| pq.as_str()).unwrap_or("/"); + let stripped = &path_and_query[ingestion::INGEST_BASE.len()..]; + let mut parts = uri.clone().into_parts(); + parts.path_and_query = Some(stripped.parse().expect("valid path-and-query")); + *req.uri_mut() = Uri::from_parts(parts).expect("valid uri"); +} diff --git a/crates/ingress-http/src/lib.rs b/crates/ingress-http/src/lib.rs index 8895d0dcc2..2d3d7ecc34 100644 --- a/crates/ingress-http/src/lib.rs +++ b/crates/ingress-http/src/lib.rs @@ -9,6 +9,8 @@ // by the Apache License, Version 2.0. mod handler; +#[cfg(feature = "ingestion")] +mod ingestion; mod layers; mod metric_definitions; mod rpc_request_dispatcher; diff --git a/crates/ingress-http/src/server.rs b/crates/ingress-http/src/server.rs index d572fc0d94..5953c535cb 100644 --- a/crates/ingress-http/src/server.rs +++ b/crates/ingress-http/src/server.rs @@ -191,7 +191,12 @@ where .layer(CorsLayer::very_permissive()) .layer(layers::load_shed::LoadShedLayer::new(concurrency_limit)) .layer(layers::tracing_context_extractor::HttpTraceContextExtractorLayer) - .service(Handler::new(schemas, dispatcher)); + .service(Handler::new(schemas.clone(), dispatcher)); + + // Route the Connect ingestion path to its own service; everything else keeps + // flowing through the layered handler above. + #[cfg(feature = "ingestion")] + let service = ingestion::IngestRouter::new(service, schemas); // todo(azmy): `CorsLayer` should sit above `RequestBodyLimitLayer` so CORS is applied // as early as possible. This is currently blocked because `CorsLayer` requires the diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 9582e5130c..80ac27f630 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -24,6 +24,7 @@ arrow-array = { version = "58", default-features = false, features = ["chrono-tz arrow-cast = { version = "58", default-features = false, features = ["prettyprint"] } arrow-ipc = { version = "58", features = ["lz4", "zstd"] } arrow-schema = { version = "58", default-features = false, features = ["canonical_extension_types"] } +async-compression = { version = "0.4", default-features = false, features = ["brotli", "gzip", "tokio", "zstd"] } aws-credential-types = { version = "1", default-features = false, features = ["test-util"] } aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } aws-lc-sys = { version = "0.40", default-features = false, features = ["prebuilt-nasm"] } @@ -35,7 +36,9 @@ aws-smithy-runtime = { version = "1", default-features = false, features = ["cli aws-smithy-runtime-api = { version = "1", features = ["client", "http-auth", "test-util"] } aws-smithy-types = { version = "1", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "http-body-1-x", "rt-tokio", "test-util"] } axum = { version = "0.8", default-features = false, features = ["http1", "http2", "json", "query", "tokio"] } +base64 = { version = "0.22" } bilrost = { version = "0.1014", default-features = false, features = ["auto-optimize", "bytestring", "derive", "smallvec", "std"] } +buffa = { version = "0.8", features = ["json"] } bytes = { version = "1", features = ["serde"] } bytestring = { version = "1", default-features = false, features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } @@ -43,6 +46,7 @@ clap = { version = "4", default-features = false, features = ["color", "derive", clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } comfy-table = { version = "7" } compact_str = { version = "0.9", default-features = false, features = ["bytes", "serde", "std"] } +compression-codecs = { version = "0.4", default-features = false, features = ["brotli", "gzip", "zstd"] } constant_time_eq = { version = "0.4" } criterion = { version = "0.5", features = ["async_tokio"] } crossbeam-epoch = { version = "0.9" } @@ -54,7 +58,8 @@ digest = { version = "0.10", features = ["mac", "std"] } either = { version = "1", features = ["use_std"] } enum-map = { version = "2", default-features = false, features = ["serde"] } flate2 = { version = "1", features = ["zlib-rs"] } -foldhash = { version = "0.2" } +foldhash-6f8ce4dd05d13bba = { package = "foldhash", version = "0.2" } +foldhash-c65f7effa3be6d31 = { package = "foldhash", version = "0.1", default-features = false, features = ["std"] } futures-channel = { version = "0.3", features = ["sink"] } futures-core = { version = "0.3" } futures-executor = { version = "0.3" } @@ -63,6 +68,7 @@ futures-util = { version = "0.3", features = ["channel", "io", "sink"] } gardal = { version = "0.0.1-alpha.9", features = ["async"] } getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } half = { version = "2", default-features = false, features = ["num-traits"] } +hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15", default-features = false, features = ["default-hasher", "inline-more", "serde"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", default-features = false, features = ["inline-more", "raw"] } hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17" } hdrhistogram = { version = "7" } @@ -112,6 +118,7 @@ serde_core = { version = "1", features = ["alloc", "rc"] } serde_json = { version = "1", features = ["alloc", "raw_value", "unbounded_depth"] } serde_with = { version = "3", features = ["base64", "hex", "json"] } simd-adler32 = { version = "0.3", default-features = false, features = ["std"] } +simdutf8 = { version = "0.1", default-features = false, features = ["std"] } slab = { version = "0.4" } smallvec = { version = "1", default-features = false, features = ["const_new", "serde", "union"] } stable_deref_trait = { version = "1" } @@ -154,6 +161,7 @@ arrow-array = { version = "58", default-features = false, features = ["chrono-tz arrow-cast = { version = "58", default-features = false, features = ["prettyprint"] } arrow-ipc = { version = "58", features = ["lz4", "zstd"] } arrow-schema = { version = "58", default-features = false, features = ["canonical_extension_types"] } +async-compression = { version = "0.4", default-features = false, features = ["brotli", "gzip", "tokio", "zstd"] } aws-credential-types = { version = "1", default-features = false, features = ["test-util"] } aws-lc-rs = { version = "1", features = ["prebuilt-nasm"] } aws-lc-sys = { version = "0.40", default-features = false, features = ["prebuilt-nasm"] } @@ -165,7 +173,9 @@ aws-smithy-runtime = { version = "1", default-features = false, features = ["cli aws-smithy-runtime-api = { version = "1", features = ["client", "http-auth", "test-util"] } aws-smithy-types = { version = "1", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "http-body-1-x", "rt-tokio", "test-util"] } axum = { version = "0.8", default-features = false, features = ["http1", "http2", "json", "query", "tokio"] } +base64 = { version = "0.22" } bilrost = { version = "0.1014", default-features = false, features = ["auto-optimize", "bytestring", "derive", "smallvec", "std"] } +buffa = { version = "0.8", features = ["json"] } bytes = { version = "1", features = ["serde"] } bytestring = { version = "1", default-features = false, features = ["serde"] } cc = { version = "1", default-features = false, features = ["parallel"] } @@ -174,6 +184,7 @@ clap = { version = "4", default-features = false, features = ["color", "derive", clap_builder = { version = "4", default-features = false, features = ["color", "env", "std", "suggestions", "usage", "wrap_help"] } comfy-table = { version = "7" } compact_str = { version = "0.9", default-features = false, features = ["bytes", "serde", "std"] } +compression-codecs = { version = "0.4", default-features = false, features = ["brotli", "gzip", "zstd"] } constant_time_eq = { version = "0.4" } criterion = { version = "0.5", features = ["async_tokio"] } crossbeam-epoch = { version = "0.9" } @@ -186,7 +197,8 @@ digest = { version = "0.10", features = ["mac", "std"] } either = { version = "1", features = ["use_std"] } enum-map = { version = "2", default-features = false, features = ["serde"] } flate2 = { version = "1", features = ["zlib-rs"] } -foldhash = { version = "0.2" } +foldhash-6f8ce4dd05d13bba = { package = "foldhash", version = "0.2" } +foldhash-c65f7effa3be6d31 = { package = "foldhash", version = "0.1", default-features = false, features = ["std"] } futures-channel = { version = "0.3", features = ["sink"] } futures-core = { version = "0.3" } futures-executor = { version = "0.3" } @@ -195,6 +207,7 @@ futures-util = { version = "0.3", features = ["channel", "io", "sink"] } gardal = { version = "0.0.1-alpha.9", features = ["async"] } getrandom-468e82937335b1c9 = { package = "getrandom", version = "0.3", default-features = false, features = ["std"] } half = { version = "2", default-features = false, features = ["num-traits"] } +hashbrown-3575ec1268b04181 = { package = "hashbrown", version = "0.15", default-features = false, features = ["default-hasher", "inline-more", "serde"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", default-features = false, features = ["inline-more", "raw"] } hashbrown-9067fe90e8c1f593 = { package = "hashbrown", version = "0.17" } hdrhistogram = { version = "7" } @@ -245,6 +258,7 @@ serde_core = { version = "1", features = ["alloc", "rc"] } serde_json = { version = "1", features = ["alloc", "raw_value", "unbounded_depth"] } serde_with = { version = "3", features = ["base64", "hex", "json"] } simd-adler32 = { version = "0.3", default-features = false, features = ["std"] } +simdutf8 = { version = "0.1", default-features = false, features = ["std"] } slab = { version = "0.4" } smallvec = { version = "1", default-features = false, features = ["const_new", "serde", "union"] } stable_deref_trait = { version = "1" }