diff --git a/Cargo.lock b/Cargo.lock index b1cf0163..e73ec19f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -384,6 +399,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "1.12.1" @@ -1965,6 +2001,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + [[package]] name = "inotify" version = "0.11.1" @@ -5333,6 +5382,7 @@ dependencies = [ "axum-server", "backon", "base64", + "brotli", "bytes", "cargo-lock", "cargo_metadata", @@ -5350,6 +5400,7 @@ dependencies = [ "http", "humantime", "humantime-serde", + "indicatif", "lightningcss", "local-ip-address", "lol_html", @@ -5477,6 +5528,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 9621cb3b..a0632359 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ panic = "abort" [dependencies] anyhow = "1" async-compression = { version = "0.4.41", features = ["gzip", "tokio"] } +brotli = "8" async-tar = { version = "0.6", default-features = false, features = ["runtime-tokio"] } axum = { version = "0.8", features = ["ws"] } axum-server = "0.8" @@ -43,6 +44,7 @@ htmlescape = "0.3.1" http = "1.4" humantime = "2" humantime-serde = "1" +indicatif = "0.18" local-ip-address = "0.6" lol_html = "2" mime_guess = "2" diff --git a/src/build.rs b/src/build.rs index 649acc12..d6e0f3b6 100644 --- a/src/build.rs +++ b/src/build.rs @@ -11,7 +11,7 @@ use tokio_stream::wrappers::ReadDirStream; use crate::common::{BUILDING, ERROR, SUCCESS, remove_dir_all}; use crate::config::{STAGE_DIR, rt::RtcBuild, types::WsProtocol}; -use crate::pipelines::HtmlPipeline; +use crate::pipelines::{HtmlPipeline, compress_dist}; pub type BuildResult = Result<()>; @@ -123,6 +123,12 @@ impl BuildSystem { let staging_dist = self.cfg.staging_dist.clone(); tracing::info!("applying new distribution"); + // Pre-compress assets in the staging area so the sidecars are moved into `dist` along with + // their originals by the move step below. + compress_dist(&self.cfg) + .await + .context("error compressing distribution assets")?; + // Build succeeded, so delete everything in `dist`, move everything // from `dist/.stage` to `dist`, and then delete `dist/.stage`. self.clean_final().await?; diff --git a/src/cmd/build.rs b/src/cmd/build.rs index fd18c2fc..8a2e55a4 100644 --- a/src/cmd/build.rs +++ b/src/cmd/build.rs @@ -3,7 +3,7 @@ use crate::{ config::{ self, Configuration, Tools, rt::{self, RtcBuild, RtcBuilder}, - types::{BaseUrl, Minify}, + types::{BaseUrl, CompressionAlgorithm, CompressionLevel, Minify}, }, }; use anyhow::Result; @@ -116,6 +116,20 @@ pub struct Build { #[arg(default_missing_value="true", num_args=0..=1)] pub allow_self_closing_script: Option, + /// Pre-compress assets into sidecar files (e.g. `index.html.br`) served via `Accept-Encoding`. + /// + /// Comma-separated list of algorithms; supported: `gzip`, `brotli`. Overrides the config file. + #[arg(long, value_delimiter = ',', env = "TRUNK_BUILD_COMPRESSION")] + pub compression: Option>, + + /// Compression effort: `low` (fastest), `medium` (default), or `high` (smallest, slowest). + #[arg(long, env = "TRUNK_BUILD_COMPRESSION_LEVEL")] + pub compression_level: Option, + + /// Skip compressing files smaller than this size, in bytes. + #[arg(long, env = "TRUNK_BUILD_COMPRESSION_MIN_SIZE")] + pub compression_min_size: Option, + // NOTE: flattened structures come last #[command(flatten)] pub core: super::core::Core, @@ -149,6 +163,9 @@ impl Build { minify, no_sri, allow_self_closing_script, + compression, + compression_level, + compression_min_size, tools, } = self; @@ -184,6 +201,12 @@ impl Build { config.build.no_sri = no_sri.unwrap_or(config.build.no_sri); config.build.allow_self_closing_script = allow_self_closing_script.unwrap_or(config.build.allow_self_closing_script); + config.build.compression.algorithms = + compression.unwrap_or(config.build.compression.algorithms); + config.build.compression.level = + compression_level.unwrap_or(config.build.compression.level); + config.build.compression.min_size = + compression_min_size.unwrap_or(config.build.compression.min_size); let config = core.apply_to(config)?; let config = tools.apply_to(config)?; @@ -212,6 +235,7 @@ impl Build { #[cfg(test)] mod test { + use crate::config::types::CompressionAlgorithm; use crate::{Trunk, TrunkSubcommands}; use clap::Parser; use rstest::rstest; @@ -229,4 +253,30 @@ mod test { assert_eq!(build.no_default_features, expected); } + + #[rstest] + #[case(&["trunk", "build"], None)] + #[case(&["trunk", "build", "--compression", "gzip"], Some(vec![CompressionAlgorithm::Gzip]))] + #[case(&["trunk", "build", "--compression", "gzip,brotli"], Some(vec![CompressionAlgorithm::Gzip, CompressionAlgorithm::Brotli]))] + fn test_compression_arg( + #[case] input: &[&str], + #[case] expected: Option>, + ) { + let cli = Trunk::parse_from(input); + let TrunkSubcommands::Build(build) = cli.action else { + panic!("must be a build command"); + }; + + assert_eq!(build.compression, expected); + } + + #[test] + fn test_compression_min_size_arg() { + let cli = Trunk::parse_from(["trunk", "build", "--compression-min-size", "2048"]); + let TrunkSubcommands::Build(build) = cli.action else { + panic!("must be a build command"); + }; + + assert_eq!(build.compression_min_size, Some(2048)); + } } diff --git a/src/config/models/build.rs b/src/config/models/build.rs index 1dc73b14..80c5c976 100644 --- a/src/config/models/build.rs +++ b/src/config/models/build.rs @@ -1,6 +1,6 @@ use crate::config::{ models::ConfigModel, - types::{BaseUrl, Minify}, + types::{BaseUrl, CompressionAlgorithm, CompressionLevel, Minify}, }; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize, de}; @@ -160,8 +160,64 @@ pub struct Build { /// The placeholder which is used in the 'nonce' attribute. #[serde(default = "default::nonce_placeholder")] pub nonce_placeholder: String, + + /// Optional pre-compression of build assets into sidecar files. + #[serde(default)] + pub compression: Compression, } +/// Config options for pre-compressing build assets into sidecar files (e.g. `index.html.gz`). +/// +/// This is disabled by default; set `algorithms` to enable it. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +pub struct Compression { + /// The compression algorithms to apply. For each enabled algorithm a sidecar file is written + /// next to the original asset (e.g. `app.js.gz`, `app.js.br`). + /// + /// Leaving this empty (the default) disables compression entirely. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub algorithms: Vec, + + /// How much effort to spend compressing: `low`, `medium` (default), or `high`. + #[serde(default)] + pub level: CompressionLevel, + + /// Skip files smaller than this size, in bytes. Compressing tiny files rarely pays off. + #[serde(default = "default::compression_min_size")] + pub min_size: u64, + + /// Only keep a compressed sidecar if its size is at most this percentage of the original size. + /// + /// For example, `90` keeps the sidecar only when it saves at least 10%. A value of `100` keeps + /// any sidecar that is not larger than the original. + #[serde(default = "default::compression_min_ratio_percent")] + pub min_ratio_percent: u8, + + /// Glob patterns (relative to the dist dir) of files to include. When empty, all files are + /// considered (subject to `exclude`). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub include: Vec, + + /// Glob patterns (relative to the dist dir) of files to exclude. Applied after `include`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, +} + +impl Default for Compression { + fn default() -> Self { + Self { + algorithms: Vec::new(), + level: CompressionLevel::default(), + min_size: default::compression_min_size(), + min_ratio_percent: default::compression_min_ratio_percent(), + include: Vec::new(), + exclude: Vec::new(), + } + } +} + +impl ConfigModel for Compression {} + fn string_or_vec<'de, T, D>(deserializer: D) -> Result, D::Error> where T: Deserialize<'de> + FromStr, @@ -233,6 +289,7 @@ impl Default for Build { allow_self_closing_script: false, create_nonce: false, nonce_placeholder: default::nonce_placeholder(), + compression: Default::default(), } } } @@ -264,6 +321,14 @@ mod default { pub fn nonce_placeholder() -> String { "{{__TRUNK NONCE__}}".to_string() } + + pub const fn compression_min_size() -> u64 { + 1024 + } + + pub const fn compression_min_ratio_percent() -> u8 { + 90 + } } mod schema { diff --git a/src/config/models/serve.rs b/src/config/models/serve.rs index ef4e88cd..a7700eaa 100644 --- a/src/config/models/serve.rs +++ b/src/config/models/serve.rs @@ -88,6 +88,14 @@ pub struct Serve { /// The CSP; {{NONE}} is replaced by a random nonce #[serde(default = "default::csp")] pub csp: Vec, + /// Serve precompressed sidecar files (e.g. `index.html.br`) based on the request's + /// `Accept-Encoding` header. + /// + /// When unset, this follows the build's compression configuration: the algorithms produced at + /// build time are served. Set to `true` to serve any available `.gz`/`.br` sidecar, or `false` + /// to disable serving precompressed files entirely. + #[serde(default)] + pub precompressed: Option, } impl Default for Serve { @@ -118,6 +126,7 @@ impl Default for Serve { proxy_no_redirect: None, disable_csp: false, csp: default::csp(), + precompressed: None, } } } diff --git a/src/config/models/test.rs b/src/config/models/test.rs index 7db24bba..24e8b7d7 100644 --- a/src/config/models/test.rs +++ b/src/config/models/test.rs @@ -214,3 +214,36 @@ async fn example_config() { .await .expect("example config should be parsable"); } + +#[test] +fn parse_compression_config() { + use crate::config::types::CompressionAlgorithm; + + let toml = r#" +[build.compression] +algorithms = ["gzip", "brotli"] +min_size = 2048 +min_ratio_percent = 80 +include = ["*.js"] +exclude = ["*.png"] +"#; + let cfg: Configuration = toml::from_str(toml).expect("config should parse"); + let compression = cfg.build.compression; + assert_eq!( + compression.algorithms, + vec![CompressionAlgorithm::Gzip, CompressionAlgorithm::Brotli] + ); + assert_eq!(compression.min_size, 2048); + assert_eq!(compression.min_ratio_percent, 80); + assert_eq!(compression.include, vec!["*.js".to_string()]); + assert_eq!(compression.exclude, vec!["*.png".to_string()]); +} + +#[test] +fn compression_disabled_by_default() { + let cfg: Configuration = toml::from_str("").expect("empty config should parse"); + assert!( + cfg.build.compression.algorithms.is_empty(), + "compression should be disabled by default" + ); +} diff --git a/src/config/rt/build.rs b/src/config/rt/build.rs index 8b12fb87..0a620e51 100644 --- a/src/config/rt/build.rs +++ b/src/config/rt/build.rs @@ -1,15 +1,16 @@ use super::{super::STAGE_DIR, RtcBuilder}; -use crate::config::models::{NodePackage, NodePackages}; +use crate::config::models::{Compression, NodePackage, NodePackages}; use crate::{ config::{ Hooks, models::{Configuration, Hook, Tools}, rt::{CoreOptions, RtcCore}, - types::{BaseUrl, Minify}, + types::{BaseUrl, CompressionAlgorithm, CompressionLevel, Minify}, }, tools::HttpClientOptions, }; use anyhow::{Context, ensure}; +use globset::{Glob, GlobSet, GlobSetBuilder}; use std::{collections::HashMap, ops::Deref, path::PathBuf}; /// Config options for the cargo build command @@ -26,6 +27,63 @@ pub enum Features { }, } +/// Runtime config for pre-compressing build assets. +/// +/// Glob patterns are pre-compiled here so that any pattern errors surface at config load time. +#[derive(Clone, Debug)] +pub struct RtcCompression { + /// The compression algorithms to apply. Empty means compression is disabled. + pub algorithms: Vec, + /// How much effort to spend compressing. + pub level: CompressionLevel, + /// Skip files smaller than this size, in bytes. + pub min_size: u64, + /// Only keep a sidecar if its size is at most this percentage of the original size. + pub min_ratio_percent: u8, + /// Files to include. `None` means "all files" (subject to `exclude`). + pub include: Option, + /// Files to exclude. `None` means "exclude nothing". + pub exclude: Option, +} + +impl RtcCompression { + fn new(compression: Compression) -> anyhow::Result { + Ok(Self { + algorithms: compression.algorithms, + level: compression.level, + min_size: compression.min_size, + min_ratio_percent: compression.min_ratio_percent, + include: compile_globs(&compression.include).context("invalid compression include")?, + exclude: compile_globs(&compression.exclude).context("invalid compression exclude")?, + }) + } + + /// Whether compression is enabled (i.e. at least one algorithm is configured). + pub fn enabled(&self) -> bool { + !self.algorithms.is_empty() + } + + /// Whether the given dist-relative path should be compressed based on include/exclude globs. + pub fn matches(&self, path: &std::path::Path) -> bool { + let included = self.include.as_ref().is_none_or(|set| set.is_match(path)); + let excluded = self.exclude.as_ref().is_some_and(|set| set.is_match(path)); + included && !excluded + } +} + +/// Compile a list of glob patterns into a [`GlobSet`], returning `None` when the list is empty. +fn compile_globs(patterns: &[String]) -> anyhow::Result> { + if patterns.is_empty() { + return Ok(None); + } + let mut builder = GlobSetBuilder::new(); + for pattern in patterns { + builder + .add(Glob::new(pattern).with_context(|| format!("invalid glob pattern: {pattern}"))?); + } + Ok(Some(builder.build().context("error building glob set")?)) +} + /// Runtime config for the build system. #[derive(Clone, Debug)] pub struct RtcBuild { @@ -95,6 +153,8 @@ pub struct RtcBuild { pub allow_self_closing_script: bool, /// When set, create nonce attributes with the option as placeholder pub create_nonce: Option, + /// Configuration for pre-compressing build assets into sidecar files. + pub compression: RtcCompression, } impl Deref for RtcBuild { @@ -189,6 +249,9 @@ impl RtcBuild { let create_nonce = build.create_nonce.then_some(build.nonce_placeholder); + let compression = RtcCompression::new(build.compression) + .context("error processing compression configuration")?; + Ok(Self { core, target, @@ -221,6 +284,7 @@ impl RtcBuild { no_sri: build.no_sri, allow_self_closing_script: build.allow_self_closing_script, create_nonce, + compression, }) } @@ -265,6 +329,8 @@ impl RtcBuild { no_sri: false, allow_self_closing_script: false, create_nonce: None, + compression: RtcCompression::new(Default::default()) + .expect("default compression config is valid"), }) } @@ -300,3 +366,35 @@ impl RtcBuilder for RtcBuild { Self::new(configuration, options) } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn rtc_compression_compiles_globs() { + let compression = Compression { + algorithms: vec![CompressionAlgorithm::Gzip], + include: vec!["*.js".into()], + exclude: vec!["vendor/*".into()], + ..Default::default() + }; + let rtc = RtcCompression::new(compression).expect("valid globs should compile"); + assert!(rtc.enabled()); + assert!(rtc.matches(std::path::Path::new("app.js"))); + assert!(!rtc.matches(std::path::Path::new("app.css"))); + assert!(!rtc.matches(std::path::Path::new("vendor/app.js"))); + } + + #[test] + fn rtc_compression_rejects_invalid_glob() { + let compression = Compression { + include: vec!["[".into()], + ..Default::default() + }; + assert!( + RtcCompression::new(compression).is_err(), + "an invalid glob pattern should be rejected" + ); + } +} diff --git a/src/config/rt/serve.rs b/src/config/rt/serve.rs index 23b490fc..069ce904 100644 --- a/src/config/rt/serve.rs +++ b/src/config/rt/serve.rs @@ -50,6 +50,8 @@ pub struct RtcServe { pub serve_base: Option, /// Disable Content-Security-Policy pub csp: Option>, + /// Whether to serve precompressed sidecar files. `None` follows the build compression config. + pub precompressed: Option, } impl Deref for RtcServe { @@ -104,6 +106,7 @@ impl RtcServe { proxy_no_redirect: _, disable_csp, csp, + precompressed, } = config.serve; let tls = tls_config( @@ -127,6 +130,7 @@ impl RtcServe { tls, serve_base, csp: (!disable_csp).then_some(csp), + precompressed, }) } diff --git a/src/config/types/compression.rs b/src/config/types/compression.rs new file mode 100644 index 00000000..d50dacc0 --- /dev/null +++ b/src/config/types/compression.rs @@ -0,0 +1,53 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use strum::{Display, EnumString}; + +/// An algorithm used to pre-compress build assets into sidecar files. +#[derive( + Copy, Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema, Display, EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum CompressionAlgorithm { + /// gzip (RFC 1952), emitted as a `.gz` sidecar. + Gzip, + /// Brotli, emitted as a `.br` sidecar. + Brotli, +} + +impl CompressionAlgorithm { + /// The file extension (without leading dot) used for the sidecar file. + pub const fn extension(&self) -> &'static str { + match self { + Self::Gzip => "gz", + Self::Brotli => "br", + } + } +} + +/// How much effort to spend compressing assets, trading speed for size. +#[derive( + Copy, + Clone, + Debug, + Default, + PartialEq, + Eq, + Hash, + Deserialize, + Serialize, + JsonSchema, + Display, + EnumString, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum CompressionLevel { + /// Fastest, largest output. + Low, + /// Balanced speed and size (the default). + #[default] + Medium, + /// Slowest, smallest output. + High, +} diff --git a/src/config/types/mod.rs b/src/config/types/mod.rs index eee6f28c..f4dea971 100644 --- a/src/config/types/mod.rs +++ b/src/config/types/mod.rs @@ -2,6 +2,7 @@ mod address_family; mod base_url; +mod compression; mod cross_origin; mod duration; mod minify; @@ -10,6 +11,7 @@ mod ws; pub use address_family::*; pub use base_url::*; +pub use compression::*; pub use cross_origin::*; pub use duration::*; pub use minify::*; diff --git a/src/pipelines/compression.rs b/src/pipelines/compression.rs new file mode 100644 index 00000000..9c4ccfac --- /dev/null +++ b/src/pipelines/compression.rs @@ -0,0 +1,271 @@ +//! Build-time pre-compression of assets into sidecar files (e.g. `index.html.gz`). +//! +//! After all asset pipelines have written their output into the staging dist directory, this step +//! walks the directory and, for each configured algorithm, writes a compressed sidecar file next +//! to the original (e.g. `app.js` -> `app.js.gz`, `app.js.br`). Static file servers and CDNs can +//! then serve the precompressed variant based on the request's `Accept-Encoding` header. +//! +//! Compression is CPU-bound, so each (file, algorithm) job runs on the blocking thread pool via +//! [`tokio::task::spawn_blocking`], with up to one job per available core in flight at a time. A +//! live progress bar is shown per job (hidden automatically when stderr is not a terminal). + +use crate::config::{ + rt::RtcBuild, + types::{CompressionAlgorithm, CompressionLevel}, +}; +use anyhow::{Context, Result}; +use flate2::{Compression, write::GzEncoder}; +use futures_util::stream::{self, StreamExt, TryStreamExt}; +use indicatif::{HumanBytes, MultiProgress, ProgressBar, ProgressStyle}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tokio::fs; +use tokio_stream::wrappers::ReadDirStream; + +/// The chunk size used when feeding data to the encoders, so progress bars advance smoothly. +const CHUNK_SIZE: usize = 256 * 1024; +/// Brotli window size (`lgwin`); 22 is the library default and a good general choice. +const BROTLI_WINDOW: u32 = 22; + +/// A single compression job: one algorithm applied to one source file. +struct Job { + /// The source file to read and compress. + src: PathBuf, + /// The sidecar file to write (e.g. `app.js.br`). + sidecar: PathBuf, + /// A short human label for the progress bar (e.g. `br app.js`). + label: String, + /// The original file size, used as the progress bar length. + size: u64, + algorithm: CompressionAlgorithm, + level: CompressionLevel, + /// Keep the sidecar only if its size is at most this percentage of the original. + min_ratio_percent: u8, +} + +/// Compress the assets in the staging dist directory according to the build's compression config. +/// +/// This is a no-op when no compression algorithms are configured. +#[tracing::instrument(level = "trace", skip(cfg))] +pub async fn compress_dist(cfg: &RtcBuild) -> Result<()> { + if !cfg.compression.enabled() { + return Ok(()); + } + + let jobs = collect_jobs(cfg) + .await + .context("error scanning staging dist dir for compression")?; + if jobs.is_empty() { + return Ok(()); + } + + // One blocking job per available core keeps every core busy without oversubscribing. + let concurrency = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + + let multi = MultiProgress::new(); + let overall = multi.add(ProgressBar::new(jobs.len() as u64)); + overall.set_style(overall_style()); + overall.set_prefix("Compressing assets"); + + let results: Vec<(usize, u64)> = stream::iter(jobs) + .map(|job| { + let multi = multi.clone(); + let overall = overall.clone(); + async move { + let bar = multi.insert_before(&overall, ProgressBar::new(job.size)); + bar.set_style(job_style()); + bar.set_message(job.label.clone()); + bar.enable_steady_tick(Duration::from_millis(120)); + + let worker_bar = bar.clone(); + let result = tokio::task::spawn_blocking(move || run_job(job, &worker_bar)) + .await + .context("compression task panicked")?; + + bar.finish_and_clear(); + overall.inc(1); + result + } + }) + .buffer_unordered(concurrency) + .try_collect() + .await?; + + overall.finish_and_clear(); + + let written: usize = results.iter().map(|(count, _)| count).sum(); + let saved: u64 = results.iter().map(|(_, bytes)| bytes).sum(); + tracing::info!( + "compressed {written} asset sidecar(s), saved {}", + HumanBytes(saved) + ); + + Ok(()) +} + +/// Walk the staging dist dir and build the list of compression jobs (after applying all filters). +async fn collect_jobs(cfg: &RtcBuild) -> Result> { + let root = cfg.staging_dist.as_path(); + let mut jobs = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + + while let Some(dir) = stack.pop() { + let mut entries = fs::read_dir(&dir) + .await + .map(ReadDirStream::new) + .with_context(|| format!("error reading dir {dir:?}"))?; + while let Some(entry) = entries.next().await { + let entry = entry.with_context(|| format!("error reading entry in {dir:?}"))?; + let path = entry.path(); + let file_type = entry + .file_type() + .await + .with_context(|| format!("error reading file type of {path:?}"))?; + + if file_type.is_dir() { + stack.push(path); + continue; + } + if !file_type.is_file() || is_sidecar(&path) { + continue; + } + + let rel = path.strip_prefix(root).unwrap_or(&path); + if !cfg.compression.matches(rel) { + continue; + } + + let size = entry + .metadata() + .await + .with_context(|| format!("error reading metadata of {path:?}"))? + .len(); + if size < cfg.compression.min_size { + continue; + } + + let rel_label = rel.to_string_lossy().into_owned(); + for &algorithm in &cfg.compression.algorithms { + jobs.push(Job { + src: path.clone(), + sidecar: sidecar_path(&path, algorithm.extension()), + label: format!("{} {rel_label}", algorithm.extension()), + size, + algorithm, + level: cfg.compression.level, + min_ratio_percent: cfg.compression.min_ratio_percent, + }); + } + } + } + + Ok(jobs) +} + +/// Run a single compression job on a blocking thread. Returns `(sidecars_written, bytes_saved)`. +fn run_job(job: Job, bar: &ProgressBar) -> Result<(usize, u64)> { + let data = std::fs::read(&job.src) + .with_context(|| format!("error reading {:?} for compression", job.src))?; + bar.set_length(data.len() as u64); + + let compressed = encode(job.algorithm, job.level, &data, bar) + .with_context(|| format!("error compressing {:?} with {}", job.src, job.algorithm))?; + + // Only keep the sidecar if it is sufficiently smaller than the original. + let max_size = data.len() * job.min_ratio_percent as usize / 100; + if compressed.len() > max_size { + return Ok((0, 0)); + } + + std::fs::write(&job.sidecar, &compressed) + .with_context(|| format!("error writing compressed file {:?}", job.sidecar))?; + let saved = (data.len() - compressed.len()) as u64; + Ok((1, saved)) +} + +/// Compress `data` with the given algorithm and level, advancing `bar` as input is consumed. +fn encode( + algorithm: CompressionAlgorithm, + level: CompressionLevel, + data: &[u8], + bar: &ProgressBar, +) -> io::Result> { + match algorithm { + CompressionAlgorithm::Gzip => { + let mut encoder = GzEncoder::new(Vec::new(), gzip_level(level)); + write_chunked(&mut encoder, data, bar)?; + encoder.finish() + } + CompressionAlgorithm::Brotli => { + let mut encoder = brotli::CompressorWriter::new( + Vec::new(), + CHUNK_SIZE, + brotli_quality(level), + BROTLI_WINDOW, + ); + write_chunked(&mut encoder, data, bar)?; + Ok(encoder.into_inner()) + } + } +} + +/// Write `data` to `writer` in chunks, incrementing `bar` by the bytes consumed after each chunk. +fn write_chunked(writer: &mut W, data: &[u8], bar: &ProgressBar) -> io::Result<()> { + for chunk in data.chunks(CHUNK_SIZE) { + writer.write_all(chunk)?; + bar.inc(chunk.len() as u64); + } + Ok(()) +} + +/// Map a [`CompressionLevel`] to a gzip (DEFLATE) level (0-9). +fn gzip_level(level: CompressionLevel) -> Compression { + match level { + CompressionLevel::Low => Compression::new(1), + CompressionLevel::Medium => Compression::new(6), + CompressionLevel::High => Compression::new(9), + } +} + +/// Map a [`CompressionLevel`] to a brotli quality (0-11). +fn brotli_quality(level: CompressionLevel) -> u32 { + match level { + CompressionLevel::Low => 2, + CompressionLevel::Medium => 5, + CompressionLevel::High => 11, + } +} + +/// Build the sidecar path by appending `.` to the original file name. +fn sidecar_path(path: &Path, ext: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push("."); + name.push(ext); + PathBuf::from(name) +} + +/// Whether the path already looks like a compressed sidecar produced by this step. +fn is_sidecar(path: &Path) -> bool { + matches!( + path.extension().and_then(|ext| ext.to_str()), + Some("gz") | Some("br") + ) +} + +/// Progress bar style for the overall compression progress. +fn overall_style() -> ProgressStyle { + ProgressStyle::with_template("{prefix:.bold.cyan} {pos}/{len} {wide_bar:.cyan/blue} {elapsed}") + .unwrap_or_else(|_| ProgressStyle::default_bar()) + .progress_chars("=> ") +} + +/// Progress bar style for an individual compression job. +fn job_style() -> ProgressStyle { + ProgressStyle::with_template( + " {spinner:.green} {msg:<28} {bytes:>9}/{total_bytes:<9} {bar:20.green/dim}", + ) + .unwrap_or_else(|_| ProgressStyle::default_spinner()) +} diff --git a/src/pipelines/compression_test.rs b/src/pipelines/compression_test.rs new file mode 100644 index 00000000..e1873906 --- /dev/null +++ b/src/pipelines/compression_test.rs @@ -0,0 +1,197 @@ +use crate::config::rt::RtcBuild; +use crate::config::types::{CompressionAlgorithm, CompressionLevel}; +use crate::pipelines::compress_dist; +use anyhow::{Context, Result}; +use globset::{Glob, GlobSetBuilder}; +use std::io::Read; + +/// Build a test config rooted at a fresh tempdir with the given compression settings applied. +async fn test_cfg(algorithms: Vec) -> Result<(tempfile::TempDir, RtcBuild)> { + let tmpdir = tempfile::tempdir().context("error building tempdir")?; + let mut cfg = RtcBuild::new_test(tmpdir.path()).await?; + cfg.compression.algorithms = algorithms; + cfg.compression.min_size = 0; + cfg.compression.min_ratio_percent = 100; + Ok((tmpdir, cfg)) +} + +/// Write a file into the staging dist dir. +async fn write_staged(cfg: &RtcBuild, name: &str, bytes: &[u8]) -> Result<()> { + let path = cfg.staging_dist.join(name); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(path, bytes).await?; + Ok(()) +} + +async fn read_staged(cfg: &RtcBuild, name: &str) -> Result> { + Ok(tokio::fs::read(cfg.staging_dist.join(name)).await?) +} + +fn exists(cfg: &RtcBuild, name: &str) -> bool { + cfg.staging_dist.join(name).exists() +} + +fn gunzip(bytes: &[u8]) -> Result> { + let mut out = Vec::new(); + flate2::read::GzDecoder::new(bytes).read_to_end(&mut out)?; + Ok(out) +} + +fn unbrotli(bytes: &[u8]) -> Result> { + let mut out = Vec::new(); + brotli::Decompressor::new(bytes, 4096).read_to_end(&mut out)?; + Ok(out) +} + +#[tokio::test] +async fn ok_roundtrip_gzip_and_brotli() -> Result<()> { + let (_tmp, cfg) = test_cfg(vec![ + CompressionAlgorithm::Gzip, + CompressionAlgorithm::Brotli, + ]) + .await?; + // Highly compressible content so the ratio gate is satisfied. + let content = "trunk compresses assets\n".repeat(100).into_bytes(); + write_staged(&cfg, "index.html", &content).await?; + + compress_dist(&cfg).await?; + + let gz = read_staged(&cfg, "index.html.gz").await?; + let br = read_staged(&cfg, "index.html.br").await?; + anyhow::ensure!(gunzip(&gz)? == content, "gzip sidecar did not roundtrip"); + anyhow::ensure!( + unbrotli(&br)? == content, + "brotli sidecar did not roundtrip" + ); + Ok(()) +} + +#[tokio::test] +async fn all_levels_roundtrip() -> Result<()> { + let content = "the quick brown fox jumps over the lazy dog\n" + .repeat(200) + .into_bytes(); + for level in [ + CompressionLevel::Low, + CompressionLevel::Medium, + CompressionLevel::High, + ] { + let (_tmp, mut cfg) = test_cfg(vec![ + CompressionAlgorithm::Gzip, + CompressionAlgorithm::Brotli, + ]) + .await?; + cfg.compression.level = level; + write_staged(&cfg, "index.html", &content).await?; + + compress_dist(&cfg).await?; + + let gz = read_staged(&cfg, "index.html.gz").await?; + let br = read_staged(&cfg, "index.html.br").await?; + anyhow::ensure!(gunzip(&gz)? == content, "gzip roundtrip failed at {level}"); + anyhow::ensure!( + unbrotli(&br)? == content, + "brotli roundtrip failed at {level}" + ); + } + Ok(()) +} + +#[tokio::test] +async fn skips_files_below_min_size() -> Result<()> { + let (_tmp, mut cfg) = test_cfg(vec![CompressionAlgorithm::Gzip]).await?; + cfg.compression.min_size = 1024; + write_staged(&cfg, "small.txt", b"tiny").await?; + + compress_dist(&cfg).await?; + + anyhow::ensure!( + !exists(&cfg, "small.txt.gz"), + "expected no sidecar for a file below min_size" + ); + Ok(()) +} + +#[tokio::test] +async fn skips_when_ratio_not_met() -> Result<()> { + let (_tmp, mut cfg) = test_cfg(vec![CompressionAlgorithm::Gzip]).await?; + // A ratio of 0% means the sidecar must be 0 bytes to be kept, which never happens. + cfg.compression.min_ratio_percent = 0; + write_staged( + &cfg, + "index.html", + &"compressible\n".repeat(100).into_bytes(), + ) + .await?; + + compress_dist(&cfg).await?; + + anyhow::ensure!( + !exists(&cfg, "index.html.gz"), + "expected no sidecar when the compression ratio gate is not met" + ); + Ok(()) +} + +#[tokio::test] +async fn disabled_when_no_algorithms() -> Result<()> { + let (_tmp, cfg) = test_cfg(vec![]).await?; + write_staged(&cfg, "index.html", &"x".repeat(100).into_bytes()).await?; + + compress_dist(&cfg).await?; + + anyhow::ensure!( + !exists(&cfg, "index.html.gz") && !exists(&cfg, "index.html.br"), + "expected no sidecars when compression is disabled" + ); + Ok(()) +} + +#[tokio::test] +async fn respects_include_and_exclude_globs() -> Result<()> { + let (_tmp, mut cfg) = test_cfg(vec![CompressionAlgorithm::Gzip]).await?; + let mut include = GlobSetBuilder::new(); + include.add(Glob::new("*.txt")?); + cfg.compression.include = Some(include.build()?); + let mut exclude = GlobSetBuilder::new(); + exclude.add(Glob::new("skip.txt")?); + cfg.compression.exclude = Some(exclude.build()?); + + let content = "data\n".repeat(100).into_bytes(); + write_staged(&cfg, "keep.txt", &content).await?; + write_staged(&cfg, "skip.txt", &content).await?; + write_staged(&cfg, "image.png", &content).await?; + + compress_dist(&cfg).await?; + + anyhow::ensure!( + exists(&cfg, "keep.txt.gz"), + "included file should be compressed" + ); + anyhow::ensure!( + !exists(&cfg, "skip.txt.gz"), + "excluded file should be skipped" + ); + anyhow::ensure!( + !exists(&cfg, "image.png.gz"), + "non-included file should be skipped" + ); + Ok(()) +} + +#[tokio::test] +async fn compresses_files_in_subdirectories() -> Result<()> { + let (_tmp, cfg) = test_cfg(vec![CompressionAlgorithm::Gzip]).await?; + let content = "nested\n".repeat(100).into_bytes(); + write_staged(&cfg, "assets/app.js", &content).await?; + + compress_dist(&cfg).await?; + + let gz = read_staged(&cfg, "assets/app.js.gz") + .await + .context("expected sidecar for nested file")?; + anyhow::ensure!(gunzip(&gz)? == content, "nested sidecar did not roundtrip"); + Ok(()) +} diff --git a/src/pipelines/mod.rs b/src/pipelines/mod.rs index 1ed9b654..780249a8 100644 --- a/src/pipelines/mod.rs +++ b/src/pipelines/mod.rs @@ -1,3 +1,6 @@ +mod compression; +#[cfg(test)] +mod compression_test; mod copy_dir; #[cfg(test)] mod copy_dir_test; @@ -15,6 +18,7 @@ mod sass; mod tailwind_css; mod tailwind_css_extra; +pub use compression::compress_dist; pub use html::HtmlPipeline; use crate::{ diff --git a/src/serve/mod.rs b/src/serve/mod.rs index d77058cd..365e5660 100644 --- a/src/serve/mod.rs +++ b/src/serve/mod.rs @@ -2,7 +2,7 @@ mod proxy; use crate::{ common::{LOCAL, NETWORK, SERVER, nonce}, - config::rt::RtcServe, + config::{rt::RtcServe, types::CompressionAlgorithm}, tls::TlsConfig, watch::WatchSystem, ws, @@ -387,13 +387,34 @@ impl State { fn router(state: Arc, cfg: Arc) -> Result { // Build static file server, middleware, error handler & WS route for reloads. + // Determine which precompressed sidecars to serve. When `precompressed` is unset, follow the + // build's compression config; an explicit value forces serving both or neither. + let (serve_gzip, serve_br) = match cfg.precompressed { + Some(true) => (true, true), + Some(false) => (false, false), + None => { + let algorithms = &cfg.watch.build.compression.algorithms; + ( + algorithms.contains(&CompressionAlgorithm::Gzip), + algorithms.contains(&CompressionAlgorithm::Brotli), + ) + } + }; + let new_serve_dir = || { + let mut serve_dir = ServeDir::new(&state.dist_dir); + if serve_gzip { + serve_dir = serve_dir.precompressed_gzip(); + } + if serve_br { + serve_dir = serve_dir.precompressed_br(); + } + serve_dir + }; + let mut serve_dir = if cfg.no_spa { - get_service(ServeDir::new(&state.dist_dir)) + get_service(new_serve_dir()) } else { - get_service( - ServeDir::new(&state.dist_dir) - .fallback(ServeFile::new(state.dist_dir.join(INDEX_HTML))), - ) + get_service(new_serve_dir().fallback(ServeFile::new(state.dist_dir.join(INDEX_HTML)))) }; for (key, value) in &state.headers { let name = HeaderName::from_bytes(key.as_bytes())