diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 952ef41..e92ea75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,10 @@ name: CI on: push: - branches: [main, master] + branches: [main, master, e2e-rspec] pull_request: branches: [main, master] + workflow_dispatch: permissions: contents: read @@ -26,3 +27,72 @@ jobs: run: bundle exec rake spec - name: Run RuboCop run: bundle exec rake rubocop + + e2e: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: ['3.4.7', '4.0.1'] + steps: + - uses: actions/checkout@v4 + - name: Install zsh + run: sudo apt-get update && sudo apt-get install -y zsh + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + # The benchmark deliberately runs two scrubbed, unbundled subprocesses that + # resolve gems the way a real (non-bundler) install does: `ready up` spawns + # by-server, and render_production_source does `require "ready"` (which + # needs zeitwerk). With bundler-cache's path install those gems live only in + # the bundle, so the scrubbed processes can't see them. Put ready's runtime + # deps on the base PATH at their locked versions -- where rbenv has them + # locally and on the macos-e2e job. + - name: Install ready's runtime deps on the base PATH + run: gem install by:1.1.0 zeitwerk:2.8.2 + - name: Run E2E specs + run: bundle exec rake spec:e2e + + # macOS is where the harness actually runs for users (zsh is the default + # shell, executables resolve through rbenv). This job reproduces the + # process-hang/leak reports on a real Mac and fails loudly if the harness + # leaves any by-server or sandbox process behind. + macos-e2e: + runs-on: macos-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Cache the rbenv-built Ruby (first run compiles it, later runs restore) + uses: actions/cache@v4 + with: + path: ~/.rbenv/versions + key: rbenv-versions-4.0.1-${{ runner.os }} + - name: Install rbenv + Ruby 4.0.1 (matches a real user's setup) + run: | + brew install rbenv ruby-build coreutils + rbenv versions --bare | grep -qx 4.0.1 || rbenv install 4.0.1 + rbenv global 4.0.1 + echo "$HOME/.rbenv/shims" >> "$GITHUB_PATH" + - name: Bundle + run: | + gem install bundler + bundle install + # A hard gtimeout wraps each step so a regression hangs the job for + # seconds, not the 6h GitHub ceiling. The whole point of this job is that + # the harness must run to completion on macOS (default zsh, rbenv). + - name: Run the e2e suite + run: gtimeout -s KILL 300 bundle exec rake spec:e2e + - name: Run the real benchmark (what bin/bench runs) + run: gtimeout -s KILL 200 bundle exec rake bench BENCH_EXE=ri BENCH_ARGS=TCPServer BENCH_RUNS=2 BENCH_WARMUPS=1 + - name: Fail if the harness leaked any process + if: always() + run: | + pgrep -fl by-server || echo "no by-server" + pgrep -fl ready-e2e || echo "no ready-e2e" + leaked=$(pgrep -f 'by-server|ready-e2e' | wc -l | tr -d ' ') + echo "leaked process count: $leaked" + if [ "$leaked" != "0" ]; then + echo "::error::harness leaked $leaked process(es)" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 785d658..30dc624 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ /.bundle/ +/vendor/ +/.idea/ +/talk.tar.gz +# Local skill caches dropped by the assistant tooling; regenerated on reload. +/_claude/poodr/ +/_claude/rspec3/ +/_claude/zsh/ /.yardoc /_yardoc/ /coverage/ @@ -9,4 +16,6 @@ Gemfile.lock *.gem .rspec_status -references \ No newline at end of file +references +# benchmark run logs +/log/ diff --git a/.rubocop.yml b/.rubocop.yml index a8d4c3b..0e70ddd 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -17,10 +17,23 @@ plugins: AllCops: NewCops: enable - TargetRubyVersion: 4.0 + # Lint at the gem's supported floor (ready.gemspec's required_ruby_version), + # which CI also exercises -- not the 4.0.1 dev pin -- so accidental + # 4.0-only syntax is caught rather than shipped to 3.4 users. + TargetRubyVersion: 3.4 Exclude: - - bin/* + # Stock bundler scripts; bin/bench is ours and stays linted. + - bin/console + - bin/setup - vendor/**/* + # Vendored read-only reference copy of an external gem; not this project's + # code (it is also gitignored, so CI never sees it). + - references/**/* + # Standalone runtime helper scripts -- a doc formatter and the rdoc `ri` + # patch -- with their own conventions, not the gem's authored library code. + - extra/**/* + # Gitignored scratch: benchmark run logs and throwaway report-parsing scripts. + - log/**/* - lib/core_ext/**/* - rakelib/project.rb - rakelib/project_version.rb @@ -128,6 +141,12 @@ Claude/MysteryRegex: Style/MutableConstant: EnforcedStyle: literals +# Data.define subclassing keeps the class body a real class body, so constants +# holding instances of the class (e.g. Span::TABLE) live where they belong; +# the block form would bind them lexically to the enclosing namespace. +Style/DataInheritance: + Enabled: false + # Shared test contexts legitimately define many helpers. RSpec/MultipleMemoizedHelpers: Max: 10 @@ -151,6 +170,35 @@ Style/Documentation: Exclude: - "spec/**/*" +# E2E specs share one expensive by-server across examples via before(:all) and +# hold it in an instance variable; that is intentional here. +RSpec/BeforeAfterAll: + Exclude: + - "spec/e2e/**/*" + +RSpec/InstanceVariable: + Exclude: + - "spec/e2e/**/*" + +# E2E examples are longer by nature (shell/server setup + several aggregated +# structural assertions per example). +RSpec/ExampleLength: + Exclude: + - "spec/e2e/**/*" + +# The e2e and benchmark suites are grouped by test type, not class namespace, +# so their paths deliberately do not mirror the described constant. +RSpec/SpecFilePathFormat: + Exclude: + - "spec/e2e/**/*" + - "spec/bench/**/*" + +# Ready::Bench::Report is a waterfall renderer whose job is to write to stdout; +# it lives under spec/support only so Zeitwerk autoloads it. +RSpec/Output: + Exclude: + - "spec/support/bench/**/*" + # Trailing commas in multiline literals and arguments. Style/TrailingCommaInArrayLiteral: EnforcedStyleForMultiline: comma diff --git a/CLAUDE.md b/CLAUDE.md index bd621e7..686afd3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,6 +19,7 @@ The project has two halves that meet at generated code: `rakelib/ready.rake` is the orchestrator that drives half 1 to produce artifacts that half 2 consumes. +It uses zeitwerk ## Commands ```bash @@ -111,13 +112,22 @@ socket. Top-level `rake ready` = restart server + compile + clean. (`lib/ready/foo_bar.rb` → `Ready::FooBar`). A spec eager-loads everything and `rake zeitwerk:validate` checks naming — run it after adding/moving files. - **Ruby 4.0.1.** The code leans on modern syntax (`case/in` pattern matching, - the `it` block param, endless methods). CI pins 4.0.1. + the `it` block param). **No endless methods** (`def x = y`) — always regular + `def`/`end` blocks. CI pins 4.0.1. - **RuboCop is heavily customized** (see `.rubocop.yml`) via `rubocop-claude` (AI guardrails). Match the house style: double quotes, **no** frozen-string comment, trailing commas in multiline literals/args, dot-aligned multiline method chains, pipeline/`.then`-chaining style, short blocks (`Metrics/BlockLength` max 8), and every class carries an rdoc `##` comment (`Style/Documentation` is on). +- **Design rulings (owner review, binding):** name the domain — real objects + over primitive hashes; never return an array/tuple (use a `Data.define` + value object whose readers carry type and unit, e.g. `PtyShell::Result`); + no abbreviations in names; polymorphism over boolean/type flags; symbols + over string keys; `Pathname` everywhere (never `File.expand_path` or + `File.join`); `system` calls pass `exception: true` unless the result is + explicitly checked; intermediate variables over nested work-doing calls; + multi-variant docs as tables/bullets, never paragraphs. - **`references/command_kit.rb/` is a vendored, read-only reference copy** of an external gem (its own git repo). It is not part of this project — don't edit it or count it when reasoning about the codebase. diff --git a/Gemfile b/Gemfile index b6c451d..ae4a688 100644 --- a/Gemfile +++ b/Gemfile @@ -11,3 +11,4 @@ gem "rubocop-performance" gem "rubocop-rake" gem "rubocop-rspec" gem "ruby-lsp", "~> 0.26.10" +gem "youplot" diff --git a/Rakefile b/Rakefile index a8fc294..e3a9a67 100644 --- a/Rakefile +++ b/Rakefile @@ -1,9 +1,100 @@ +# Everything below develops the ready gem itself (gem build/release, specs, +# rubocop, benchmarks) and depends on dev-only gems. None of it is needed to RUN +# ready: the runtime `ready`/`compile`/`clobber`/`ready:*` tasks live in +# rakelib/ready.rake, which rake auto-loads independently of this file. When +# ready runs as an installed gem the gemspec and dev gems are absent, so loading +# this file would crash `ready up|compile|clobber` (e.g. bundler/gem_tasks +# raising "Unable to determine name from existing gemspec"). Load the dev tasks +# only in a source checkout, detected by the gemspec's presence. +return unless (Pathname(__dir__) / "ready.gemspec").exist? + require "bundler/gem_tasks" require "rspec/core/rake_task" RSpec::Core::RakeTask.new(:spec) +namespace :spec do + desc "Run the end-to-end (:e2e) specs (needs zsh + by/by-server)" + task :e2e do + ok = system({ "READY_E2E" => "1" }, RbConfig.ruby, "-S", "rspec", "--tag", "e2e") + abort("e2e specs failed") unless ok + end +end + +def bench_protocol + executable = ENV.fetch("BENCH_EXE", "ri") + # TCPServer is the default workload for the default tool only; any other + # executable runs bare unless BENCH_ARGS says otherwise. + default_arguments = executable == "ri" ? "TCPServer" : "" + Ready::Bench::Protocol.new( + executable_name: executable, + arguments: ENV.fetch("BENCH_ARGS", default_arguments).split, + preload_gems: bench_preload_gems, + rounds: Integer(ENV.fetch("BENCH_RUNS", "15")), + warmups: Integer(ENV.fetch("BENCH_WARMUPS", "3")), + ) +end + +# The hot server preloads the gems a readyfile declares (BENCH_READYFILE), +# falling back to rdoc, which ships the default tool. The build_dir is +# irrelevant here -- only gem names are read -- but Readyfile requires an +# existing directory, so the readyfile's own parent satisfies it. +def bench_preload_gems + readyfile_path = ENV.fetch("BENCH_READYFILE", nil) + return ["rdoc"] if readyfile_path.nil? + + readyfile_path = Pathname(readyfile_path) + Ready::Readyfile.open(readyfile_path, build_dir: readyfile_path.expand_path.parent).gem_names +end + +def bench_runner + require "ready" + require "zeitwerk" + Zeitwerk::Loader.new.tap do |loader| + loader.inflector.inflect("cli" => "CLI") + loader.push_dir(Pathname(__dir__) / "spec/support", namespace: Ready) + loader.setup + end + Ready::Bench::Runner.new(protocol: bench_protocol) +end + +# Appends this run's headline numbers so a caller sequencing several +# benchmarks (bin/bench --plot) can chart them afterwards. +def export_bench_results(runner, results_path) + results = Ready::Bench::ResultsLog.new(results_path) + results.append(command: runner.invocation, arm: :cold, + full_milliseconds: runner.cold_summary.duration_of(:full)) + results.append(command: runner.invocation, arm: :hot, + full_milliseconds: runner.hot_summary.duration_of(:full)) +end + +# Plot mode (BENCH_RESULTS set by bin/bench --plot) exports the headline +# numbers for the CLI to chart and stays silent on stdout, so the plot is the +# only output; otherwise the full waterfall report is the output. Either way +# the run narrates its progress to stderr. +def run_bench(verbose:) + runner = bench_runner.call + results_path = ENV.fetch("BENCH_RESULTS", nil) + if results_path + export_bench_results(runner, results_path) + else + runner.render(verbose:) + end +end + +desc "Print the cold-vs-hot startup waterfall (needs zsh + by-server + rbenv); bin/bench is the front door" +task :bench do + run_bench(verbose: false) +end + +namespace :bench do + desc "rake bench plus a legend table explaining every span row" + task :verbose do + run_bench(verbose: true) + end +end + require "rubocop/rake_task" RuboCop::RakeTask.new diff --git a/_claude/e2e_evaluation.md b/_claude/e2e_evaluation.md new file mode 100644 index 0000000..f742851 --- /dev/null +++ b/_claude/e2e_evaluation.md @@ -0,0 +1,89 @@ +# Evaluating the PTY/expect harness as an rspec E2E basis for `ready` + +**Date:** 2026-07-11 · Branch: `e2e-rspec` (off `cli-polish`) + +## Verdict: strong yes — and it's proven, not theoretical + +I extracted the harness's core and ran a two-layer proof-of-concept against this +repo: + +- **Layer 1** — the `PtyShell` core drives a real `zsh -f -i` under a pty; the + marker sync correctly handles the command-echo/output double-print; timing is + captured; clean teardown. (`scratchpad/poc_layer1.rb`) +- **Layer 2 (the real thing)** — built a sandbox (`ready up` with a `.readyfile` + of `executables: [rake]`), then over the pty: sourced the plugin, confirmed + `whence -v rake` → `rake is an alias for ready_rake`, ran `rake --version`, and + got `rake, version 13.4.2` back **through the compiled stub → `ready_by` → the + live `by-server`** in ~80ms. (`scratchpad/poc_layer2.rb`) + +So the approach can test `ready` fully end-to-end from Ruby (hence rspec), and it +simultaneously proved the runtime dispatch actually works. + +## Why PTY is the *right* (and only) approach for `ready` + +`ready`'s value lives entirely in the zsh runtime: the plugin bootstrap +(`readyinit`), aliasing bare commands to `ready_*` stubs, the stubs' RUBYOPT +sanitization, completions, and dispatch to the persistent `by-server`. None of +that is reachable from a subprocess or a non-interactive shell — it needs a real +interactive zsh with a tty. The existing rspec suite (`spec/ready/`) covers the +*generator* (Ruby that emits stubs); it structurally cannot cover whether a +generated stub, loaded into a live shell, dispatches and returns correct output. +PTY closes exactly that gap. + +## Reuse as-is (the gold) + +- **`PtyShell` core** — pty spawn + marker-based completion detection. The + quote-split marker (`DO''NE1`) avoids the command-echo false-match (visible in + Layer 1's double echo). Keep verbatim as `spec/support/pty_shell.rb`. +- **Loud failure on desync** — `expect!` raises with the parked tail buffer on + timeout/EOF, so a hung shell fails the example instead of hanging silently. +- **Independent outside clock** — pty-observed wall time enables perf-regression + assertions (the whole point of `ready`). + +## Benchmark-specific (adapt or drop for functional E2E) + +`Marks` / `Report` / `Runner`, the `/home/claude/prof` marks log + `prof.zsh`, +`zprof`, and the hardcoded `kamal_hot/cold` arms + `2.12.0` check are profiling +scaffolding. Functional E2E only needs "source plugin → run stub → assert +output/exit". Keep the waterfall machinery for a *separate*, optional +perf-regression spec. + +## Mapping into rspec + +- `spec/support/pty_shell.rb` — extracted `PtyShell`. +- A sandbox shared context: temp `READY_PREFIX` + `.readyfile`, run `ready up` + once in `before(:all)`, kill the by-server + rm the socket in `after(:all)`. + Bring the server up once and reuse it across examples (bring-up is the slow + part, ~seconds; a stub roundtrip is ~80ms). +- Examples: `source `, assert the alias exists, run the stub, assert + output/exit, optionally `expect(wall_ms).to be < N`. (Exactly the PoC, + restructured.) + +## Risks / caveats (all manageable) + +1. **CI needs zsh + a pty.** GitHub Actions ubuntu/macOS have both — add `zsh` to + the workflow. Not Windows. +2. **`expect` is stdlib but a *bundled* gem now.** Present here + (`.../4.0.0/expect.rb`); for portability add `gem "expect"` (and `pty` is a + default gem) to the test group. +3. **Server lifecycle & isolation.** Use a temp `READY_PREFIX`/socket (never + `/tmp/ready`) and guarantee teardown — an orphaned `by-server` is exactly what + lingered twice during this PoC. Use `after(:all)` (kill pid + rm socket) plus + an `at_exit` safety net. +4. **Speed.** Keep E2E to a handful of high-value examples; tag them `:e2e` so + the fast unit loop can exclude them and CI can run them. +5. **Flakiness.** Marker sync makes it deterministic, but interactive tty timing + can wobble under CI load. `zsh -f` (no rc, no theme escapes) + a controlled + `PS1` + generous timeout mitigate. Consider wrapping each example in an overall + `Timeout.timeout` since IO#expect's timeout is inter-character, not a total + deadline. +6. **Buildable `ready` assumed.** The PoC needed a working `ready up` (now true + after the `cli-polish` fixes and the rvm removal). In clean-rbenv CI this holds. + +## Recommendation + +Adopt it. Extract `PtyShell` into `spec/support/`, add a sandboxed `ready up` +fixture, write a small `:e2e` spec asserting dispatch works (and optionally that +hot latency beats a cold baseline), and keep the `Marks`/`zprof` waterfall as an +optional separate perf harness. This gives `ready` the one test class it lacks: +proof that a generated stub actually works in a live shell. diff --git a/bench/prof.zsh b/bench/prof.zsh new file mode 100644 index 0000000..0c97b6e --- /dev/null +++ b/bench/prof.zsh @@ -0,0 +1,46 @@ +# Benchmark harness helpers. Source into a controlled `zsh -f` shell. +# +# Marks are " " lines appended to +# $READY_MARKS. $EPOCHREALTIME is CLOCK_REALTIME, the same wall clock Ruby's +# Process::CLOCK_REALTIME reads, so zsh and Ruby marks subtract cleanly. +zmodload zsh/datetime + +# bench_mark +# Appends one mark for the current run. Called from inside measured regions +# (the generated hot stub marks command_start with it), so it stays a bare +# one-line append -- no option juggling to keep its own cost negligible. +# The timestamp is expanded before the redirection opens the file, so a +# mark's own ~50us write cost always lands in the span it OPENS, never the +# span it closes; instrument self-cost can't inflate the span being reported. +bench_mark() { + print -r -- "${READY_RUN_ID} $1 ${EPOCHREALTIME}" >> $READY_MARKS +} + +# bench_harness -- +# Runs one command under the harness, bracketing it with the harness_start / +# harness_end marks. The clock is read bare on both sides and the log writes +# are deferred until after the run, so the ~48us append never lands inside the +# measured span. Locals are declared up front for the same reason: nothing +# runs between the first clock read and the command but the command itself. +bench_harness() { + emulate -L zsh + setopt extendedglob + + # `status` is a read-only zsh special (a synonym for $?), so the exit code + # must land in a differently named local or the assignment errors. + local run_id= start_time= end_time= exit_code= + + run_id=$1 + shift + [[ $1 = -- ]] && shift + export READY_RUN_ID=$run_id + + start_time=${EPOCHREALTIME} + "$@" + exit_code=$? + end_time=${EPOCHREALTIME} + + print -r -- "${run_id} harness_start ${start_time}" >> $READY_MARKS + print -r -- "${run_id} harness_end ${end_time}" >> $READY_MARKS + print -r -- "${run_id} exit_status ${exit_code}" >> $READY_MARKS +} diff --git a/bench/results.md b/bench/results.md new file mode 100644 index 0000000..6c2e133 --- /dev/null +++ b/bench/results.md @@ -0,0 +1,144 @@ +# `ready`: cold vs. warm startup on real Ruby CLI commands + +Benchmark of **6 real, non-trivial commands** (not `--version` probes), each run +**cold** (a fresh Ruby boot per invocation, through an instrumented copy of its +real rubygems stub) versus **ready** (a zsh function stub dispatching to a warm, +persistent `by-server`). Each command's readyfile preloads that CLI's own library +so the warm server is actually warm. + +Every number is **verified**: the harness tees each run's stdout+stderr to +`log/bench.log` and aborts the whole benchmark if any run exits non-zero. Every +hot-arm run here was confirmed to produce the same real output as running the +command directly — the `ri` docs, the `colorls` listing, the hex string, the +`youplot` chart, the `kamal` tree, the `codeball` pack — so a command that failed +or no-op'd cannot masquerade as a fast "success." + +## TL;DR + +- Warm dispatch has a **flat ~29–62 ms floor** regardless of how heavy the cold + command is (`kamal` cold 485 ms → warm 31 ms; `ri` cold 510 ms → warm 38 ms). +- Speedups run **4.4×–15.5×**; every command saves **140–470 ms per invocation**. +- The cold cost is: a shared **~10 ms VM boot** (ready pays it too — its `by` + client boots a VM), a **~40 ms rubygems load** on top (cold only), a + **dependency-resolution cost that scales with graph size** (`activate deps`: + 30 ms for most → **350 ms for `ronin`**), and **the command's real work** + (83 ms `youplot` → 409 ms `ri`). `ready` keeps the boot and eliminates + everything above it. + +## How it's measured + +| | | +|---|---| +| Host | GitHub Codespace, Linux x86_64 (shared runner) | +| Ruby | 4.0.1 (+PRISM), via rbenv | +| Tool | `bin/bench` (this repo), 5 measured rounds + 1 warmup, order-alternating | +| Statistic | **floor (minimum across rounds)** — the structural cost; medians run ~10–30% higher | +| Verification | each run must exit 0; every hot run cross-checked to emit the command's real output | + +Three commands were adapted so **both arms do identical work** (the cold and hot +arms run in different working directories, and the harness passes arguments +space-separated): + +- **`colorls`** — pinned to a fixed directory (`-l /lib/ready`) instead of + the cwd. +- **`youplot`** — the pipeline `seq 1 100 | awk '{print $1,$1^2}'` was pre-rendered + to a data file, passed as a file argument (not piped); title `y=x^2` (the + original `"y = x^2"` has spaces the harness can't carry in one argument). +- **`codeball`** — a fixed Ruby file, identical in both arms. + +> **On "cold".** These are the directly-measured **in-shell** cold full (boot + +> rubygems + activation + the command). The rbenv shim is reported +> [separately](#the-rbenv-shim-reported-separately) because this box's rbenv is +> pathologically slow and noisy. + +## Speedup (in-shell cold vs. warm), by ratio + +| command | in-shell cold (ms) | ready (ms) | speedup | saved (ms) | +|---|--:|--:|--:|--:| +| `kamal accessory tree` | 485 | 31 | **15.5×** | 454 | +| `ri --no-pager --format=ansi TCPServer` | 510 | 38 | **13.5×** | 472 | +| `ronin encode --hex --string rubyconf2026` | 523 | 62 | **8.4×** | 461 | +| `codeball pack ` | 197 | 29 | **6.7×** | 167 | +| `colorls -l ` | 197 | 30 | **6.5×** | 166 | +| `youplot line … -t y=x^2 ` | 184 | 42 | **4.4×** | 142 | + +## Where the cold cost goes (ms, floor) + +| command | shell | ruby vm boot | rubygems | activate deps | the command | in-shell cold | ready | +|---|--:|--:|--:|--:|--:|--:|--:| +| `youplot` | 5 | 10 | 41 | 30 | 83 | **184** | 42 | +| `colorls` | 5 | 10 | 42 | 34 | 98 | **197** | 30 | +| `codeball` | 4 | 9 | 40 | 30 | 103 | **197** | 29 | +| `kamal` | 5 | 9 | 40 | 49 | 371 | **485** | 31 | +| `ri` | 5 | 9 | 41 | 31 | 409 | **510** | 38 | +| `ronin` | 5 | 9 | 42 | **350** | 103 | **523** | 62 | + +*(**`ruby vm boot`** is the bare `--disable-gems` VM spawn — a ~10 ms floor **both +cold and ready pay** (ready's `by` client boots a VM too). **`rubygems`** is the +framework load *on top* of the boot, cold-only. `the command` = loading and doing +the actual work, including `require`-ing its dependencies. Rows sum to at most +`in-shell cold` — the gap is process reap plus min-of-parts slack.)* + +## What the layers mean + +**`ruby vm boot` is a shared floor (~9–10 ms) — `ready` does *not* remove it.** +Both arms spawn a Ruby VM (`ruby --disable-gems`): cold via the command's stub, +ready via its `by` client. It's the price of having a Ruby process at all. + +**`rubygems` is the ~40 ms load *on top* of the boot** — `require "rubygems"`, +constant across commands. This is the cold-only part: `ready`'s booted VM connects +a socket instead of loading rubygems, so the server pays it **once**. This — not +the boot — is what "ready eliminates rubygems" means. + +**`activate deps` scales with the *dependency graph*, not the work.** It's +`Gem.activate_bin_path` resolving the transitive gem graph onto `$LOAD_PATH` — +~30 ms for most, but **350 ms for `ronin`** even though `ronin encode` is a tiny +subcommand: the CLI still activates ronin's whole ~135-gem world before running. + +**`the command` is the real work** — and with realistic workloads it's the honest +cost, not a `--version` short-circuit: `ri` renders TCPServer's docs in ANSI +(409 ms), `kamal` boots its full CLI (371 ms), `youplot` renders the chart +(83 ms). This is what the warm server has already paid. + +**Warm dispatch is a flat ~20 ms floor plus a little residual.** `ready`'s +dispatch overhead (by-client boot + socket round-trip) is ~20 ms for every +command; the rest is whatever per-invocation work the command still does warm +(`ronin` is highest at 62 ms — its command still runs in the worker). Independent +of cold cost, because the server already paid boot, rubygems, activation, and the +command's requires. + +## The rbenv shim (reported separately) + +A real cold invocation also pays the rbenv **shim** on `$PATH` before the Ruby +process starts; `ready` eliminates it (a zsh function — no lookup, fork, or exec). +On this codespace the shim is **pathological** (`~/.rbenv/shims/ruby` ≈ 670 ms +floor, `rbenv exec` swings 50→650 ms — almost certainly its RVM/rbenv `$PATH` +conflict) and the per-command probe is noisy, so it's excluded from the tables +rather than folded in. A healthy rbenv shim is ~20–40 ms; directionally `ready` +removes it, and on a broken setup that alone is worth hundreds of ms. + +## Known limits (from earlier runs) + +`ready` can't dispatch a CLI whose stub computes its load path from `__FILE__`/ +`__dir__` (bogus under `by`'s `eval` — e.g. `rbs`, `rougify`), or one whose +executable ships in a differently-named gem it can't resolve (`rspec`, from +`rspec-core`). None of the 6 commands above hit these; they're noted because the +exit-check is what surfaces them instead of timing a silent failure. + +## Reproduce + +Per command, a readyfile preloads its library (`gems:` entries are **require +paths**), then: + +``` +bin/bench --readyfile --rounds 5 --warmups 1 -- +``` + +| command | preload (`gems:`) | exact invocation | +|---|---|---| +| `ri` | `rdoc` | `ri --no-pager --format=ansi TCPServer` | +| `colorls` | `colorls` | `colorls -l /lib/ready` | +| `ronin` | `ronin`, `ronin/support` | `ronin encode --hex --string rubyconf2026` | +| `youplot` | `youplot` | `youplot line -w 50 -h 15 -t y=x^2 ` (data = `seq 1 100 \| awk '{print $1,$1^2}'`) | +| `kamal` | `kamal` | `kamal accessory tree` | +| `codeball` | `codeball` | `codeball pack ` | diff --git a/bench/run_multibenchmark_test.sh b/bench/run_multibenchmark_test.sh new file mode 100755 index 0000000..481d7cb --- /dev/null +++ b/bench/run_multibenchmark_test.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# Push-button reproduction of the "ready on real CLI commands" benchmark +# (see bench/results.md). Run from anywhere: +# +# bash bench/run_multibenchmark_test.sh +# +# It installs the target gems if missing, builds the fixtures both arms share, +# and runs `bin/bench` for each command. Override the round counts with env: +# +# ROUNDS=15 WARMUPS=3 bash bench/run_multibenchmark_test.sh +# +# Prerequisites (the ready dev setup you already have): Ruby 4.0.1 via rbenv, +# zsh, the by/by-server gem, and `bundle install` done in this repo. +# Assumes the repo path has no spaces (the bench passes args space-separated). +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" +OUT="$ROOT/log/results"; FIX="$ROOT/log/fixtures" +mkdir -p "$OUT" "$FIX" +ROUNDS="${ROUNDS:-5}"; WARMUPS="${WARMUPS:-1}" + +say() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +# --- 1. dependencies (idempotent) ------------------------------------------ +say "1/4 Ensuring gems are installed" +bundle check >/dev/null 2>&1 || bundle install +for exe_gem in colorls:colorls youplot:youplot kamal:kamal codeball:codeball; do + exe="${exe_gem%%:*}"; gem="${exe_gem##*:}" + if command -v "$exe" >/dev/null 2>&1; then echo " $exe: present" + else echo " installing $gem..."; gem install "$gem" --no-document; fi +done + +# codeball needs the native filemagic gem (libmagic). Best-effort: if it won't +# load, codeball is skipped rather than failing the whole run. +CODEBALL_OK=1 +if ! ruby -e "require 'filemagic'" >/dev/null 2>&1; then + echo " installing libmagic + ruby-filemagic (for codeball)..." + case "$(uname -s)" in + Darwin) command -v brew >/dev/null && brew list libmagic >/dev/null 2>&1 || brew install libmagic 2>/dev/null || true + gem install ruby-filemagic -- --with-magic-dir="$(brew --prefix libmagic 2>/dev/null)" 2>/dev/null || true ;; + Linux) (dpkg -s libmagic-dev >/dev/null 2>&1 || sudo apt-get install -y libmagic-dev) 2>/dev/null || true + gem install ruby-filemagic --no-document 2>/dev/null || true ;; + esac + ruby -e "require 'filemagic'" >/dev/null 2>&1 || { CODEBALL_OK=0; echo " !! filemagic unavailable -- skipping codeball"; } +fi + +# --- 2. fixtures shared by both arms --------------------------------------- +say "2/4 Building fixtures" +DATA="$FIX/youplot_data.txt"; seq 1 100 | awk '{print $1, $1^2}' > "$DATA" # y = x^2 +DIR="$ROOT/lib/ready" # colorls lists this +FILE="$ROOT/lib/ready/executable.rb" # codeball packs this +echo " data=$DATA dir=$DIR file=$FILE" + +# --- 3. the commands (exe -> preload gems, exact args) --------------------- +declare -A PRELOAD=( + [ri]="rdoc" [colorls]="colorls" [ronin]="ronin ronin/support" + [youplot]="youplot" [kamal]="kamal" [codeball]="codeball" +) +declare -A ARGS=( + [ri]="--no-pager --format=ansi TCPServer" + [colorls]="-l $DIR" + [ronin]="encode --hex --string rubyconf2026" + [youplot]="line -w 50 -h 15 -t y=x^2 $DATA" + [kamal]="accessory tree" + [codeball]="pack $FILE" +) +ORDER=(ri colorls ronin youplot kamal codeball) +[ "$CODEBALL_OK" = 1 ] || ORDER=(ri colorls ronin youplot kamal) + +# --- 4. run ---------------------------------------------------------------- +say "4/4 Benchmarking ($ROUNDS rounds, $WARMUPS warmup)" +: > "$OUT/summary.txt" +for cli in "${ORDER[@]}"; do + rf="$OUT/rf_$cli" + { echo "gems:"; for g in ${PRELOAD[$cli]}; do echo " - $g"; done + echo "executables:"; echo " - $cli"; } > "$rf" + echo ">>> $cli ${ARGS[$cli]}" + if ./bin/bench --readyfile "$rf" --rounds "$ROUNDS" --warmups "$WARMUPS" \ + -- "$cli" ${ARGS[$cli]} > "$OUT/out_$cli.txt" 2>&1; then + line=$(grep -m1 'ready is [0-9]' "$OUT/out_$cli.txt" || echo '(no headline)') + printf ' %-10s %s\n' "$cli" "$line" | tee -a "$OUT/summary.txt" + else + printf ' %-10s FAILED -- see %s and log/bench.log\n' "$cli" "$OUT/out_$cli.txt" | tee -a "$OUT/summary.txt" + fi +done + +say "Done. Summary:"; cat "$OUT/summary.txt" +echo; echo "Full per-command reports: $OUT/out_.txt | all output: $ROOT/log/bench.log" diff --git a/bin/bench b/bin/bench new file mode 100755 index 0000000..8bc6568 --- /dev/null +++ b/bin/bench @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby +# Development-only ergonomic wrapper around the proven benchmark rake tasks: +# flags become the BENCH_* environment and rake bench / bench:verbose do the +# work. Lives in bin/ and never ships in the gem. + +require "bundler/setup" +require "ready" +require "zeitwerk" + +Zeitwerk::Loader.new.tap do |loader| + loader.inflector.inflect("cli" => "CLI") + loader.push_dir(Pathname(__dir__).parent / "spec/support", namespace: Ready) + loader.setup +end + +Ready::Bench::CLI.start diff --git a/demo/demofns.zsh b/demo/demofns.zsh new file mode 100644 index 0000000..663fda2 --- /dev/null +++ b/demo/demofns.zsh @@ -0,0 +1,34 @@ +unready_shell() { + local fnname=$0 + local i + local -a fns=(${functions[(I)*ready_*]}) + local -a toremove=(${0} reready) + local -a raliases + fns=(${fns:|toremove}) + + for i in $fns; do + print -u2 "Removing fn ${(qqq)i}" + unfunction $i + done + + raliases=(${(k)aliases[(R)*ready*]}) + + for i in $raliases; do + print -u2 "Removing alias ${(qqq)i}" + unalias $i + done +} + +reready() { + unready_shell + command pkill -f by-server + command ready clobber + yes | command gem uninstall ready || true + command bundle exec rake clobber + command bundle exec rake build + command gem install \ + --conservative \ + --no-document \ + --no-update-sources \ + --local pkg/ready* +} \ No newline at end of file diff --git a/demo/rubyconf.rec b/demo/rubyconf.rec new file mode 100644 index 0000000..0dbc977 --- /dev/null +++ b/demo/rubyconf.rec @@ -0,0 +1,40 @@ +%rec: Issue +%key: Id +%typedef: text_t regexp /^.*$/ +%typedef: Owner_t enum me claude +%typedef: Status_t enum open in_progress complete +%type: Id int +%type: Title line +%type: Desc text_t +%type: Owner Owner_t +%type: Status Status_t +%type: Updated date +%auto: Id Updated +%mandatory: Title Desc Owner Status + +Id: 0 +Updated: Mon, 13 Jul 2026 20:59:56 -0500 +Title: Tough items still not figured out +Desc: ++ 1. By needs a few patches: ++ a. Need to provide process substitution support - the process sub fds also need to be passed via send_io ++ 2. Interactives like ssh, fzf, irb and friends can be unpredictable and extremely difficult to debug/test ++ a. kamal ssh not working e2e ++ b. irb occasionally spits out bizarre errors ++ c. SIGINT currently kills the whole process, rather than propogating to the interactive +Owner: me +Status: open + +Id: 1 +Updated: Mon, 13 Jul 2026 21:08:41 -0500 +Title: ready compile should default to all - shouldn't be required +Desc: +Owner: me +Status: open + +Id: 2 +Updated: Mon, 13 Jul 2026 22:06:59 -0500 +Title: Interesting quirks +Desc: 1. bundler/setup and bundle exec were causing major issues, took a lot of debugging but eventually traced the RUBYOPT env var being set by bundler. the zsh stubs surgically remove that env variable before loading the stub (doesn't affect global env, just the run) +Owner: me +Status: open diff --git a/docs/superpowers/plans/2026-07-12-phase-c-layer-benchmark.md b/docs/superpowers/plans/2026-07-12-phase-c-layer-benchmark.md new file mode 100644 index 0000000..b270c39 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-phase-c-layer-benchmark.md @@ -0,0 +1,640 @@ +# Phase C — Layered Cold-vs-Hot Startup Benchmark Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** A benchmark that attributes a rubygems-CLI startup to its layers (shell / rbenv_shim / ruby_boot / rubygems / dep_activate / tool_run / reap) and shows each collapse when served hot by `by-server` — surfaced as `rake bench` (prints the waterfall) and a `:e2e` structural guard. + +**Architecture:** Both arms are instrumented with faithful **copies** in temp dirs (zero real-file edits, zero gem-runtime changes). Cold = instrumented rbenv-shim + rubygems-stub copies + a RUBYOPT prelude; hot = a real `Ready::Sandbox` server plus a stub generated from `Ready::Executable#render` with marks injected. Marks are `CLOCK_REALTIME`/`$EPOCHREALTIME` lines in a shared file; pure parsing/aggregation is unit-tested, the arms are `:e2e`. Reuses `Ready::{Sandbox,PtyShell,Executable,Bench::{Marks,Stats,Report}}`. + +**Tech Stack:** Ruby 3.4.7+/4.0.1, RSpec, zsh 5.9 (`zmodload zsh/datetime`), rbenv, `by`/`by-server`, `CLOCK_REALTIME`. + +## Global Constraints + +- **Design ref:** `docs/superpowers/specs/2026-07-12-phase-c-layer-benchmark-design.md` (layer table, marks, numbers). Read it. +- **Zero production-runtime edits.** All instrumentation is generated copies in temp dirs. Do NOT modify `lib/ready/fn.zsh.erb`, `lib/ready/executable.rb`, or any real gem/rbenv file. +- **Zeitwerk under `Ready`.** New classes live in `spec/support/bench/` → `Ready::Bench::*` (autoloaded by the spec/support loader). Committed fixtures go in `bench/`. +- **Clock:** `$EPOCHREALTIME` (zsh, after `zmodload zsh/datetime`) == `Process.clock_gettime(Process::CLOCK_REALTIME)` (Ruby). One shared append-only markfile; lines `" "`. +- **Aggregation policy:** min/floor for process-creation spans (`shell`, `rbenv_shim`, `dispatch_infra`, full envelope), median for in-process spans. ≥15 warm runs; discard warmups (first run inflates ~2×). +- **Isolation:** temp dirs only; unset `GEM_HOME`/`GEM_PATH` when the dir is missing (as `Ready::Sandbox#build_env` does); mandatory teardown (by-server stop + kill by argv + `rm -rf`); assert zero `by-server` + zero temp dirs after. +- **House style / rubocop:** library files (`spec/support/bench/*`) rubocop-clean; specs match existing conventions. Bench classes need no `##` rdoc (spec/** excluded) but add short ones. +- **Target:** `irb` (default rep) via `gems: [irb]` preload; `rake` fallback. Not coupled to any tool. + +--- + +## Task 1: Ready layer span model + aggregation + +**Files:** +- Modify: `spec/support/bench/marks.rb` (replace `SPANS` with the ready layer model; keep `parse`/`spans`) +- Create: `spec/support/bench/aggregator.rb` → `Ready::Bench::Aggregator` +- Modify/Test: `spec/bench/marks_spec.rb`; Create `spec/bench/aggregator_spec.rb` + +**Interfaces:** +- Produces: `Ready::Bench::Marks::SPANS` (Array of `[label, from, to]` for the ready layers); `Ready::Bench::Marks.parse(path)`, `.spans(marks)` (unchanged signatures). +- Produces: `Ready::Bench::Aggregator.new(span_kind:)` where `span_kind` is a `Hash{label => :min|:median}`; `#combine(list_of_span_hashes) -> {label => ms}` applying min or median per label. + +- [ ] **Step 1: Update the marks_spec fixture + expectations to the ready spans** — `spec/bench/marks_spec.rb`: change the fixture log and assertions to the ready layer marks. + +```ruby +require "tempfile" + +RSpec.describe Ready::Bench::Marks do + def fixture(contents) + file = Tempfile.new("marks") + file.write(contents) + file.close + file.path + end + + let(:cold) do + fixture(<<~LOG) + cold.1 envelope_start 1000.000 + cold.1 shim_start 1000.001 + cold.1 ruby_up 1000.050 + cold.1 rubygems_ready 1000.100 + cold.1 dep_activated 1000.140 + cold.1 ruby_exit 1000.243 + cold.1 envelope_end 1000.244 + LOG + end + + it "parses a run into a mark=>time map" do + expect(described_class.parse(cold)["cold.1"]["dep_activated"]).to eq(1000.140) + end + + it "computes ready layer spans in ms" do + spans = described_class.spans(described_class.parse(cold)["cold.1"]) + aggregate_failures do + expect(spans["rbenv_shim"]).to be_within(1e-6).of(49.0) # shim_start->ruby_up + expect(spans["rubygems"]).to be_within(1e-6).of(50.0) # ruby_up->rubygems_ready + expect(spans["dep_activate"]).to be_within(1e-6).of(40.0) # rubygems_ready->dep_activated + expect(spans["tool_run"]).to be_within(1e-6).of(103.0) # dep_activated->ruby_exit + expect(spans["full"]).to be_within(1e-6).of(244.0) # envelope_start->envelope_end + end + end + + it "omits spans whose endpoints are missing (hot arm has no shim)" do + hot = { "envelope_start" => 1.0, "server_entry" => 1.053, "envelope_end" => 1.072 } + spans = described_class.spans(hot) + aggregate_failures do + expect(spans).not_to have_key("rbenv_shim") + expect(spans["dispatch_infra"]).to be_within(1e-6).of(53.0) + expect(spans["full"]).to be_within(1e-6).of(72.0) + end + end +end +``` + +- [ ] **Step 2: Run → fail** — `bundle exec rspec spec/bench/marks_spec.rb` (fails: old SPANS lack these labels). + +- [ ] **Step 3: Replace `SPANS`** — `spec/support/bench/marks.rb`, swap the `SPANS` constant (keep `parse`/`spans` bodies): + +```ruby + SPANS = [ + ["shell", "envelope_start", "shim_start"], + ["rbenv_shim", "shim_start", "ruby_up"], + ["rubygems", "ruby_up", "rubygems_ready"], + ["dep_activate", "rubygems_ready", "dep_activated"], + ["tool_run", "dep_activated", "ruby_exit"], + ["reap", "ruby_exit", "envelope_end"], + ["dispatch_infra", "envelope_start", "server_entry"], + ["server_tool_run", "server_entry", "envelope_end"], + ["full", "envelope_start", "envelope_end"], + ].freeze +``` + +Remove the `preamble` `from ||= ...` fallback line in `.spans` (no longer needed) so `.spans` is just the `each_with_object` over present endpoints. + +- [ ] **Step 4: Run → pass** — `bundle exec rspec spec/bench/marks_spec.rb` → 3 pass. + +- [ ] **Step 5: Aggregator test** — `spec/bench/aggregator_spec.rb`: + +```ruby +RSpec.describe Ready::Bench::Aggregator do + subject(:agg) { described_class.new(span_kind: { "shell" => :min, "tool_run" => :median }) } + + it "takes the min for process spans and median for in-process spans" do + runs = [ + { "shell" => 3.0, "tool_run" => 100.0 }, + { "shell" => 1.0, "tool_run" => 110.0 }, + { "shell" => 2.0, "tool_run" => 120.0 }, + ] + result = agg.combine(runs) + aggregate_failures do + expect(result["shell"]).to eq(1.0) + expect(result["tool_run"]).to eq(110.0) + end + end + + it "defaults an unlisted span to median" do + expect(described_class.new(span_kind: {}).combine([{ "x" => 4.0 }, { "x" => 2.0 }])["x"]).to eq(3.0) + end +end +``` + +- [ ] **Step 6: Run → fail** — `bundle exec rspec spec/bench/aggregator_spec.rb`. + +- [ ] **Step 7: Implement Aggregator** — `spec/support/bench/aggregator.rb`: + +```ruby +module Ready + module Bench + ## + # Collapses many per-run span hashes into one, applying min to + # process-creation spans (jittery, floor is the structural number) and + # median to in-process spans. + class Aggregator + def initialize(span_kind:) + @span_kind = span_kind + end + + def combine(runs) + labels = runs.flat_map(&:keys).uniq + labels.each_with_object({}) do |label, out| + values = runs.filter_map { |r| r[label] } + next if values.empty? + + out[label] = @span_kind[label] == :min ? values.min : Stats.median(values) + end + end + end + end +end +``` + +- [ ] **Step 8: Run → pass; lint; commit** + +```bash +bundle exec rspec spec/bench/marks_spec.rb spec/bench/aggregator_spec.rb +bundle exec rubocop spec/support/bench/marks.rb spec/support/bench/aggregator.rb +git add spec/support/bench/marks.rb spec/support/bench/aggregator.rb spec/bench/marks_spec.rb spec/bench/aggregator_spec.rb +git commit -m "bench: ready layer span model + min/median aggregator" +``` + +--- + +## Task 2: Committed instrumentation fixtures (prelude + prof.zsh) + +**Files:** +- Create: `bench/prelude.rb`, `bench/prof.zsh` +- Test: exercised by the `:e2e` arms (Tasks 3–4); a lightweight loadability check here. + +**Interfaces:** +- Produces: `bench/prelude.rb` — a RUBYOPT `-r` target that appends `ruby_up ` to `$READY_MARKS`. +- Produces: `bench/prof.zsh` — defines `bench_mark ` (held-fd append to `$READY_MARKS`) and `bench_envelope -- ` (bare-read `envelope_start`, run cmd, `envelope_end`). + +- [ ] **Step 1: Write `bench/prelude.rb`** (single statement; runs as the first Ruby user code): + +```ruby +# Emitted as the first Ruby user code via RUBYOPT=-r; marks interpreter-up. +File.write(ENV.fetch("READY_MARKS"), "#{ENV.fetch('READY_RUN_ID')} ruby_up #{Process.clock_gettime(Process::CLOCK_REALTIME)}\n", mode: "a") +``` + +- [ ] **Step 2: Write `bench/prof.zsh`**: + +```zsh +# Benchmark envelope + mark helpers. Source in a controlled `zsh -f` shell. +zmodload zsh/datetime + +# Append " " to $READY_MARKS using a held fd (fast). +bench_mark() { print -r -- "${READY_RUN_ID} $1 ${EPOCHREALTIME}" >> $READY_MARKS } + +# bench_envelope -- +# Bare-read envelope_start (deferred write), run cmd, read envelope_end first. +bench_envelope() { + local rid=$1; shift; [[ $1 == -- ]] && shift + export READY_RUN_ID=$rid + local __start=${EPOCHREALTIME} + "$@" + local __end=${EPOCHREALTIME} + print -r -- "${rid} envelope_start ${__start}" >> $READY_MARKS + print -r -- "${rid} envelope_end ${__end}" >> $READY_MARKS +} +``` + +- [ ] **Step 3: Smoke-check the fixtures load** — run: + +```bash +cd /workspaces/ready2 +zsh -f -c 'source bench/prof.zsh; READY_RUN_ID=t READY_MARKS=/tmp/m.$$ bench_mark hi; cat /tmp/m.$$; rm -f /tmp/m.$$' +ruby -e 'ENV["READY_MARKS"]="/tmp/p.#{Process.pid}"; ENV["READY_RUN_ID"]="t"; load "bench/prelude.rb"; puts File.read(ENV["READY_MARKS"]); File.delete(ENV["READY_MARKS"])' +``` +Expected: `t hi ` and `t ruby_up `. + +- [ ] **Step 4: Commit** + +```bash +git add bench/prelude.rb bench/prof.zsh +git commit -m "bench: RUBYOPT ruby_up prelude + prof.zsh envelope/mark helpers" +``` + +--- + +## Task 3: Cold arm — instrumented shim + stub copies + +**Files:** +- Create: `spec/support/bench/cold_arm.rb` → `Ready::Bench::ColdArm` +- Test: `spec/bench/cold_arm_spec.rb` (copy-generation, fast), plus `:e2e` run in Task 6. + +**Interfaces:** +- Consumes: `bench/prelude.rb`, `Ready.root`. +- Produces: `Ready::Bench::ColdArm.new(exe:, workdir:, marks_path:)`; `#instrument!` writes an instrumented shim copy (`workdir/`), a direct-shim copy (`workdir/_direct`), and an instrumented stub copy into `workdir`, returning nothing; `#command(run_id:, direct: false) -> Array` the argv to run the arm under a shell that has `READY_MARKS`/`READY_RUN_ID`/`RUBYOPT`/`PATH` set; `#real_shim -> Pathname` (`~/.rbenv/shims/`), `#real_stub -> Pathname` (`rbenv which `). +- The instrumented stub inserts marks by string-matching the standard rubygems stub lines (`require 'rubygems'`, `Gem.use_gemdeps`, the `Gem.activate_and_load_bin_path`/`load Gem.activate_bin_path` line) — the same shape verified across `irb`/`rake`/`rdoc`/`erb`. + +- [ ] **Step 1: Copy-generation test** — `spec/bench/cold_arm_spec.rb` (uses a synthetic stub so it's hermetic/fast): + +```ruby +require "tmpdir" + +RSpec.describe Ready::Bench::ColdArm do + it "injects marks into a rubygems-stub copy at the standard boundaries" do + Dir.mktmpdir do |dir| + stub = File.join(dir, "src_stub") + File.write(stub, <<~RUBY) + #!/usr/bin/ruby + require 'rubygems' + Gem.use_gemdeps + version = ">= 0.a" + Gem.activate_and_load_bin_path('irb', 'irb', version) + RUBY + out = described_class.instrument_stub(File.read(stub), name: "irb") + aggregate_failures do + expect(out).to match(/at_exit\b.*ruby_exit/m) + expect(out).to match(/Gem\.use_gemdeps\n.*rubygems_ready/m) + expect(out).to match(/dep_activated.*\n\s*Gem\.activate_and_load_bin_path/m) + end + end + end +end +``` + +- [ ] **Step 2: Run → fail.** `bundle exec rspec spec/bench/cold_arm_spec.rb`. + +- [ ] **Step 3: Implement `ColdArm`** — `spec/support/bench/cold_arm.rb`. Core `instrument_stub` (pure, tested) + the copy/command plumbing: + +```ruby +require "fileutils" + +module Ready + module Bench + ## + # Generates faithful, mark-instrumented COPIES of the real rbenv shim and + # rubygems stub for a gem executable, so a cold invocation can be attributed + # per layer without touching any real file. + class ColdArm + MARK = %(__m=->(n){File.write(ENV.fetch("READY_MARKS"),"#{ENV.fetch('READY_RUN_ID')} \#{n} #{'#{Process.clock_gettime(Process::CLOCK_REALTIME)}'}\\n",mode:"a")}) + + def self.instrument_stub(src, name:) + helper = %(at_exit{File.write(ENV.fetch("READY_MARKS"),"\#{ENV.fetch('READY_RUN_ID')} ruby_exit \#{Process.clock_gettime(Process::CLOCK_REALTIME)}\\n",mode:"a")}\n) + + %(def __bmark(n);File.write(ENV.fetch("READY_MARKS"),"\#{ENV.fetch('READY_RUN_ID')} \#{n} \#{Process.clock_gettime(Process::CLOCK_REALTIME)}\\n",mode:"a");end\n) + out = src.sub(/\A(#!.*\n)?/) { "#{Regexp.last_match(1)}#{helper}" } + out = out.sub(/(Gem\.use_gemdeps\s*\n)/) { "#{Regexp.last_match(1)}__bmark('rubygems_ready')\n" } + out.sub(/^(\s*)(load Gem\.activate_bin_path|Gem\.activate_and_load_bin_path)/) do + "#{Regexp.last_match(1)}__bmark('dep_activated')\n#{Regexp.last_match(1)}#{Regexp.last_match(2)}" + end + end + + def initialize(exe:, workdir:, marks_path:) + @exe = exe + @workdir = Pathname(workdir) + @marks_path = marks_path + end + + def real_shim = Pathname(File.expand_path("~/.rbenv/shims/#{@exe}")) + def real_stub = Pathname(`rbenv which #{@exe}`.strip) + + def instrument! + FileUtils.mkdir_p(@workdir) + write_shim(@workdir / @exe, rbenv: true) + write_shim(@workdir / "#{@exe}_direct", rbenv: false) + (@workdir / "stub").write(self.class.instrument_stub(real_stub.read, name: @exe)) + end + + # env a shell must export before running #command + def env(run_id:) + { + "READY_MARKS" => @marks_path.to_s, + "READY_RUN_ID" => run_id, + "RUBYOPT" => "-r#{Ready.root / 'bench' / 'prelude.rb'}", + "PATH" => "#{@workdir}:#{ENV['PATH']}", + } + end + + private + + def write_shim(path, rbenv:) + target = rbenv ? "exec rbenv exec \"#{@exe}\" \"$@\"" : "exec ruby \"#{@workdir / 'stub'}\" \"$@\"" + path.write(<<~SH) + #!/usr/bin/env bash + set -e + printf '%s shim_start %s\\n' "$READY_RUN_ID" "$EPOCHREALTIME" >> "$READY_MARKS" + export RBENV_ROOT="$HOME/.rbenv" + #{target} + SH + path.chmod(0o755) + end + end + end +end +``` + +Note: the direct variant execs the instrumented `stub` copy under the version ruby (no rbenv) so it works where rbenv is absent (CI reduced cold arm); the rbenv variant runs the real `rbenv exec ` chain, and the RUBYOPT prelude marks `ruby_up` regardless. `PATH` puts `@workdir/` first so the instrumented shim wins. + +- [ ] **Step 4: Run → pass; lint; commit** + +```bash +bundle exec rspec spec/bench/cold_arm_spec.rb +bundle exec rubocop spec/support/bench/cold_arm.rb +git add spec/support/bench/cold_arm.rb spec/bench/cold_arm_spec.rb +git commit -m "bench: ColdArm — mark-instrumented rbenv-shim + rubygems-stub copies" +``` + +--- + +## Task 4: Hot arm — render-based instrumented stub over a live sandbox + +**Files:** +- Create: `spec/support/bench/hot_arm.rb` → `Ready::Bench::HotArm` +- Test: `spec/bench/hot_arm_spec.rb` (stub-generation, fast), `:e2e` run in Task 6. + +**Interfaces:** +- Consumes: `Ready::Sandbox`, `Ready::Executable`, `Ready::PtyShell`, `Ready::Bench::ColdArm::…` (none), `bench/prof.zsh`. +- Produces: `Ready::Bench::HotArm.new(exe:, exe_path:, sandbox:, marks_path:)`; `.instrument_source(rendered, name:) -> String` (pure: injects `server_entry` at top + `pre_tool` before the tool's `IRB.start`-style call); `#stub_function(run_id:) -> String` a `ready_` zsh function whose inlined source carries the marks; `#dispatch(shell:, run_id:)` runs it in a `Ready::PtyShell` and appends to the markfile. +- The sandbox is built with `gems: []` (so the server preloads it) — see Task 6 for wiring. `exe_path` is the on-disk gem exe (contains `exe`, so `Ready::Executable.new(exe_path).render` inlines without a resolver change). + +- [ ] **Step 1: Source-instrumentation test** — `spec/bench/hot_arm_spec.rb`: + +```ruby +RSpec.describe Ready::Bench::HotArm do + it "prepends a server_entry mark and injects pre_tool before the tool start" do + rendered = <<~RUBY + Process.setproctitle "irb" + require 'irb' + IRB.start(__FILE__) + RUBY + out = described_class.instrument_source(rendered, name: "irb") + aggregate_failures do + expect(out).to match(/\At=Process\.clock_gettime.*server_entry/m) + expect(out).to match(/pre_tool.*\n\s*IRB\.start/m) + end + end +end +``` + +- [ ] **Step 2: Run → fail.** `bundle exec rspec spec/bench/hot_arm_spec.rb`. + +- [ ] **Step 3: Implement `HotArm`** — `spec/support/bench/hot_arm.rb`: + +```ruby +module Ready + module Bench + ## + # Runs a gem executable through the warm by-server and marks the dispatch + # infra vs the (preloaded) tool_run, using the REAL Ready::Executable render + # output with marks injected — no gem-runtime edit. + class HotArm + def self.mark(name) + %(File.open(ENV.fetch("READY_MARKS"),"a"){|f| f.puts "\#{ENV.fetch('READY_RUN_ID')} #{name} \#{Process.clock_gettime(Process::CLOCK_REALTIME)}"}) + end + + def self.instrument_source(rendered, name:) + out = "t=#{mark('server_entry')}\n#{rendered}" + out.sub(/^(\s*)([A-Z][\w:]*\.(?:start|run)\b|main\b)/) do + "#{Regexp.last_match(1)}#{mark('pre_tool')}\n#{Regexp.last_match(1)}#{Regexp.last_match(2)}" + end + end + + def initialize(exe:, exe_path:, sandbox:, marks_path:) + @exe = exe + @exe_path = exe_path + @sandbox = sandbox + @marks_path = marks_path + end + + # A faithful ready_ zsh function (mirrors fn.zsh.erb) whose inlined + # source carries the marks. Escaped for `ready_by -e`. + def stub_function + source = self.class.instrument_source(Ready::Executable.new(@exe_path.to_s).render, name: @exe) + <<~ZSH + ready_#{@exe}() { + emulate -L zsh + autoload -Uz ready_by + BY_SOCKET=#{@sandbox.sock_path} ready_by -e #{Shellwords.escape(source)} "$@" + } + ZSH + end + + # Prepend to the pty shell env so marks land in our file. + def shell_setup(run_id:) + "export READY_MARKS=#{@marks_path} READY_RUN_ID=#{run_id}" + end + end + end +end +``` + +- [ ] **Step 4: Run → pass; lint; commit** + +```bash +bundle exec rspec spec/bench/hot_arm_spec.rb +bundle exec rubocop spec/support/bench/hot_arm.rb +git add spec/support/bench/hot_arm.rb spec/bench/hot_arm_spec.rb +git commit -m "bench: HotArm — render-based instrumented stub over the warm server" +``` + +--- + +## Task 5: Runner + `rake bench` + +**Files:** +- Create: `spec/support/bench/runner.rb` → `Ready::Bench::Runner` +- Modify: `Rakefile` (add `bench` task) +- Test: `:e2e` in Task 6 (the Runner drives real processes). + +**Interfaces:** +- Consumes: `Ready::{Sandbox,PtyShell}`, `Ready::Bench::{ColdArm,HotArm,Marks,Aggregator,Report}`. +- Produces: `Ready::Bench::Runner.new(exe: "irb", lib: "irb", runs: 15, warmups: 3)`; `#call -> Ready::Bench::Report` after building the sandbox, interleaving arms, aggregating; `#render` prints it; `#teardown`. + +- [ ] **Step 1: Implement `Runner`** — `spec/support/bench/runner.rb`. It: (a) builds `Ready::Sandbox` with a readyfile carrying `gems: [lib]` + `executables: [rake]` (rake gives a compilable name so `ready up` builds the server; the target is dispatched via HotArm's own stub); (b) resolves the target's on-disk gem exe via `Gem::Specification.find_by_name(lib).bin_file(exe)` inside `Bundler.with_unbundled_env`; (c) instruments ColdArm; (d) for each run id, runs cold (direct + rbenv when available) and hot in interleaved order, collecting per-run spans via `Marks`; (e) aggregates with `Aggregator` (min for `shell`/`rbenv_shim`/`dispatch_infra`/`full`, median else); (f) returns a `Report`. + +```ruby +require "bundler" + +module Ready + module Bench + ## + # Orchestrates the interleaved cold/hot benchmark and builds a Report. + class Runner + PROCESS_SPANS = %w[shell rbenv_shim dispatch_infra full].freeze + + def initialize(exe: "irb", lib: "irb", runs: 15, warmups: 3, rbenv: nil) + @exe = exe + @lib = lib + @runs = runs + @warmups = warmups + @rbenv = rbenv.nil? ? system("command -v rbenv >/dev/null 2>&1") : rbenv + end + + def call + build + (1..(@runs + @warmups)).each { |i| one_round(i) } + cold = @cold_runs.drop(@warmups) + hot = @hot_runs.drop(@warmups) + agg = Aggregator.new(span_kind: PROCESS_SPANS.to_h { |s| [s, :min] }) + Report.new({ "cold" => [agg.combine(cold)], "hot" => [agg.combine(hot)] }, + { "cold" => cold.map { |s| s["full"] / 1000.0 }, "hot" => hot.map { |s| s["full"] / 1000.0 } }) + ensure + teardown + end + + def teardown + @sandbox&.teardown + FileUtils.rm_f(@marks) if @marks + end + + private + + def build + @marks = Pathname(Dir.mktmpdir("bench")) / "marks" + @marks.write("") + @sandbox = Ready::Sandbox.build(executables: ["rake"], gems: [@lib]) + @exe_path = Bundler.with_unbundled_env { Gem::Specification.find_by_name(@lib).bin_file(@exe) } + @cold = ColdArm.new(exe: @exe, workdir: @marks.dirname / "cold", marks_path: @marks) + @cold.instrument! + @hot = HotArm.new(exe: @exe, exe_path: Pathname(@exe_path), sandbox: @sandbox, marks_path: @marks) + @cold_runs = [] + @hot_runs = [] + end + + def one_round(i) + order = i.even? ? %i[cold hot] : %i[hot cold] + order.each { |arm| run_arm(arm, "#{arm}.#{i}") } + end + + def run_arm(arm, run_id) + before = @marks.read.lines.size + arm == :cold ? run_cold(run_id) : run_hot(run_id) + marks = Marks.parse(@marks.to_s)[run_id] || {} + (arm == :cold ? @cold_runs : @hot_runs) << Marks.spans(marks) + end + + def run_cold(run_id) + shell = Ready::PtyShell.new(@cold.env(run_id:)) + shell.run("source #{Ready.root / 'bench' / 'prof.zsh'}") + cmd = @rbenv ? @exe : "#{@exe}_direct" + shell.run("bench_envelope #{run_id} -- #{cmd} --version >/dev/null 2>&1") + ensure + shell&.close + end + + def run_hot(run_id) + shell = Ready::PtyShell.new(@sandbox.shell_env) + shell.run("source #{@sandbox.plugin_path}") + shell.run("source #{Ready.root / 'bench' / 'prof.zsh'}") + shell.run(@hot.shell_setup(run_id:)) + shell.run(@hot.stub_function.gsub("\n", "; ")) + shell.run("bench_envelope #{run_id} -- ready_#{@exe} --version >/dev/null 2>&1") + ensure + shell&.close + end + end + end +end +``` + +- [ ] **Step 2: Add `gems:` support to `Ready::Sandbox`** — `spec/support/sandbox.rb`: `self.build(executables:, gems: [])`, `initialize(executables:, gems: [])`, and `write_readyfile` emits a `gems:` block when non-empty. (Needed so the server preloads the target lib.) + +```ruby + def self.build(executables:, gems: []) + new(executables:, gems:).tap(&:up) + end + + def initialize(executables:, gems: []) + @executables = executables + @gems = gems + # ... unchanged temp-dir setup ... + end + + def write_readyfile + lines = [] + lines += ["gems:", *@gems.map { |g| " - #{g}" }] unless @gems.empty? + lines += ["executables:", *@executables.map { |e| " - #{e}" }] + readyfile.write("#{lines.join("\n")}\n") + end +``` + +- [ ] **Step 3: Add the `bench` rake task** — `Rakefile`: + +```ruby +desc "Print the cold-vs-hot startup waterfall (needs zsh + rbenv + by-server)" +task :bench do + require_relative "spec/support/pty_shell" + require_relative "spec/support/sandbox" + Dir[File.expand_path("spec/support/bench/*.rb", __dir__)].sort.each { |f| require f } + require "ready" + Ready::Bench::Runner.new(exe: ENV.fetch("BENCH_EXE", "irb"), lib: ENV.fetch("BENCH_LIB", "irb")).call.render +end +``` + +- [ ] **Step 4: Run it and observe the waterfall** — `bundle exec rake bench 2>&1 | tail -20`. Expected: a printed waterfall with cold ≫ hot on `rbenv_shim`/`rubygems`/`dep_activate`, hot `dispatch_infra` populated. Iterate on mark-parsing until every canonical span is present for at least the median run. Confirm teardown (no `by-server`, no temp dirs). + +- [ ] **Step 5: Lint; commit** + +```bash +bundle exec rubocop spec/support/bench/runner.rb spec/support/sandbox.rb +git add spec/support/bench/runner.rb spec/support/sandbox.rb Rakefile +git commit -m "bench: Runner + rake bench (interleaved cold/hot waterfall)" +``` + +--- + +## Task 6: `:e2e` structural guard + +**Files:** +- Create: `spec/e2e/benchmark_spec.rb` + +**Interfaces:** +- Consumes: `Ready::Bench::Runner`. + +- [ ] **Step 1: Write the guard** — `spec/e2e/benchmark_spec.rb` (reduced cold arm via `rbenv: false` so it needs no rbenv; structural asserts, no tight timing): + +```ruby +RSpec.describe "ready startup benchmark", :e2e do + it "shows the hot arm eliminating the rubygems + dep_activate layers and beating cold" do + report = Ready::Bench::Runner.new(exe: "irb", lib: "irb", runs: 5, warmups: 2, rbenv: false).call + cold = report.instance_variable_get(:@runs)["cold"].first + hot = report.instance_variable_get(:@runs)["hot"].first + aggregate_failures do + expect(cold["rubygems"]).to be > 5.0 # cold pays rubygems boot + expect(cold["dep_activate"]).to be > 5.0 # ...and gem activation + expect(hot["full"]).to be < cold["full"] # hot is faster, wide margin + expect(hot).not_to have_key("rbenv_shim") # reduced cold arm / hot has none + end + end +end +``` + +- [ ] **Step 2: Run → pass** — `READY_E2E=1 bundle exec rspec spec/e2e/benchmark_spec.rb 2>&1 | tail`. Expected: 1 example passes; no orphaned `by-server` after. + +- [ ] **Step 3: Full e2e + orphan check** — `bundle exec rake spec:e2e` (now 5 examples incl. benchmark); `pgrep -af by-server | grep -v grep || echo clean`. + +- [ ] **Step 4: Lint; commit; push; update PR #2** + +```bash +bundle exec rubocop spec/e2e/benchmark_spec.rb +git add spec/e2e/benchmark_spec.rb +git commit -m "test: e2e — benchmark structural guard (hot eliminates rubygems/dep layers)" +git push +``` + +--- + +## Verification (end-to-end) +1. Fast loop: `bundle exec rake spec` → Phase B + new bench unit specs pass; no e2e; rubocop clean on `spec/support/bench/*`. +2. `bundle exec rake bench` → prints a waterfall; cold ≫ hot on the boot layers; teardown leaves zero `by-server`/temp dirs. +3. `bundle exec rake spec:e2e` → dispatch + benchmark guards pass. +4. Zero production-runtime files changed: `git diff --stat ..HEAD -- lib/ zsh/` is empty except none. + +## Self-review notes +- **Spec coverage:** layer span model → T1; fixtures → T2; cold instrumentation → T3; hot render-based marks → T4; interleave/aggregate/report + `rake bench` → T5; `:e2e` guard + CI (reduced cold arm) → T6. Preload cliff → sandbox `gems:` in T5. Zero-runtime-edit constraint → all copies, verified in Verification step 4. +- **Types:** `Marks::SPANS`/`.spans`, `Aggregator#combine`, `ColdArm.instrument_stub`/`#env`, `HotArm.instrument_source`/`#stub_function`/`#shell_setup`, `Runner#call -> Report`, `Sandbox.build(executables:, gems:)` are used consistently across tasks. +- **Empirical note:** the process-spawning tasks (T3–T6) need on-machine iteration to get every mark parsed on the noisy container (mark-regex robustness, run counts); the `rake bench` step (T5.4) is the iteration point. This is expected for a measurement harness and is not a placeholder. diff --git a/docs/superpowers/specs/2026-07-12-phase-c-layer-benchmark-design.md b/docs/superpowers/specs/2026-07-12-phase-c-layer-benchmark-design.md new file mode 100644 index 0000000..7f91b08 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-phase-c-layer-benchmark-design.md @@ -0,0 +1,82 @@ +# Phase C — Layered Cold-vs-Hot Startup Benchmark (design) + +**Date:** 2026-07-12 · Branch: `e2e-rspec` (extends the Phase A/B harness) + +## Context & goal + +`ready` eliminates the fixed boot tax a Ruby CLI pays on every invocation. This benchmark **attributes where that time goes, per layer**, and shows how much each layer collapses when the command is served hot by the persistent `by-server`. It is the running artifact behind the RubyConf talk's `ri TCPServer` breakdown (`notes.md`), generalized: the launch layers are **identical for every rubygems CLI**, so the benchmark measures them with a representative executable and is **not coupled to any specific tool**. + +Grounded in a measured investigation (6 agents, `irb` on this container, `CLOCK_REALTIME` on both shell and Ruby sides). Absolute ms are environment-specific (this box runs ~4× the quiet floor); **the layer structure and the collapse ratios are the transferable finding.** + +## The layer model + +An invocation of a rubygems-installed CLI (`irb`, `ri`, `rubocop`, `rake`, …) pays these layers. `ready`'s value = the middle rows collapse to ~0 because the server paid them once. + +| span | boundary (from → to) | arm | cold ms (irb, floor) | hot ms | note | +|---|---|---|--:|--:|---| +| `shell` | `envelope_start` → `shim_start`/`stub_enter` | both | ~1.1 | 0.018 | hot: in-process alias→function, no PATH walk / exec | +| `rbenv_shim` | `shim_start` → `ruby_up` (minus ruby_boot) | cold | ~47 (→150+ long PATH) | ~0 | ~11 rbenv-* bash spawns; skipped hot | +| `ruby_boot` | interpreter init (via `--disable-gems` baseline) | cold | ~15 | ~0 | server pre-booted | +| `rubygems` | `ruby_up` → `rubygems_ready` (+ pre-`ruby_up` autoload) | cold | ~50 | ~0 | client runs `--disable-gems`; server warm | +| `dep_activate` | `dep_activated` → `ruby_exit` (combined) or → `tool_entry` (deep) | cold | ~32–43 | ~0 | `Gem.activate_bin_path`; skipped hot (source is `eval`'d) | +| `tool_run` | `tool_entry`/`dep_activated` → `ruby_exit` | both | ~103 (`require` ~85 + `IRB.start` ~18) | 0.23 + ~18 | the `require` collapses (warm CoW) **iff preloaded** | +| `reap` | `ruby_exit` → `envelope_end` | both | ~0.29 | ~0.29 | **retained** — a process still exits, shell still `wait()`s | +| `dispatch_infra` | `envelope_start` → `server_entry` | hot | — | ~53 | **new hot cost**: `by` client Ruby boot (~44) + socket + fork + worker | +| **full envelope** | `envelope_start` → `envelope_end` | both | ~240–520 | ~72 | **≈7×** | + +**Headline findings (talk beats):** +- ~183ms of the ~192ms in-process cold boot is tax the server pays once (`rbenv_shim`+`ruby_boot`+`rubygems`+`dep_activate`+`require`). +- The **hot residual is dominated (~44 of 72ms) by the `by` client being a full Ruby boot**, spawned only to hand FDs over the socket — a C/socket client shim could cut hot dispatch from ~53ms toward ~10ms. (Future `ready` optimization; call it out, don't build it here.) +- Validates the talk's numbers: rubygems layer ≈ 82ms = autoload 50 + `activate_bin_path` 32; rbenv ≈ 47ms floor, PATH-length-sensitive up to ~150ms. + +### Canonical marks (default mode) +`envelope_start` · `shim_start`(cold)/`stub_enter`(hot) · `ruby_up`(cold) · `rubygems_ready`(cold) · `dep_activated`(cold) · `server_entry`(hot) · `ruby_exit`(both, **`at_exit`**) · `envelope_end`. +Deep mode (opt-in): `bin_path_resolved`, `tool_entry` (TracePoint on `Gem.bin_path(name,name)`), `pre_tool`(hot) to split activation vs tool code. + +## Measurement technique — **zero production-runtime changes** + +Both arms are instrumented with **faithful copies**, never real system/gem files. Fidelity validated once by comparing an *uninstrumented* copy's wall time to the real thing (192.58 vs 192.36ms). + +- **Cold arm** (`Ready::Bench::ColdArm`): generate, in a temp dir, (a) a byte-identical copy of `~/.rbenv/shims/` (named `` so `program=`) that emits `shim_start` (line 1) and injects `RUBYOPT="-r $RUBYOPT"` before `exec rbenv exec`; (b) a copy of `$(rbenv which )` (the rubygems stub) that marks `rubygems_ready` after `require 'rubygems'; Gem.use_gemdeps`, `dep_activated` before `Gem.activate_and_load_bin_path`, and registers an `at_exit` `ruby_exit` **first** (CLIs call `exit`; control never returns). Also generate an `_direct` shim variant that skips rbenv (`exec $V/bin/`) so the `rbenv_shim` span is derivable by subtraction **and** so CI can run a reduced cold arm without rbenv. +- **RUBYOPT prelude** (`bench/prelude.rb`, committed): one statement appending `ruby_up ` to `$READY_MARKS`. As `-r` it is the first user code (after interp+rubygems autoload) — so it cannot bracket the autoload from inside; that ~50ms is captured via the external `--disable-gems` baseline delta. +- **Hot arm** (`Ready::Bench::HotArm`): stand up a real `Ready::Sandbox` whose readyfile **preloads the target's library** (`gems: [irb]`) so forks inherit it warm; obtain the *real* inlined stub source via `Ready::Executable.new().render` — passing the on-disk exe path (which contains `exe`) hits `Executable`'s direct-path branch, so a default-but-unbundled gem like `irb` inlines faithfully **without any resolver change** (bare `Executable.new("irb")` would fail under bundler via `Gem.bin_path`). Inject `server_entry` (first eval'd statement) + optional `pre_tool` marks into that copy, wrapped in a faithful `ready_` function; drive it and read the markfile. Uses a **flushed markfile** (clock read first, then `File.open('a')`), never stderr (CLI `--version` exit drops buffered stderr). +- **Envelope** (`bench/prof.zsh`, committed): `zmodload zsh/datetime`; `envelope_start` is a **bare** `$EPOCHREALTIME` read with its log line deferred until after the command returns (mark-write is ~48µs and would swamp sub-ms spans); a held-open-fd helper writes the rest; `envelope_end` read first-thing on return. +- **Clock**: `$EPOCHREALTIME` (zsh) == `Process.clock_gettime(CLOCK_REALTIME)` (Ruby), verified same-domain. Pin/quiesce NTP during a run. + +## Deliverables + +1. **`rake bench`** — builds the sandbox once, runs interleaved hot/cold (alternating order to cancel drift), aggregates and prints the waterfall via the existing `Ready::Bench::Report`. Reports *this machine's* numbers. +2. **`spec/e2e/benchmark_spec.rb`** (`:e2e`) — a structural regression guard: asserts the hot arm eliminates `rbenv_shim`+`rubygems`+`dep_activate` (each ~0 ± small) and `hot_envelope ≪ cold_envelope`. Won't flake (≈7× with wide margin). Uses the **reduced cold arm** (no rbenv) so it runs in the existing e2e CI job. + +Reuses Phase A/B: `Ready::Sandbox`, `Ready::PtyShell`, `Ready::Executable`, `Ready::Bench::{Marks,Stats,Report}`. + +## File structure (all additive) + +- `bench/prof.zsh` (create) — zsh envelope wrapper + mark helper. +- `bench/prelude.rb` (create) — RUBYOPT `ruby_up` prelude. +- `spec/support/bench/cold_arm.rb` (create) — `Ready::Bench::ColdArm`: generate instrumented shim/stub copies, run, return marks; `direct:` variant. +- `spec/support/bench/hot_arm.rb` (create) — `Ready::Bench::HotArm`: sandbox + render-based instrumented stub, run, return marks. +- `spec/support/bench/runner.rb` (create) — `Ready::Bench::Runner`: warmups + interleaved runs + aggregation (min for process-creation spans, median for in-process), returns `{runs_by_arm, pty_walls}` for `Report`. +- `spec/support/bench/marks.rb` (modify) — update `SPANS` to the ready layer model (keep `parse`/`spans` generic); update `spec/bench/marks_spec.rb`. +- `spec/e2e/benchmark_spec.rb` (create) — the `:e2e` guard. +- `spec/bench/{cold_arm,hot_arm,runner}_spec.rb` (create) — unit-ish specs for the pure logic (mark parsing/aggregation) using fixture marks; the process-spawning paths are exercised by the `:e2e` benchmark. +- `Rakefile` (modify) — `rake bench` task. + +## Decisions (chosen) + +1. **Target = `irb`** via the `gems:`-preload path (best collapse story; the talk's spirit). `rake` is the bundle-native fallback if a machine can't preload irb. +2. **CI** = the `:e2e` guard runs the **reduced cold arm** (`irb_direct`, no rbenv) so it works in the existing zsh-enabled e2e job; the full `rbenv_shim` span is a local-only addendum that `rake bench` exercises when `rbenv` is present. +3. **Numbers** = the benchmark reports the running machine's measurements each time; the spec/README quote floor numbers with an explicit "environment-specific; structure & ratios are the finding" caveat. + +## Risks & mitigations + +- **Flakiness** (overlayfs + shared VM → 5–6× swings): report **min/floor for process-creation spans**, **median for in-process spans**; ≥15 warm runs (ideally ≥41), discard warmups (first run inflates `tool_run` ~2×). +- **PATH length multiplier**: each rbenv script PATH-walks for `bash`; pin & record `PATH` in the report. +- **Stale `GEM_HOME`/`GEM_PATH`** (removed rvm): unset when its dir is missing (both arms), as `Ready::Sandbox` already does. +- **Preload cliff (correctness trap)**: `require` is warm only because the readyfile lists the lib under `gems:`. The benchmark **must assert the target is preloaded** or it silently measures a non-preloaded (full-cold-per-call) path. +- **`ruby_exit` must be `at_exit`** (registered first); `at_exit` undershoots true process death by ~2.7ms VM teardown that folds into `reap` — the report states which boundary it uses. +- **Isolation/cleanup**: sandbox + all copies in temp dirs; mandatory teardown (by-server stop + kill by argv + `rm -rf`); assert zero `by-server` and zero temp dirs remain. +- **CI portability**: cold arm needs zsh + Ruby; hot needs by-server + zsh (all in the e2e job). rbenv-dependent full cold arm is gated behind an `rbenv` availability probe and skipped otherwise. + +## Out of scope +A non-Ruby `by` client (the ~44ms residual lever) — noted as a finding/future optimization, not built here. diff --git a/exe/ready b/exe/ready index 9b08d18..33490d2 100755 --- a/exe/ready +++ b/exe/ready @@ -3,8 +3,8 @@ # Use Bundler when developing against a checkout (a Gemfile is present in the # gem root), but stay bundler-free in production so an installed `ready` gem # resolves its dependencies through RubyGems. Requiring bundler/setup directly -# (rather than invoking via `bundle exec`) means production installs — which -# ship no Gemfile (see the gemspec's file list) — run without Bundler at all. +# (rather than invoking via `bundle exec`) means production installs (which +# ship no Gemfile, per the gemspec's file list) run without Bundler at all. # In a dev checkout Bundler is active and, as expected, propagates # `-rbundler/setup` in RUBYOPT to child processes; the generated zsh stubs strip # that at runtime (see fn.zsh.erb). diff --git a/extra/ri.rb b/extra/ri.rb index bf0e33c..b335014 100755 --- a/extra/ri.rb +++ b/extra/ri.rb @@ -7,7 +7,7 @@ use_system: true, use_gems: true, use_stdout: true, - formatter: RDoc::Markup::ToMarkdown, + formatter: RDoc::Markup::ToAnsi } $ri_driver = RDoc::RI::Driver.new diff --git a/lib/ready/by_executable.rb b/lib/ready/by_executable.rb index 31ba070..53b0e98 100644 --- a/lib/ready/by_executable.rb +++ b/lib/ready/by_executable.rb @@ -44,8 +44,8 @@ def command_args def to_alias command_args - .then { |it| Shellwords.join(it) } - .then { |it| "#{it} \"${@}\"" } + .then { Shellwords.join(it) } + .then { "#{it} \"${@}\"" } end end end diff --git a/lib/ready/cli.rb b/lib/ready/cli.rb index f21777f..8984fa8 100644 --- a/lib/ready/cli.rb +++ b/lib/ready/cli.rb @@ -1,4 +1,5 @@ require "command_kit/commands" +require "command_kit/options/version" module Ready ## @@ -9,15 +10,15 @@ module Ready # class wires the sub-commands together with command_kit; each sub-command # lives in its own file under `cli/`. class CLI - include CommandKit::Commands + include CommandKit::Options::Version command_name "ready" + version Ready::VERSION command Init command Up command Compile command Clobber - end end diff --git a/lib/ready/cli/clobber.rb b/lib/ready/cli/clobber.rb index 92b4844..0b86bdd 100644 --- a/lib/ready/cli/clobber.rb +++ b/lib/ready/cli/clobber.rb @@ -6,7 +6,6 @@ class CLI # Removes every compiled ready build artifact, by delegating to the # `rake clobber` task. class Clobber < CommandKit::Command - include RakeCommand description "Remove all compiled ready build artifacts" @@ -17,7 +16,6 @@ class Clobber < CommandKit::Command def run rake("clobber") end - end end end diff --git a/lib/ready/cli/compile.rb b/lib/ready/cli/compile.rb index eee1029..65f68b9 100644 --- a/lib/ready/cli/compile.rb +++ b/lib/ready/cli/compile.rb @@ -6,17 +6,17 @@ class CLI # Compiles ready stubs and prints them to stdout, or compiles everything via # rake: # - # * `ready compile all` — compile every stub (`rake ready:compile`) - # * `ready compile by` — the persistent `by` client alias - # * `ready compile NAME ...` — a zsh function stub per CLI name + # * `ready compile` / `ready compile all`: compile every stub + # (`rake ready:compile`) -- bare `compile` defaults to `all`, like `make` + # * `ready compile by`: the persistent `by` client alias + # * `ready compile NAME ...`: a zsh function stub per CLI name # # The `--rubygems`/`--yjit` flags only apply to `by`; `--environment` only # applies to named CLIs. class Compile < CommandKit::Command - include RakeCommand - usage "[options] {all | by | NAME [NAME ...]}" + usage "[options] [all | by | NAME [NAME ...]]" option :environment, short: "-e", value: { @@ -38,14 +38,15 @@ class Compile < CommandKit::Command option :yjit, long: "--[no-]yjit", desc: "(by only) Enable YJIT. Off by default" - argument :names, required: true, + argument :names, required: false, repeats: true, usage: "all | by | NAME", - desc: "`all`, `by`, or one or more CLI names to compile" + desc: "`all` (the default when omitted), `by`, or one or more CLI names to compile" description "Compile ready stubs for the given CLI name(s)" examples [ + "", "all", "by --yjit", "irb rspec", @@ -58,7 +59,7 @@ class Compile < CommandKit::Command # from {#options}, which command_kit populates for us. # def initialize(**kwargs) - super(**kwargs) + super @environment = {} end @@ -72,8 +73,8 @@ def initialize(**kwargs) # def run(*names) case names - in ["all"] then rake("ready:compile") - in ["by"] then print by_alias + in [] | ["all"] then rake("ready:compile") + in ["by"] then print by_alias else reject_reserved_names!(names) print gem_script(names) @@ -105,7 +106,6 @@ def reject_reserved_names!(names) print_error "#{reserved.join(", ")} cannot be combined with other names" exit(1) end - end end end diff --git a/lib/ready/cli/init.rb b/lib/ready/cli/init.rb index 5aa2630..2e8e0e5 100644 --- a/lib/ready/cli/init.rb +++ b/lib/ready/cli/init.rb @@ -9,7 +9,6 @@ class CLI # # eval "$(ready init)" # or paste the line into ~/.zshrc class Init < CommandKit::Command - # The bundled zsh plugin entry point, resolved relative to the gem. PLUGIN_PATH = Ready.root / "zsh" / "ready" / "ready.plugin.zsh" @@ -26,7 +25,6 @@ def run puts "source #{PLUGIN_PATH}" end - end end end diff --git a/lib/ready/cli/rake_command.rb b/lib/ready/cli/rake_command.rb index 4c80c72..ee101cf 100644 --- a/lib/ready/cli/rake_command.rb +++ b/lib/ready/cli/rake_command.rb @@ -1,3 +1,4 @@ +require "English" require "rbconfig" module Ready @@ -13,7 +14,6 @@ class CLI # spawns via `RUBYOPT`); in production there is no Gemfile, so the build runs # bundler-free and resolves the installed gem through RubyGems. module RakeCommand - ## # Runs the given rake task(s) in {Ready.root} and exits non-zero if rake # fails. @@ -25,9 +25,8 @@ def rake(*tasks) # Propagate rake's own exit status where we can; fall back to 1 if the # process could not be spawned at all ($? unset). - exit($?&.exitstatus || 1) + exit($CHILD_STATUS&.exitstatus || 1) end - end end end diff --git a/lib/ready/cli/up.rb b/lib/ready/cli/up.rb index 842e181..c5fb7db 100644 --- a/lib/ready/cli/up.rb +++ b/lib/ready/cli/up.rb @@ -6,7 +6,6 @@ class CLI # Compiles every stub and (re)starts the ready server, by delegating to the # `rake ready` task. class Up < CommandKit::Command - include RakeCommand description "Compile all stubs and start the ready server" @@ -17,7 +16,6 @@ class Up < CommandKit::Command def run rake("ready") end - end end end diff --git a/lib/ready/configuration.rb b/lib/ready/configuration.rb index 38e38ae..25428a8 100644 --- a/lib/ready/configuration.rb +++ b/lib/ready/configuration.rb @@ -1,5 +1,3 @@ -require "pathname" - module Ready ## # Resolves ready's runtime configuration (paths, prefixes, the readyfile) @@ -21,7 +19,7 @@ def build_dir def sock_path fetched = fetch_env :sock_path do - File.expand_path "ready.sock", prefix + prefix / "ready.sock" end Pathname(fetched) @@ -35,7 +33,7 @@ def readyfile def open_readyfile fetched = fetch_env :readyfile do - File.expand_path ".readyfile", Dir.home + Pathname(Dir.home) / ".readyfile" end Readyfile.open(fetched, build_dir:) @@ -43,19 +41,20 @@ def open_readyfile def fetch_env(key, default: nil, &block) key = key.to_s.upcase - value = ENV.fetch("READY_#{key}") do - case [default, block] - in String, nil then default - in nil, Proc then yield - in nil, nil then raise "Expected ENV var #{key} to be found but was not" - else - raise ArgumentError "args #{key.inspect}, default: #{default.inspect}, block: #{block.inspect} are invalid" - end - end - + value = ENV.fetch("READY_#{key}") { resolve_fallback(key, default, block) } raise "Expected value of #{key} to not be empty" if value.empty? value end + + def resolve_fallback(key, default, block) + case [default, block] + in String, nil then default + in nil, Proc then block.call + in nil, nil then raise "Expected ENV var #{key} to be found but was not" + else + raise ArgumentError, "args #{key.inspect}, default: #{default.inspect}, block: #{block.inspect} are invalid" + end + end end end diff --git a/lib/ready/executable.rb b/lib/ready/executable.rb index 67ae478..bf55940 100644 --- a/lib/ready/executable.rb +++ b/lib/ready/executable.rb @@ -1,5 +1,4 @@ require "stringio" -require "pathname" module Ready ## @@ -8,18 +7,30 @@ module Ready class Executable attr_reader :name, :is_gem + # Executables whose gem name differs from the command, so `path` isn't a + # wall of one-off special cases. + GEM_BIN_OVERRIDES = { + "bundle" => %w[bundler bundle], + "ri" => %w[rdoc ri], + "yri" => %w[yard yri], + "rstore" => %w[reversal-store rstore], + "rougify" => %w[rouge rougify], + }.freeze + + SHEBANG_LINE = /^.*#!.*\n/ + REQUIRE_RELATIVE = /(require_relative(?:\(| )\s*[\x27"]([^\s\x27"]+)[\x27"]\)?)/ + def initialize(name) @name = name @is_gem = !File.exist?(name) end + # "bin/" when the executable sits directly in a bin directory, else + # the bare basename. Anchored on the parent directory's name so a path like + # /home/robin/foo doesn't match "bin" as a substring. def convert_path_to_bin(path) pathname = Pathname(path) - if /bin/.match?(pathname.dirname.to_s) - "bin/#{pathname.basename}" - else - pathname.basename - end + pathname.dirname.basename.to_s == "bin" ? "bin/#{pathname.basename}" : pathname.basename end def render @@ -35,60 +46,49 @@ def render end def path - unless @is_gem - if @name.to_s.include? "exe" - @path ||= @name - @name = @name.split("/").last - return @path - end - @path ||= convert_path_to_bin(@name) - return @path - end - - return Gem.bin_path("bundler", "bundle") if @name == "bundle" - return Gem.bin_path("rdoc", "ri") if @name == "ri" - return Gem.bin_path("yard", "yri") if @name == "yri" - return Gem.bin_path("reversal-store", "rstore") if @name == "rstore" - return Gem.bin_path("rouge", "rougify") if @name == "rougify" - return `rbenv which gem`.chomp if @name == "gem" - - @path ||= @is_gem ? gem_path : system_path - @path + @path ||= @is_gem ? gem_executable_path : on_disk_path end def source raise "No executable found for '#{@name}'" if path.nil? - clean = File.read(path) - &.gsub(/^.*#!.*\n/, "") - &.strip - &.then { StringIO.new it } - - @source ||= if clean.string.include?("require_relative") - string = clean.string - matches = string.scan(/(require_relative(?:\(| )\s*[\x27"]([^\s\x27"]+)[\x27"]\)?)/) - matches.each do |match, relpath| - absolute_path = File.expand_path(relpath, File.dirname(path)) - replacement = match.dup - replacement.gsub!("require_relative", "require") - replacement.gsub!(relpath, absolute_path) - string.gsub!(match, replacement) - end - string - else - clean.string - end + @source ||= rewrite_require_relative(File.read(path).gsub(SHEBANG_LINE, "").strip) end private - def gem_path - Gem.bin_path(@name, @name) + def on_disk_path + if @name.to_s.include?("exe") + resolved = @name + @name = @name.split("/").last + return resolved + end + convert_path_to_bin(@name) end - def system_path - rbenv_path = `rbenv which #{@name} 2>/dev/null`.strip - rbenv_path.empty? ? nil : rbenv_path + def gem_executable_path + override = GEM_BIN_OVERRIDES[@name] + return Gem.bin_path(*override) if override + return `rbenv which gem`.chomp if @name == "gem" + + gem_path + end + + # Rewrites `require_relative "x"` to an absolute `require`, so the source + # can be eval'd by the persistent server outside its original directory. + def rewrite_require_relative(code) + return code unless code.include?("require_relative") + + code.scan(REQUIRE_RELATIVE).each do |statement, relpath| + absolute = (Pathname(path).dirname / relpath).expand_path.to_s + rewritten = statement.gsub("require_relative", "require").gsub(relpath, absolute) + code = code.gsub(statement, rewritten) + end + code + end + + def gem_path + Gem.bin_path(@name, @name) end end end diff --git a/lib/ready/fn.zsh.erb b/lib/ready/fn.zsh.erb index 9ef4bce..f2dadfa 100644 --- a/lib/ready/fn.zsh.erb +++ b/lib/ready/fn.zsh.erb @@ -23,6 +23,6 @@ ready_<%= name %> () { __ready_debug "Sanitized RUBYOPT: ${(kv)parameters[RUBYOPT]}" fi - <%= env.map { |pair| pair.join("=") }.join(" ") %> \ - ready_by -e <%= ruby.shellescape %> "$@" + <%= env.map { |pair| pair.join("=") }.join(" ") %> ready_by \ + -e <%= ruby.shellescape %> "$@" } diff --git a/lib/ready/readyfile.rb b/lib/ready/readyfile.rb index fdf2e7b..599f52f 100644 --- a/lib/ready/readyfile.rb +++ b/lib/ready/readyfile.rb @@ -1,4 +1,3 @@ -require "pathname" require "yaml" module Ready @@ -12,8 +11,11 @@ def self.open(path, build_dir:) # A missing, empty, or null readyfile is not an error: it simply declares # no gems or executables. YAML.parse_file returns false for an empty file, # and a document such as "---" parses to nil. + # YAML.parse_file returns `false` for an empty file (not nil), so guard on + # truthiness, not with `&.` -- false&.to_ruby would blow up. A "---" + # document is truthy but to_ruby's to nil, which the `|| {}` folds to empty. document = (YAML.parse_file(path.to_s) if path.exist?) - config = (document.to_ruby if document) || {} + config = (document ? document.to_ruby : {}) || {} unless config.is_a?(Hash) raise Error, "#{path}: readyfile must be a YAML mapping of gems:/executables:, got #{config.class}" diff --git a/lib/ready/readyfile/executable.rb b/lib/ready/readyfile/executable.rb index f307a51..3d4f3b8 100644 --- a/lib/ready/readyfile/executable.rb +++ b/lib/ready/readyfile/executable.rb @@ -1,5 +1,3 @@ -require "pathname" - ## # A single executable declared in a readyfile, mapping its name to the # compiled path under the build directory. @@ -29,8 +27,16 @@ def realpath private def validate! + validate_name! + validate_build_dir! + end + + def validate_name! raise ArgumentError, "Name cannot be nil" if name.nil? raise ArgumentError, "Name cannot be empty" if name.empty? + end + + def validate_build_dir! raise ArgumentError, "build_dir cannot be nil for executable #{name.inspect}" if build_dir.nil? return if build_dir.directory? diff --git a/lib/ready/version.rb b/lib/ready/version.rb index a4b1dd0..1085776 100644 --- a/lib/ready/version.rb +++ b/lib/ready/version.rb @@ -1,3 +1,3 @@ module Ready - VERSION = "0.0.1".freeze + VERSION = "0.0.2".freeze end diff --git a/lib/ready/zsh_function.rb b/lib/ready/zsh_function.rb index 0774fa3..a71d7eb 100644 --- a/lib/ready/zsh_function.rb +++ b/lib/ready/zsh_function.rb @@ -1,5 +1,4 @@ require "erb" -require "pathname" module Ready ## @@ -24,13 +23,5 @@ def to_s env: environment, ) end - - private - - def render_environment_string - return "" if @environment.empty? - - @environment.map { |key, value| "#{key}=#{value}" }.then { |it| it.join(" ") } - end end end diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..4f62f10 --- /dev/null +++ b/notes.md @@ -0,0 +1,71 @@ +that's a lot of stuff to cover - i think we need to distill it. was thinking I'd start with a breakdown of where time is spent when you run something like "ri TCPServer" + +1. your shell layer (maybe get a quick show of hands of who uses bash vs zsh?) I think it will be heavily skewed zsh these days due to it being default on mac. This is where every cli tool starts, and "ri" either resolves to a function or to the PATH. If you are using rbenv to manage your rubies, like I and many of you are, your shell will find ~/.rbenv/shims/ri (quick, <5ms) + +2. the rbenv layer. The ri shim is a bash script that exec's itself with "rbenv exec ri", which is where it finds the ruby entrypoint to ri under your current ruby. Version managers vary, but getting this perfectly right across various OS's ruby versions, overlapping package managers, system ruby, etc without being able to use a language like ruby is non-trivial. This spawns dozens of subshells, walks directories, all via shell script. (slowest, 150ms) + +3. the rubygems stub +We are now in ruby, but still not at the "real" ri yet. Every executable installed via rubygems has one of these " +require 'rubygems' + +Gem.use_gemdeps + +version = ">= 0.a" + +str = ARGV.first +if str + str = str.b[/\A_(.*)_\z/, 1] + if str and Gem::Version.correct?(str) + version = str + ARGV.shift + end +end + +if Gem.respond_to?(:activate_and_load_bin_path) + Gem.activate_and_load_bin_path('rdoc', 'ri', version) +else + load Gem.activate_bin_path('rdoc', 'ri', version) +end" + +This loads rubygems, loads dependencies of ri, and then .... never actually runs the real kamal via its shebang ruby - at the bottom, it just loads it directly as a library file. Notice that we are executing the shebang ruby here, and its no longer usr/bin/env ruby -its been fully resolved to the realpath of it. + Up until the last line, very slow, around 80ms + +And then load is finally called on the activated bin path, and we can now say ready + + +And a process like this one happens for every standard issue ruby CLI tool you use. So what if, we could just always be ready? + +Could we run all but the last layer ahead of time, and keep it hot & ready? + +Turns out this kind of thing has a history in ruby, and some of you know exactly what I'm talking about, some of you don't and use it daily, and well, some of you don't and probably disabled it years ago after reading some stack overflow post on why your rails server wasn't starting. ( shows stack overflow screenshots with dozens of upvotes saying DISABLE_SPRING=1" + +I'm talking of course, about our buddy who peaked in high school, and one who I still personally call a friend, Mr. rails/spring + + +--- + +## "but what about bootsnap?" (prep for the reflexive objection) + +Someone will raise bootsnap. It's not a threat, it's a gift — the honest answer shows we understand the layers better than the objection does. Bootsnap and ready operate on DISJOINT layers and don't actually compete. + +What bootsnap does (two things): +1. load-path cache — memoizes `require 'x'` -> absolute path so you skip the $LOAD_PATH walk on every require. +2. compile cache — stores RubyVM::InstructionSequence bytecode so a require skips parse+compile. (also caches YAML/JSON compile, immaterial here.) + +The ordering is the whole game. Bootsnap is itself a gem, and it hooks rubygems' already-patched Kernel#require. So rubygems must be fully loaded (and bundler/setup run first, to populate $LOAD_PATH) before `bootsnap/setup` can layer on top. Its interception begins strictly AFTER rubygems + bundler are up. It cannot, even in principle, accelerate its own prerequisites. + +Map that onto our layers (numbers from a real run of `ronin`): +- shell (~5ms) + rbenv (~40ms): not in ruby yet -> bootsnap can't touch it. +- `require "rubygems"` (~45ms): loads BEFORE bootsnap exists -> can't touch it. +- `Gem.activate_bin_path` — spec resolution + dependency activation (~343ms, the biggest layer): can't touch it. +- tool + its deps' code getting required (~82ms): partially yes, this is bootsnap's home. + +So bootsnap is blind to the two biggest layers. And the biggest one for a precise reason worth saying out loud: the ISeq cache saves parse+compile, NOT execution. Activation is rubygems EXECUTING — scanning specifications/, building spec stubs, resolving the graph. There's no bytecode to cache; it's work, not compilation. Bootsnap only shaves the tail, and only the compile slice of the tail. + +And bootsnap is project-centered; ready is global. This isn't just cultural — there's no mechanism. Bootsnap's cache is anchored to a project (tmp/cache/bootsnap) and armed in that app's boot.rb. A globally-installed colorls / ronin / ri has no boot.rb you own. Your only lever is jamming RUBYOPT=-rbootsnap/setup into every ruby invocation — and even then you (a) still pay rubygems + activation in full, (b) pay bootsnap's setup on every call, (c) have a global cache with no project to scope it. You basically can't apply it here, usefully. + +Synthesis line for the talk: bootsnap shrinks the TAIL (compiling/resolving lots of code) inside a process that's already past rubygems and activation — it shines when a big app boots the same huge tree repeatedly in dev. ready eliminates the HEAD (shell, rbenv, rubygems load, dependency activation) by keeping a process hot past all of it. They're complementary, not competing — a ready server could even use bootsnap internally to speed its own one-time warmup. For the per-invocation tax on a global CLI, bootsnap structurally cannot reach where the time is. ready is aimed exactly there. + + + + diff --git a/rakelib/ready.rake b/rakelib/ready.rake index 6dde6f9..ee2b363 100644 --- a/rakelib/ready.rake +++ b/rakelib/ready.rake @@ -14,7 +14,7 @@ READY_SOCKET = configuration.sock_path READY_BUILD_DIR = configuration.build_dir READYFILE = configuration.readyfile -def run_command(*args, out: $stdout, **kwargs) +def run_command(*args, out: $stdout, chdir: Dir.pwd, **kwargs) case args in Hash => env, *cmd in cmd then env = {} @@ -24,7 +24,7 @@ def run_command(*args, out: $stdout, **kwargs) puts cmd.join(" ") Bundler.with_unbundled_env do - Open3.popen2(env, *cmd, **kwargs) do |sin, sout, wait| + Open3.popen2(env, *cmd.map(&:to_s), chdir: chdir.to_s, **kwargs) do |sin, sout, wait| sin.close IO.copy_stream sout, out result = wait.value @@ -87,7 +87,7 @@ namespace :ready do end file READYFILE do - touch READYFILE + touch READYFILE.to_s end ready_path = EXE_DIR / "ready" @@ -98,8 +98,8 @@ namespace :ready do extra = READY_PREFIX / "extra.rb" - CLEAN.include build_tempdir - CLOBBER.include READY_BUILD_DIR.parent / "**/*" + CLEAN.include FileList[build_tempdir, READY_BUILD_DIR] + CLOBBER.include FileList[READY_BUILD_DIR.parent / "**/*"] env = { "JRUBY_OPTS" => "--dev -J--enable-native-access=ALL-UNNAMED", diff --git a/ready.gemspec b/ready.gemspec index 5d7e4f1..c0d2bdc 100644 --- a/ready.gemspec +++ b/ready.gemspec @@ -13,7 +13,8 @@ Gem::Specification.new do |spec| files = IO.popen(["git", "ls-files", "-z"], chdir: __dir__, err: IO::NULL) { |ls| ls.readlines("\x0", chomp: true).reject do |f| (f == gemspec_file) || - f.start_with?("bin/", "test/", "spec/", "features/", ".git", "Gemfile") + f.start_with?("bin/", "test/", "spec/", "features/", "bench/", "docs/", ".git", "Gemfile", + "_claude/", "talk/", "notes.md", "setup.sh", "CLAUDE.md") end } files = Dir.glob("{lib,exe}/**/*").push("README.md", "LICENSE.txt", "Rakefile") if files.empty? diff --git a/readyfile b/readyfile new file mode 100644 index 0000000..12c3d48 --- /dev/null +++ b/readyfile @@ -0,0 +1,10 @@ +# gems: are preloaded into the warm by-server, which does `require ` +# verbatim -- so each entry must be a REQUIRE PATH, not a gem name. Both the +# ronin library (`require "ronin"`) and ronin-support (`require "ronin/support"`, +# NOT "ronin-support", which is a LoadError) are preloaded so the ronin CLI's +# own code is already warm in the server, not just its support library. +gems: + - ronin + - ronin/support +executables: + - ronin diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..0c900b7 --- /dev/null +++ b/setup.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +# +# jb-backend-bootstrap.sh +# ------------------------ +# Idempotently install a JetBrains "remote development" IDE backend on a Linux +# server, install a fixed set of host plugins, and hand you a connection link. +# Designed to run on a fresh, short-lived machine (cloud-init, a base image, or +# `ssh host 'bash -s' < jb-backend-bootstrap.sh`). +# +# CLIENT SIDE: JetBrains is migrating Remote Development into the Toolbox App; +# JetBrains Gateway is being deprecated (still works during the transition). +# This is purely a client-side change -- the SERVER side below is identical for +# Toolbox, Gateway, or a full IDE: same backend, same remote-dev-server.sh. +# The `run` link this script prints opens in whichever client you use. +# +# No JetBrains license is needed to *install/warm up* a backend. A license is +# only checked on your LOCAL machine when the thin client actually connects. +# +# Usage: +# ./jb-backend-bootstrap.sh # defaults below +# IDE_CODE=GO PLUGINS="org.jetbrains.plugins.go" ./jb-backend-bootstrap.sh +# MODE=run PROJECT_DIR=~/code/app ./jb-backend-bootstrap.sh +# +set -euo pipefail + +# ============================ CONFIG (override via env) ====================== +# Product code (releases API): IIU=IDEA Ultimate, IIC=IDEA Community, +# PCP=PyCharm Pro, PCC=PyCharm Community, GO=GoLand, WS=WebStorm, RD=Rider, +# CL=CLion, RM=RubyMine, PS=PhpStorm, RR=RustRover, DG=DataGrip. +IDE_CODE="${IDE_CODE:-IIU}" + +# "latest" or a pinned version string, e.g. "2026.1.4". Pinning is recommended +# so Gateway's thin client version stays reproducible across machines. +IDE_VERSION="${IDE_VERSION:-latest}" + +# Where the backend gets unpacked. Keyed by build so multiple builds coexist +# and re-runs are cheap no-ops. +INSTALL_ROOT="${INSTALL_ROOT:-$HOME/.jetbrains/backends}" + +# The project the backend will index / serve. +PROJECT_DIR="${PROJECT_DIR:-$HOME/project}" + +# Space-separated *host* plugin IDs (NOT display names). These run on the +# backend: language support, inspections, GitToolBox, etc. Find an ID on a +# plugin's Marketplace page (URL is .../plugin/-) or in its +# META-INF/plugin.xml tag. +# NOTE: client-only plugins (IdeaVim, themes, keymaps) do NOT belong here — +# install those in your local JetBrains Client / via Settings Sync instead. +PLUGINS="${PLUGINS:-}" # e.g. "zielu.gittoolbox Pythonid org.jetbrains.plugins.go" + +# run = start a headless backend now and print a connection link + a ready +# Toolbox deep link. Works with Toolbox, Gateway, or a full IDE. +# Recommended for automated spin-ups. +# register = LEGACY (Gateway only): mark this backend discoverable by Gateway. +# No effect for the Toolbox App; kept for existing Gateway users. +MODE="${MODE:-run}" + +# For MODE=run only: how the client should dial back in over SSH. +SSH_HOST="${SSH_HOST:-$(hostname -f 2>/dev/null || hostname)}" +SSH_USER="${SSH_USER:-$USER}" +SSH_PORT="${SSH_PORT:-22}" +# ============================================================================ + +log() { printf '\033[1;34m[jb]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1;31m[jb] ERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +have() { command -v "$1" >/dev/null 2>&1; } + +have curl || die "curl is required" +have tar || die "tar is required" +have jq || have python3 || die "need jq (preferred) or python3 to parse the releases API" + +# One scratch dir for the whole run; one EXIT trap. +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT + +# Correct Python fallback parser (used only if jq is absent). Reads JSON on +# stdin — no heredoc/pipe conflict, no arg-size limit. +PYPARSE="$WORK/parse.py" +cat > "$PYPARSE" <<'PY' +import sys, json +want, pkey = sys.argv[1], sys.argv[2] +data = json.load(sys.stdin) or {} +rels = next(iter(data.values()), []) +r = rels[0] if want == "latest" else next((x for x in rels if x.get("version") == want), None) +if not r: + sys.exit(f"no release matching '{want}'") +dl = r.get("downloads", {}).get(pkey) or sys.exit(f"no '{pkey}' download for build {r.get('build')}") +print("\t".join([r["build"], dl["link"], dl.get("checksumLink", "")])) +PY + +# --- Pick the right Linux build for this CPU architecture -------------------- +case "$(uname -m)" in + x86_64|amd64) PLATFORM_KEY="linux" ;; + aarch64|arm64) PLATFORM_KEY="linuxARM64" ;; + *) die "Unsupported architecture: $(uname -m)" ;; +esac + +# --- Resolve build number + download link from the releases API -------------- +# Emits: "\t\t" (curl pipes JSON to stdin either way) +resolve() { + local url="https://data.services.jetbrains.com/products/releases?code=${IDE_CODE}&type=release" + if have jq; then + curl -fsSL "$url" | jq -r --arg want "$IDE_VERSION" --arg k "$PLATFORM_KEY" ' + .[] as $rels + | (if $want == "latest" then $rels[0] + else ($rels | map(select(.version == $want)) | .[0]) end) as $r + | if $r == null then error("no release matching \($want)") + else [$r.build, $r.downloads[$k].link, ($r.downloads[$k].checksumLink // "")] | @tsv + end' + else + curl -fsSL "$url" | python3 "$PYPARSE" "$IDE_VERSION" "$PLATFORM_KEY" + fi +} + +log "Resolving ${IDE_CODE} ${IDE_VERSION} (${PLATFORM_KEY}) ..." +IFS=$'\t' read -r BUILD LINK SHALINK < <(resolve) \ + || die "could not resolve a release (check IDE_CODE / IDE_VERSION)" + +DEST="${INSTALL_ROOT}/${IDE_CODE}-${BUILD}" +SERVER_SH="${DEST}/bin/remote-dev-server.sh" + +# --- Download + extract (idempotent) ----------------------------------------- +if [[ -x "$SERVER_SH" ]]; then + log "Backend ${IDE_CODE} build ${BUILD} already present at ${DEST} — skipping download." +else + log "Downloading ${LINK}" + curl -fSL "$LINK" -o "$WORK/ide.tar.gz" + + if [[ -n "$SHALINK" ]]; then + log "Verifying SHA-256 ..." + expected="$(curl -fsSL "$SHALINK" | awk '{print $1}')" + echo "${expected} ${WORK}/ide.tar.gz" | sha256sum -c - >/dev/null \ + || die "checksum mismatch — aborting" + fi + + mkdir -p "$DEST" + # tarball has a single top-level dir (idea-IU-/...); flatten it. + tar -xzf "$WORK/ide.tar.gz" -C "$DEST" --strip-components=1 + log "Installed to ${DEST}" +fi + +# Authoritative launcher product code (IU, PY, GO, ...) for Toolbox's ideHint. +# This differs from the releases-API code (IIU->IU, PCP->PY, DG->DB, ...), so +# read it from the backend itself rather than guessing. +if have jq; then + PRODUCT_CODE="$(jq -r '.productCode' "${DEST}/product-info.json" 2>/dev/null || echo "$IDE_CODE")" +else + PRODUCT_CODE="$(python3 -c "import json;print(json.load(open('${DEST}/product-info.json'))['productCode'])" 2>/dev/null || echo "$IDE_CODE")" +fi + +# --- Install host plugins ---------------------------------------------------- +# Real signature is: remote-dev-server.sh installPlugins +# (JetBrains' docs omit the project path, but the backend requires it.) +if [[ -n "${PLUGINS// /}" ]]; then + mkdir -p "$PROJECT_DIR" + log "Installing host plugins: ${PLUGINS}" + # shellcheck disable=SC2086 + "$SERVER_SH" installPlugins "$PROJECT_DIR" $PLUGINS \ + || die "plugin install failed (check plugin IDs and network access to Marketplace)" +fi + +# --- Make it usable ---------------------------------------------------------- +case "$MODE" in + run) + mkdir -p "$PROJECT_DIR" + # Toolbox deep link: click it locally to SSH in and open the project. + # (Toolbox still handles the historical jetbrains://gateway/ssh/... scheme.) + TOOLBOX_LINK="jetbrains://gateway/ssh/environment?h=${SSH_HOST}&u=${SSH_USER}&p=${SSH_PORT}&launchIde=true&ideHint=${PRODUCT_CODE}-${BUILD}&projectHint=${PROJECT_DIR}" + log "Toolbox one-click connect (open on your LOCAL machine):" + log " ${TOOLBOX_LINK}" + log "" + log "Starting headless backend for ${PROJECT_DIR} ..." + log "The command below also prints a jetbrains:// join link — open it in" + log "the Toolbox App, Gateway, or any full JetBrains IDE." + export REMOTE_DEV_NON_INTERACTIVE=1 + exec "$SERVER_SH" run "$PROJECT_DIR" \ + --ssh-link-host "$SSH_HOST" \ + --ssh-link-user "$SSH_USER" \ + --ssh-link-port "$SSH_PORT" + ;; + register) + # Legacy path for users still on JetBrains Gateway. No effect for Toolbox. + log "[legacy] Registering backend with JetBrains Gateway ..." + "$SERVER_SH" registerBackendLocationForGateway + log "Done. In Gateway: SSH into this host and pick build ${BUILD} from the list." + log "If you use the Toolbox App instead, skip this — just SSH to the host in" + log "Toolbox, or use MODE=run to get a connection link." + ;; + *) + die "unknown MODE '$MODE' (use 'run' or 'register')" + ;; +esac diff --git a/spec/bench/cli_spec.rb b/spec/bench/cli_spec.rb new file mode 100644 index 0000000..fccfbf6 --- /dev/null +++ b/spec/bench/cli_spec.rb @@ -0,0 +1,92 @@ +require "tmpdir" + +RSpec.describe Ready::Bench::CLI do + subject(:cli) { described_class.new } + + describe ".command_segments" do + it "splits argv into one command per -- separator" do + segments = described_class.command_segments(%w[ri TCPServer -- ronin help -- kamal version]) + expect(segments).to eq([%w[ri TCPServer], %w[ronin help], %w[kamal version]]) + end + + it "keeps a separator-free argv as a single segment" do + expect(described_class.command_segments(%w[ri TCPServer])).to eq([%w[ri TCPServer]]) + end + end + + describe "#invocations_under_test" do + it "parses each typed segment into an invocation, verbatim" do + ri = Ready::Bench::Invocation.new(executable_name: "ri", arguments: ["TCPServer"]) + ronin = Ready::Bench::Invocation.new(executable_name: "ronin", arguments: ["help"]) + expect(cli.invocations_under_test([%w[ri TCPServer], %w[ronin help]])).to eq([ri, ronin]) + end + + it "falls back to one run on the rake task's default" do + expect(cli.invocations_under_test([])).to eq([nil]) + end + + context "with a readyfile and no typed commands" do + def with_readyfile + Dir.mktmpdir do |dir| + readyfile_path = Pathname(dir) / "readyfile" + readyfile_path.write("gems:\n - rdoc\nexecutables:\n - ri\n - rake\n") + yield readyfile_path + end + end + + it "benches every executable the readyfile declares, bare" do + bare = %w[ri rake].map { Ready::Bench::Invocation.bare(it) } + with_readyfile do |readyfile_path| + cli.option_parser.parse(["--readyfile", readyfile_path.to_s]) + expect(cli.invocations_under_test([])).to eq(bare) + end + end + end + end + + describe "#environment_for" do + it "exports nothing by default, deferring every default to the rake task" do + expect(cli.environment_for(nil)).to eq({}) + end + + it "exports a typed command verbatim -- what you typed is what runs" do + invocation = Ready::Bench::Invocation.parse(%w[ronin help]) + expect(cli.environment_for(invocation)) + .to eq("BENCH_EXE" => "ronin", "BENCH_ARGS" => "help") + end + + it "maps the flags onto the BENCH_* contract the rake task reads" do + cli.option_parser.parse(["--readyfile", "readyfile", "--rounds", "5", "--warmups", "1"]) + expect(cli.environment_for(nil)) + .to eq("BENCH_READYFILE" => "readyfile", "BENCH_RUNS" => "5", "BENCH_WARMUPS" => "1") + end + + it "routes results to a file only when plotting" do + cli.option_parser.parse(["--plot", "stacked"]) + expect(cli.environment_for(nil).keys).to eq(["BENCH_RESULTS"]) + end + end + + describe "#plotter_for" do + let(:comparisons) { [] } + + it "draws stacked bars with our renderer" do + expect(cli.plotter_for(:stacked, comparisons)).to be_a(Ready::Bench::Plot::Stacked) + end + + it "draws side-by-side pairs with stock youplot" do + expect(cli.plotter_for(:youplot, comparisons)).to be_a(Ready::Bench::Plot::Youplot) + end + end + + describe "#task_name" do + it "runs the plain bench task by default" do + expect(cli.task_name).to eq("bench") + end + + it "runs bench:verbose when asked for the legend" do + cli.option_parser.parse(["--verbose"]) + expect(cli.task_name).to eq("bench:verbose") + end + end +end diff --git a/spec/bench/comparison_spec.rb b/spec/bench/comparison_spec.rb new file mode 100644 index 0000000..0b723c2 --- /dev/null +++ b/spec/bench/comparison_spec.rb @@ -0,0 +1,13 @@ +RSpec.describe Ready::Bench::Comparison do + subject(:comparison) do + described_class.new(command: "ri TCPServer", cold_milliseconds: 500.0, hot_milliseconds: 40.0) + end + + it "reports how many times faster hot is" do + expect(comparison.speedup).to be_within(1e-6).of(12.5) + end + + it "reports what ready eliminates from every invocation" do + expect(comparison.eliminated_milliseconds).to be_within(1e-6).of(460.0) + end +end diff --git a/spec/bench/harness_log_spec.rb b/spec/bench/harness_log_spec.rb new file mode 100644 index 0000000..ac6390c --- /dev/null +++ b/spec/bench/harness_log_spec.rb @@ -0,0 +1,15 @@ +RSpec.describe Ready::Bench::HarnessLog do + subject(:log) { described_class.new(invocation: "ri TCPServer") } + + it "lives at the project's log/bench.log" do + expect(log.path).to eq(Ready.root / "log" / "bench.log") + end + + it "tees a command's stdout and stderr under a run-id header, silent on the pty", :aggregate_failures do + fragment = log.tee("hot.3", "bench_harness hot.3 -- ready_ri TCPServer") + expect(fragment).to include("### hot.3 ###") + expect(fragment).to include("bench_harness hot.3 -- ready_ri TCPServer") + expect(fragment).to include("2>&1 | tee -a #{log.path}") + expect(fragment).to end_with(">/dev/null") + end +end diff --git a/spec/bench/hot_arm_spec.rb b/spec/bench/hot_arm_spec.rb new file mode 100644 index 0000000..3200a86 --- /dev/null +++ b/spec/bench/hot_arm_spec.rb @@ -0,0 +1,31 @@ +RSpec.describe Ready::Bench::HotArm do + describe ".instrument_source" do + subject(:instrumented) { described_class.instrument_source(rendered_source) } + + let(:rendered_source) do + <<~RUBY + Process.setproctitle "irb" + require 'irb' + IRB.start(__FILE__) + RUBY + end + + it "marks server entry before any of the rendered source runs" do + expect(instrumented.index("server_entry")).to be < instrumented.index("setproctitle") + end + + it "marks pre_tool immediately before the tool's entry call" do + expect(instrumented).to include(%(ready_bench_mark("pre_tool")\nIRB.start)) + end + end + + describe "#stub_function" do + it "marks command_start as the zsh function's first act after emulate" do + sandbox = instance_double(Ready::Sandbox, sock_path: Pathname("/tmp/bench/ready.sock")) + marks_log = Ready::Bench::MarksLog.new("/tmp/bench/marks") + hot_arm = described_class.new(executable_name: "irb", rendered_source: "IRB.start(__FILE__)\n", + sandbox:, marks_log:) + expect(hot_arm.stub_function).to include(%( emulate -L zsh\n bench_mark command_start\n)) + end + end +end diff --git a/spec/bench/marks_log_spec.rb b/spec/bench/marks_log_spec.rb new file mode 100644 index 0000000..e6d4885 --- /dev/null +++ b/spec/bench/marks_log_spec.rb @@ -0,0 +1,42 @@ +require "tempfile" + +RSpec.describe Ready::Bench::MarksLog do + subject(:marks_log) { described_class.new(file.path) } + + let(:file) do + Tempfile.new("marks").tap do |tempfile| + tempfile.write(<<~LOG) + cold.1 harness_start 1000.000 + cold.1 ruby_up 1000.065 + cold.1 exit_status 0 + hot.1 harness_start 2000.000 + hot.1 exit_status 1 + LOG + tempfile.close + end + end + + it "reconstructs a run from its own lines only", :aggregate_failures do + run = marks_log.run("cold.1") + expect(run.recorded?(:harness_start)).to be true + expect(run.recorded?(:ruby_up)).to be true + expect(run.milliseconds_between(:harness_start, :ruby_up)).to be_within(1e-6).of(65.0) + end + + it "returns a run with no marks for an id that never logged any" do + expect(marks_log.run("cold.99").recorded?(:harness_start)).to be false + end + + it "reads exit_status as the run's status, not as a mark time", :aggregate_failures do + run = marks_log.run("cold.1") + expect(run.exit_status).to eq(0) + expect(run.recorded?(:exit_status)).to be false + expect(run.succeeded?).to be true + end + + it "carries a non-zero exit status through as a failure", :aggregate_failures do + run = marks_log.run("hot.1") + expect(run.exit_status).to eq(1) + expect(run.succeeded?).to be false + end +end diff --git a/spec/bench/plot/stacked_spec.rb b/spec/bench/plot/stacked_spec.rb new file mode 100644 index 0000000..65ff3b5 --- /dev/null +++ b/spec/bench/plot/stacked_spec.rb @@ -0,0 +1,53 @@ +RSpec.describe Ready::Bench::Plot::Stacked do + subject(:plot) { described_class.new(comparisons) } + + def comparison(command, cold, hot) + Ready::Bench::Comparison.new(command:, cold_milliseconds: cold, hot_milliseconds: hot) + end + + describe "the common case" do + let(:comparisons) { [comparison("ri", 500.0, 50.0), comparison("rake", 250.0, 125.0)] } + + # Global max is 500ms -> 40 cells, so 0.08 cells/ms: ri stacks 4 hot + 36 + # eliminated, rake 10 hot + 10 eliminated. + let(:ri_bar) { ("#" * 4) + ("." * 36) } + let(:rake_bar) { ("#" * 10) + ("." * 10) } + + it "stacks ready and eliminated into one bar per command, on a shared scale", :aggregate_failures do + expect { plot.render }.to output( + a_string_including(ri_bar).and(including("50.0 ready")).and(including("500.0 cold")) + .and(including("(10.0x)")).and(including(rake_bar)).and(including("125.0 ready")) + .and(including("(2.0x)")), + ).to_stdout + end + end + + describe "a regression (hot slower than cold)" do + let(:comparisons) { [comparison("slow", 100.0, 150.0)] } + + it "never overflows the bar width, and reports the sub-1x speedup", :aggregate_failures do + line = capture(plot).lines.last + bar = line[/[#.]+/] + expect(bar.length).to eq(described_class::WIDTH) + expect(line).to include("(0.7x)") + end + end + + describe "a command longer than the others" do + let(:comparisons) { [comparison("bundler-audit check", 300.0, 40.0), comparison("ri", 500.0, 50.0)] } + + it "aligns every bar to the widest command" do + bar_columns = capture(plot).lines.drop(1).map { it.index(/[#.]/) } + expect(bar_columns.uniq.length).to eq(1) + end + end + + def capture(plot) + original = $stdout + $stdout = StringIO.new + plot.render + $stdout.string + ensure + $stdout = original + end +end diff --git a/spec/bench/plot/youplot_spec.rb b/spec/bench/plot/youplot_spec.rb new file mode 100644 index 0000000..fcc566a --- /dev/null +++ b/spec/bench/plot/youplot_spec.rb @@ -0,0 +1,31 @@ +RSpec.describe Ready::Bench::Plot::Youplot do + subject(:plot) { described_class.new(comparisons) } + + def comparison(command, cold, hot) + Ready::Bench::Comparison.new(command:, cold_milliseconds: cold, hot_milliseconds: hot) + end + + describe "#title" do + it "names the plot after a lone command" do + one = described_class.new([comparison("ronin encode", 800.0, 40.0)]) + expect(one.send(:title)).to eq("full startup (ms): ronin encode") + end + + it "falls back to the generic title for several commands" do + two = described_class.new([comparison("ri", 500.0, 40.0), comparison("rake", 130.0, 60.0)]) + expect(two.send(:title)).to eq("full startup (ms): cold vs ready") + end + end + + describe "#label" do + it "uses bare arm labels for a lone command (the command is the title)" do + one = described_class.new([comparison("ronin encode", 800.0, 40.0)]) + expect(one.send(:label, comparison("ronin encode", 800.0, 40.0), "ready")).to eq("ready") + end + + it "prefixes the command when several commands share the plot" do + two = described_class.new([comparison("ri", 500.0, 40.0), comparison("rake", 130.0, 60.0)]) + expect(two.send(:label, comparison("ri", 500.0, 40.0), "cold")).to eq("ri cold") + end + end +end diff --git a/spec/bench/progress_spec.rb b/spec/bench/progress_spec.rb new file mode 100644 index 0000000..3c504f5 --- /dev/null +++ b/spec/bench/progress_spec.rb @@ -0,0 +1,27 @@ +require "stringio" + +RSpec.describe Ready::Bench::Progress do + subject(:progress) { described_class.new(command: "ri TCPServer", io:) } + + let(:io) { StringIO.new } + + before do + progress.building + progress.warming_up(3) + 3.times { progress.tick } + progress.measuring(4) + 4.times { progress.tick } + progress.done + end + + it "narrates the build, then warmup and measured phases separately", :aggregate_failures do + expect(io.string).to include("ri TCPServer: building sandbox") + expect(io.string).to include("ri TCPServer: warming up (3 rounds) ...") + expect(io.string).to include("ri TCPServer: measuring 4 rounds ....") + expect(io.string).to end_with(" done\n") + end + + it "keeps the measured count as the user asked, not measured + warmup" do + expect(io.string).not_to include("7 rounds") + end +end diff --git a/spec/bench/report_spec.rb b/spec/bench/report_spec.rb new file mode 100644 index 0000000..108f4a0 --- /dev/null +++ b/spec/bench/report_spec.rb @@ -0,0 +1,54 @@ +RSpec.describe Ready::Bench::Report do + subject(:report) do + described_class.new(cold: arm_result(:cold, full: 200.0), hot: arm_result(:hot, full: 40.0), protocol:) + end + + let(:protocol) do + Ready::Bench::Protocol.new(executable_name: "ri", arguments: ["TCPServer"], + preload_gems: ["rdoc"], rounds: 1, warmups: 0) + end + + def arm_result(name, full:) + Ready::Bench::ArmResult.new(name:, warmups: 0).tap do |result| + waterfall = Ready::Bench::Waterfall.new(rubygems: 5.0, full:) + result.record(Ready::Bench::Measurement.new(waterfall:, wall_clock_seconds: full / 1000.0)) + end + end + + it "leads with the caveat-less numbers and the speedup, rounded to a tenth" do + expect { report.render }.to output(a_string_including("ready is 5.0x faster, saving 160.0 ms per run")).to_stdout + end + + it "labels the fast arm 'ready', not 'hot'", :aggregate_failures do + expect { report.render }.to output(/ready median/).to_stdout + expect { report.render }.not_to output(/hot median/).to_stdout + end + + it "carries the cold-minus-ready delta as its own table column" do + expect { report.render }.to output(/span.*delta/).to_stdout + end + + it "says exactly what was tested in the preamble" do + expect { report.render }.to output(/invoked as: ri TCPServer/).to_stdout + end + + it "omits the legend by default" do + expect { report.render }.not_to output(/what it measures/).to_stdout + end + + it "prints a condensed slide summary of the cold layers" do + expect { report.render }.to output(/slide summary/).to_stdout + end + + context "when verbose" do + subject(:report) do + described_class.new(cold: arm_result(:cold, full: 200.0), hot: arm_result(:hot, full: 40.0), + protocol:, verbose: true) + end + + it "appends a legend and the harness vocabulary", :aggregate_failures do + expect { report.render }.to output(/what it measures/).to_stdout + expect { report.render }.to output(/^terms$/).to_stdout + end + end +end diff --git a/spec/bench/results_log_spec.rb b/spec/bench/results_log_spec.rb new file mode 100644 index 0000000..751473b --- /dev/null +++ b/spec/bench/results_log_spec.rb @@ -0,0 +1,28 @@ +require "tempfile" + +RSpec.describe Ready::Bench::ResultsLog do + subject(:results_log) { described_class.new(file.path) } + + let(:file) { Tempfile.new("bench-results") } + + def record(command, cold:, hot:) + results_log.append(command:, arm: :cold, full_milliseconds: cold) + results_log.append(command:, arm: :hot, full_milliseconds: hot) + end + + it "reads appended arm rows back as one comparison per command, in order" do + record("ri TCPServer", cold: 500.0, hot: 40.0) + record("rake", cold: 130.0, hot: 65.0) + + ri = Ready::Bench::Comparison.new(command: "ri TCPServer", cold_milliseconds: 500.0, hot_milliseconds: 40.0) + rake = Ready::Bench::Comparison.new(command: "rake", cold_milliseconds: 130.0, hot_milliseconds: 65.0) + expect(results_log.comparisons).to eq([ri, rake]) + end + + it "keeps the same executable on different inputs as distinct comparisons" do + record("ri TCPServer", cold: 500.0, hot: 40.0) + record("ri Socket", cold: 480.0, hot: 38.0) + + expect(results_log.comparisons.map(&:command)).to eq(["ri TCPServer", "ri Socket"]) + end +end diff --git a/spec/bench/round_spec.rb b/spec/bench/round_spec.rb new file mode 100644 index 0000000..e1742ed --- /dev/null +++ b/spec/bench/round_spec.rb @@ -0,0 +1,21 @@ +RSpec.describe Ready::Bench::Round do + it "flags warmup rounds" do + aggregate_failures do + expect(described_class.new(number: 1, warmup: true).warmup?).to be true + expect(described_class.new(number: 4, warmup: false).warmup?).to be false + end + end + + it "alternates arm order by round parity to cancel drift", :aggregate_failures do + expect(described_class.new(number: 3, warmup: false).arm_order).to eq(%i[hot cold]) + expect(described_class.new(number: 4, warmup: false).arm_order).to eq(%i[cold hot]) + end + + it "names each arm's run after the round" do + round = described_class.new(number: 3, warmup: false) + aggregate_failures do + expect(round.run_id(:cold)).to eq("cold.3") + expect(round.run_id(:hot)).to eq("hot.3") + end + end +end diff --git a/spec/bench/rubygems_stub_spec.rb b/spec/bench/rubygems_stub_spec.rb new file mode 100644 index 0000000..018223d --- /dev/null +++ b/spec/bench/rubygems_stub_spec.rb @@ -0,0 +1,31 @@ +RSpec.describe Ready::Bench::RubygemsStub do + subject(:stub) { described_class.new(source) } + + let(:source) do + <<~RUBY + #!/usr/bin/ruby + require 'rubygems' + Gem.use_gemdeps + version = ">= 0.a" + if Gem.respond_to?(:activate_and_load_bin_path) + Gem.activate_and_load_bin_path('irb', 'irb', version) + else + load Gem.activate_bin_path('irb', 'irb', version) + end + RUBY + end + + let(:instrumented) { stub.instrumented_source } + + it "injects marks at the stub's standard boundaries", :aggregate_failures do + expect(instrumented).to include(%(at_exit { ready_bench_mark("ruby_exit") })) + expect(instrumented).to include(%(Gem.use_gemdeps\nready_bench_mark("rubygems_ready"))) + expect(instrumented).to include(%(ready_bench_mark("bin_path_resolved"))) + end + + it "splits activation from executing the tool so each is its own span", :aggregate_failures do + expect(instrumented).to include("activated_bin_path = Gem.activate_bin_path('irb', 'irb', version)") + expect(instrumented).not_to include("Gem.activate_and_load_bin_path(") + expect(instrumented).to include("load activated_bin_path") + end +end diff --git a/spec/bench/run_spec.rb b/spec/bench/run_spec.rb new file mode 100644 index 0000000..76f9957 --- /dev/null +++ b/spec/bench/run_spec.rb @@ -0,0 +1,33 @@ +RSpec.describe Ready::Bench::Run do + subject(:run) do + described_class.new(id: "cold.3", mark_times: { harness_start: 1000.0, ruby_exit: 1000.2 }) + end + + it "knows which marks it recorded" do + aggregate_failures do + expect(run.recorded?(:harness_start)).to be true + expect(run.recorded?(:server_entry)).to be false + end + end + + it "measures the milliseconds between two recorded marks" do + expect(run.milliseconds_between(:harness_start, :ruby_exit)).to be_within(1e-6).of(200.0) + end + + describe "success of the invocation" do + def run_with(exit_status) + described_class.new(id: "cold.1", mark_times: {}, exit_status:) + end + + it "succeeds only on a zero exit status", :aggregate_failures do + expect(run_with(0).succeeded?).to be true + expect(run_with(1).succeeded?).to be false + expect(run_with(nil).succeeded?).to be false + end + + it "explains why it failed", :aggregate_failures do + expect(run_with(2).failure_reason).to eq("exited 2") + expect(run_with(nil).failure_reason).to include("no exit status") + end + end +end diff --git a/spec/bench/shim_spec.rb b/spec/bench/shim_spec.rb new file mode 100644 index 0000000..e206803 --- /dev/null +++ b/spec/bench/shim_spec.rb @@ -0,0 +1,51 @@ +require "fileutils" +require "tmpdir" + +RSpec.describe Ready::Bench::Shim do + let(:workdir) { Pathname(Dir.mktmpdir("shim")) } + let(:prelude_path) { workdir / "prelude.rb" } + + after { FileUtils.rm_rf(workdir) } + + shared_examples "a shim variant" do + before { shim.write! } + + let(:script) { (workdir / shim.command_word).read } + + it "writes an executable file named by its command word" do + expect(workdir / shim.command_word).to be_executable + end + + it "marks command_start, arms the prelude, and execs its variant's target", :aggregate_failures do + expect(script).to include("command_start") + expect(script).to include("--disable-gems -r#{prelude_path}") + expect(script).to include(expected_exec_line) + end + end + + describe Ready::Bench::RbenvShim do + subject(:shim) { described_class.new(executable_name: "irb", workdir:, prelude_path:) } + + let(:expected_exec_line) { %(exec rbenv exec "irb" "$@") } + + it "shares the tool's own name so PATH resolution cannot tell it apart" do + expect(shim.command_word).to eq("irb") + end + + it_behaves_like "a shim variant" + end + + describe Ready::Bench::DirectShim do + subject(:shim) do + described_class.new(executable_name: "irb", workdir:, prelude_path:, stub_path: workdir / "stub") + end + + let(:expected_exec_line) { %(exec "#{RbConfig.ruby}" "#{workdir / "stub"}" "$@") } + + it "answers a distinct command word so both variants can share a PATH" do + expect(shim.command_word).to eq("irb_direct") + end + + it_behaves_like "a shim variant" + end +end diff --git a/spec/bench/slide_summary_spec.rb b/spec/bench/slide_summary_spec.rb new file mode 100644 index 0000000..d4b1d77 --- /dev/null +++ b/spec/bench/slide_summary_spec.rb @@ -0,0 +1,85 @@ +RSpec.describe Ready::Bench::SlideSummary do + subject(:summary) { described_class.new(cold:, rbenv_shim_overhead: 40.0) } + + let(:cold) do + cold_arm(shell: 5.0, launch: 40.0, rubygems: 5.0, activation: 37.0, + tool_run: 19.0, reap: 2.0, full: 108.0) + end + + def cold_arm(**durations) + cold_arm_over(durations) + end + + # A cold ArmResult with one measured round per span=>ms hash given. + def cold_arm_over(*rounds) + Ready::Bench::ArmResult.new(name: :cold, warmups: 0).tap do |result| + rounds.each do |durations| + waterfall = Ready::Bench::Waterfall.new(**durations) + result.record(Ready::Bench::Measurement.new(waterfall:, wall_clock_seconds: 0.15)) + end + end + end + + def milliseconds_for(label) + summary.lines.find { it.label == label }.rounded_milliseconds + end + + it "reports each cold layer as its own rounded-millisecond line", :aggregate_failures do + expect(milliseconds_for("shell")).to eq(5) + expect(milliseconds_for("activate deps")).to eq(37) + expect(milliseconds_for("the tool")).to eq(19) + end + + it "keeps the ruby vm boot as its own layer, separate from rubygems", :aggregate_failures do + # both arms boot a VM; only cold loads rubygems -- folding them would imply + # ready eliminates the boot, which it does not. + expect(milliseconds_for("ruby vm boot")).to eq(40) + expect(milliseconds_for("rubygems")).to eq(5) + end + + it "orders the layers with the rbenv shim after the shell", :aggregate_failures do + expect(summary.lines.map(&:label)).to eq( + ["shell", "rbenv shim", "ruby vm boot", "rubygems", "activate deps", "the tool"], + ) + expect(milliseconds_for("rbenv shim")).to eq(40) + end + + it "omits the rbenv shim when no overhead was measured" do + without_shim = described_class.new(cold:, rbenv_shim_overhead: nil) + expect(without_shim.lines.map(&:label)).not_to include("rbenv shim") + end + + it "totals the real end-to-end cold time through the shim, tilde-marked", :aggregate_failures do + expect(summary.total_line.rounded_milliseconds).to eq(148) + expect(summary.total_line.prefix).to eq("~") + end + + it "renders aligned label/millisecond columns for a slide", :aggregate_failures do + rendered = summary.render + expect(rendered).to match(/ruby vm boot\s+40 ms/) + expect(rendered).to match(/rubygems\s+5 ms/) + expect(rendered).to match(/total\s+~148 ms/) + end + + context "with several measured rounds" do + subject(:summary) { described_class.new(cold: multi_cold, rbenv_shim_overhead: nil) } + + let(:multi_cold) do + cold_arm_over( + { shell: 6.0, launch: 12.0, rubygems: 50.0, activation: 40.0, tool_run: 90.0, reap: 3.0, full: 201.0 }, + { shell: 5.0, launch: 10.0, rubygems: 45.0, activation: 33.0, tool_run: 80.0, reap: 2.0, full: 175.0 }, + ) + end + + it "takes each layer's minimum, not its median", :aggregate_failures do + expect(milliseconds_for("ruby vm boot")).to eq(10) # min launch + expect(milliseconds_for("rubygems")).to eq(45) # min rubygems + expect(milliseconds_for("activate deps")).to eq(33) # min activation + expect(milliseconds_for("the tool")).to eq(80) # min tool_run + end + + it "never lets the layers exceed the total" do + expect(summary.lines.sum(&:milliseconds)).to be <= summary.total_line.milliseconds + end + end +end diff --git a/spec/bench/span_spec.rb b/spec/bench/span_spec.rb new file mode 100644 index 0000000..d9bc220 --- /dev/null +++ b/spec/bench/span_spec.rb @@ -0,0 +1,31 @@ +RSpec.describe Ready::Bench::Span do + subject(:span) do + described_class.new(label: :launch, opening_mark: :command_start, closing_mark: :ruby_up, + summary: :minimum, description: "interpreter boot") + end + + describe "#measure" do + it "returns the milliseconds between its two marks" do + run = Ready::Bench::Run.new(id: "cold.1", mark_times: { command_start: 10.0, ruby_up: 10.064 }) + expect(span.measure(run)).to be_within(1e-6).of(64.0) + end + + it "returns nil when the run did not record both marks" do + run = Ready::Bench::Run.new(id: "hot.1", mark_times: { ruby_up: 10.0 }) + expect(span.measure(run)).to be_nil + end + end + + describe "#summarize" do + it "takes the floor of a :minimum span (jitter only ever adds time)" do + expect(span.summarize([3.0, 1.0, 2.0])).to eq(1.0) + end + + it "takes the median of a :median span" do + median_span = described_class.new(label: :tool_run, opening_mark: :bin_path_resolved, + closing_mark: :ruby_exit, summary: :median, + description: "the tool itself") + expect(median_span.summarize([100.0, 120.0, 110.0])).to eq(110.0) + end + end +end diff --git a/spec/bench/stats_spec.rb b/spec/bench/stats_spec.rb new file mode 100644 index 0000000..e6c3fef --- /dev/null +++ b/spec/bench/stats_spec.rb @@ -0,0 +1,9 @@ +RSpec.describe Ready::Bench::Stats do + it "returns the middle value for odd counts" do + expect(described_class.median([3, 1, 2])).to eq(2) + end + + it "averages the two middle values for even counts" do + expect(described_class.median([1, 2, 3, 4])).to eq(2.5) + end +end diff --git a/spec/bench/waterfall_spec.rb b/spec/bench/waterfall_spec.rb new file mode 100644 index 0000000..cdc06d1 --- /dev/null +++ b/spec/bench/waterfall_spec.rb @@ -0,0 +1,56 @@ +RSpec.describe Ready::Bench::Waterfall do + describe ".of" do + subject(:waterfall) { described_class.of(run) } + + context "with a cold run's marks" do + let(:mark_times) do + { harness_start: 1000.000, command_start: 1000.001, ruby_up: 1000.065, rubygems_ready: 1000.065, + bin_path_resolved: 1000.097, ruby_exit: 1000.200, harness_end: 1000.201 } + end + + let(:run) { Ready::Bench::Run.new(id: "cold.1", mark_times:) } + + it "measures every span whose marks the run recorded", :aggregate_failures do + expect(waterfall.duration_of(:launch)).to be_within(1e-6).of(64.0) + expect(waterfall.duration_of(:activation)).to be_within(1e-6).of(32.0) + expect(waterfall.duration_of(:tool_run)).to be_within(1e-6).of(103.0) + expect(waterfall.duration_of(:full)).to be_within(1e-6).of(201.0) + end + end + + context "with a hot run's marks (no shim, no rubygems)" do + let(:run) do + Ready::Bench::Run.new(id: "hot.1", mark_times: { + harness_start: 1.000, command_start: 1.001, server_entry: 1.053, harness_end: 1.072 + }) + end + + it "measures only the spans whose marks exist", :aggregate_failures do + expect(waterfall.measured?(:launch)).to be false + expect(waterfall.duration_of(:shell)).to be_within(1e-6).of(1.0) + expect(waterfall.duration_of(:dispatch_overhead)).to be_within(1e-6).of(52.0) + expect(waterfall.duration_of(:full)).to be_within(1e-6).of(72.0) + end + end + end + + describe ".summarizing" do + subject(:summary) { described_class.summarizing(waterfalls) } + + let(:waterfalls) do + [ + described_class.new(shell: 3.0, tool_run: 100.0), + described_class.new(shell: 1.0, tool_run: 110.0), + described_class.new(shell: 2.0, tool_run: 120.0), + ] + end + + it "takes the floor of a process-creation span" do + expect(summary.duration_of(:shell)).to eq(1.0) + end + + it "takes the median of an in-process span" do + expect(summary.duration_of(:tool_run)).to eq(110.0) + end + end +end diff --git a/spec/e2e/benchmark_spec.rb b/spec/e2e/benchmark_spec.rb new file mode 100644 index 0000000..1b1498b --- /dev/null +++ b/spec/e2e/benchmark_spec.rb @@ -0,0 +1,17 @@ +RSpec.describe "ready startup benchmark", :e2e do + it "shows the hot arm eliminating the boot layers and beating cold by a wide margin" do + protocol = Ready::Bench::Protocol.new(executable_name: "ri", arguments: ["TCPServer"], + preload_gems: ["rdoc"], rounds: 4, warmups: 2) + runner = Ready::Bench::Runner.new(protocol:, rbenv: false).call + cold = runner.cold_summary + hot = runner.hot_summary + aggregate_failures do + expect(cold.duration_of(:activation)).to be > 5.0 # cold pays gem activation + expect(cold.duration_of(:tool_run)).to be > 5.0 # ...and the doc lookup itself + expect(hot.duration_of(:full)).to be < cold.duration_of(:full) + expect(hot.measured?(:launch)).to be false # hot skips the rbenv/boot layer + expect(hot.duration_of(:shell)).to be < 5.0 # a function call, not a fork/exec + expect(hot.duration_of(:dispatch_overhead)).to be > 0 + end + end +end diff --git a/spec/e2e/dispatch_spec.rb b/spec/e2e/dispatch_spec.rb new file mode 100644 index 0000000..2174001 --- /dev/null +++ b/spec/e2e/dispatch_spec.rb @@ -0,0 +1,26 @@ +RSpec.describe "ready end-to-end dispatch", :e2e do + before(:all) { @sandbox = Ready::Sandbox.build(executables: ["rake"]) } + after(:all) { @sandbox&.teardown } + + def in_shell + shell = Ready::PtyShell.new(@sandbox.shell_env) + shell.run("source #{@sandbox.plugin_path}") + yield shell + ensure + shell&.close + end + + it "aliases the bare command to the ready_ stub via readyinit" do + in_shell do |shell| + result = shell.run("whence -v rake") + expect(result.output).to include("rake is an alias for ready_rake") + end + end + + it "dispatches through the by-server and returns the executable's real output" do + in_shell do |shell| + result = shell.run("rake --version") + expect(result.output).to match(/rake, version \d+\.\d+/) + end + end +end diff --git a/spec/e2e/pty_shell_spec.rb b/spec/e2e/pty_shell_spec.rb new file mode 100644 index 0000000..4fde9fb --- /dev/null +++ b/spec/e2e/pty_shell_spec.rb @@ -0,0 +1,32 @@ +RSpec.describe Ready::PtyShell, :e2e do + subject(:shell) { described_class.new } + + after { shell.close } + + it "runs a command over a pty and captures its output", :aggregate_failures do + result = shell.run("print hello-from-zsh") + expect(result.output).to include("hello-from-zsh") + expect(result.wall_clock_seconds).to be > 0 + end + + context "when constructed with environment variables" do + subject(:shell) { described_class.new("READY_PROBE" => "xyz123") } + + it "injects them into the interactive shell" do + result = shell.run("print $READY_PROBE") + expect(result.output).to include("xyz123") + end + end + + context "when a foreground child is left holding the pty" do + it "closes promptly by signalling the process group instead of blocking" do + # `cat` with no args occupies the pty and would make a graceful `exit` + # hang forever -- exactly the macOS cold-arm failure. + shell.instance_variable_get(:@in).puts("cat") + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + shell.close + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + expect(elapsed).to be < 3 + end + end +end diff --git a/spec/rakefile_spec.rb b/spec/rakefile_spec.rb new file mode 100644 index 0000000..5ccf8f7 --- /dev/null +++ b/spec/rakefile_spec.rb @@ -0,0 +1,26 @@ +require "tmpdir" +require "fileutils" +require "rbconfig" + +# Regression: run as an installed gem, `ready up|compile|clobber` load the +# shipped Rakefile from a directory with no gemspec. The dev-only tooling it +# requires (bundler/gem_tasks, rspec, rubocop, gempilot) must NOT load there, or +# bundler/gem_tasks aborts with "Unable to determine name from existing gemspec". +RSpec.describe "Rakefile runtime safety" do + # Load only the Rakefile (no gemspec, no dev gems), as an installed gem would. + def rake_output(dir) + IO.popen( + { "BUNDLE_GEMFILE" => nil, "RUBYOPT" => nil }, + [RbConfig.ruby, Gem.bin_path("rake", "rake"), "-f", (Pathname(dir) / "Rakefile").to_s, "--tasks"], + chdir: dir, err: %i[child out], &:read + ) + end + + it "loads without dev tooling when no gemspec is present (installed-gem layout)" do + Dir.mktmpdir do |dir| + FileUtils.cp(Ready.root / "Rakefile", dir) + + expect(rake_output(dir)).not_to include("Unable to determine name from existing gemspec") + end + end +end diff --git a/spec/ready/by_executable_spec.rb b/spec/ready/by_executable_spec.rb index b878921..cab3d0f 100644 --- a/spec/ready/by_executable_spec.rb +++ b/spec/ready/by_executable_spec.rb @@ -1,6 +1,6 @@ RSpec.describe Ready::ByExecutable do describe "#to_alias" do - it "disables rubygems and yjit by default" do + it "disables rubygems and yjit by default", :aggregate_failures do line = described_class.new.without_rubygems.without_yjit.to_alias expect(line).to include("--disable-gems") expect(line).not_to include("--yjit") diff --git a/spec/ready/cli/compile_spec.rb b/spec/ready/cli/compile_spec.rb index 1e22583..4dcc433 100644 --- a/spec/ready/cli/compile_spec.rb +++ b/spec/ready/cli/compile_spec.rb @@ -9,7 +9,7 @@ def run_compile(*argv) end describe "the `by` client alias" do - it "prints the by alias for `by`" do + it "prints the by alias for `by`", :aggregate_failures do output, status = run_compile("by") expect(status).to eq(0) expect(output).to include("--disable-gems") @@ -29,51 +29,53 @@ def run_compile(*argv) describe "named CLIs" do let(:script) { instance_double(Ready::ZshScript, to_s: "STUB") } - it "routes names and accumulated -e env through ZshScript" do - expect(Ready::ZshScript).to receive(:new) - .with(names: ["irb", "rspec"], environment: { "BY_SOCKET" => "/x", "RAILS_ENV" => "production" }) - .and_return(script) + before { allow(Ready::ZshScript).to receive(:new).and_return(script) } + it "routes names and accumulated -e env through ZshScript", :aggregate_failures do output, status = run_compile("-e", "BY_SOCKET=/x", "-e", "RAILS_ENV=production", "irb", "rspec") + expect(Ready::ZshScript).to have_received(:new) + .with(names: ["irb", "rspec"], environment: { "BY_SOCKET" => "/x", "RAILS_ENV" => "production" }) expect(status).to eq(0) expect(output).to eq("STUB") end it "preserves `=` inside an -e value" do - expect(Ready::ZshScript).to receive(:new) - .with(names: ["rails"], environment: { "DB" => "postgres://x=y" }) - .and_return(script) - run_compile("-e", "DB=postgres://x=y", "rails") + + expect(Ready::ZshScript).to have_received(:new) + .with(names: ["rails"], environment: { "DB" => "postgres://x=y" }) end end - describe "`all`" do - it "delegates to the ready:compile rake task" do - command = described_class.new(stdout: StringIO.new, stderr: StringIO.new) - expect(command).to receive(:rake).with("ready:compile") - command.run("all") + describe "`all`, the default" do + def compile_via_run(*names) + described_class.new(stdout: StringIO.new, stderr: StringIO.new).tap do |command| + allow(command).to receive(:rake) + command.run(*names) + end + end + + it "delegates `all` to the ready:compile rake task" do + expect(compile_via_run("all")).to have_received(:rake).with("ready:compile") + end + + it "defaults to compiling everything when given no names, like `make`" do + expect(compile_via_run).to have_received(:rake).with("ready:compile") end end describe "input validation" do - it "rejects an -e argument without `=`" do + it "rejects an -e argument without `=`", :aggregate_failures do _output, status, stderr = run_compile("-e", "NOEQUALS", "irb") expect(status).to eq(1) - expect(stderr).to match(/invalid --environment/) + expect(stderr).to include("invalid --environment") end - it "rejects `all`/`by` combined with other names" do + it "rejects `all`/`by` combined with other names", :aggregate_failures do _output, status, stderr = run_compile("all", "irb") expect(status).to eq(1) - expect(stderr).to match(/cannot be combined/) - end - - it "exits with an insufficient-arguments error when given no arguments" do - _output, status, stderr = run_compile - expect(status).to eq(1) - expect(stderr).to match(/insufficient number of arguments/i) + expect(stderr).to include("cannot be combined") end end end diff --git a/spec/ready/cli/init_spec.rb b/spec/ready/cli/init_spec.rb index 301145c..2b64efb 100644 --- a/spec/ready/cli/init_spec.rb +++ b/spec/ready/cli/init_spec.rb @@ -1,25 +1,29 @@ require "stringio" RSpec.describe Ready::CLI::Init do - it "prints a source line pointing at the bundled zsh plugin" do + def run_init(*argv) stdout = StringIO.new - described_class.new(stdout: stdout).run - expect(stdout.string).to match(%r{\Asource .+/zsh/ready/ready\.plugin\.zsh\n\z}) + stderr = StringIO.new + status = described_class.main(argv, stdout: stdout, stderr: stderr) + [stdout.string, status, stderr.string] + end + + it "prints a source line pointing at the bundled zsh plugin" do + output, = run_init + expect(output).to eq("source #{described_class::PLUGIN_PATH}\n") end it "points at a plugin file that exists" do expect(described_class::PLUGIN_PATH).to exist end - it "errors and exits non-zero when the plugin is missing" do + it "errors and exits non-zero when the plugin is missing", :aggregate_failures do stub_const("#{described_class}::PLUGIN_PATH", Pathname("/no/such/ready.plugin.zsh")) - stdout = StringIO.new - stderr = StringIO.new - status = described_class.main([], stdout: stdout, stderr: stderr) + output, status, stderr = run_init expect(status).to eq(1) - expect(stdout.string).to be_empty - expect(stderr.string).to match(/not found/) + expect(output).to be_empty + expect(stderr).to include("not found") end end diff --git a/spec/ready/cli/rake_command_spec.rb b/spec/ready/cli/rake_command_spec.rb index 3fb1437..b40351a 100644 --- a/spec/ready/cli/rake_command_spec.rb +++ b/spec/ready/cli/rake_command_spec.rb @@ -1,21 +1,46 @@ +require "fileutils" +require "rbconfig" require "stringio" +require "tmpdir" -# Exercised through Up, the simplest command that includes the mixin. +# Exercised through Up, the simplest command that includes the mixin. Instead of +# stubbing the Kernel calls on the command under test, each example points +# Ready.root at a throwaway project with its own Rakefile and drives the real +# `rake` shell-out, then asserts on what the spawned process actually did. RSpec.describe Ready::CLI::RakeCommand do subject(:command) { Ready::CLI::Up.new(stdout: StringIO.new, stderr: StringIO.new) } - it "runs rake under the current Ruby, in the project root" do - expect(command).to receive(:system) - .with(RbConfig.ruby, "-S", "rake", "ready", chdir: Ready.root.to_s) - .and_return(true) + let(:project_root) { Pathname(Dir.mktmpdir).realpath } - command.run + before do + project_root.join("Rakefile").write(rakefile) + allow(Ready).to receive(:root).and_return(project_root) end - it "exits when rake fails" do - allow(command).to receive(:system).and_return(false) - expect(command).to receive(:exit) + after { FileUtils.remove_entry(project_root) } - command.run + context "when the rake task succeeds" do + let(:rakefile) do + <<~'RUBY' + require "rbconfig" + task(:ready) { File.write("ready.log", "#{RbConfig.ruby}\n#{Dir.pwd}") } + RUBY + end + + it "runs rake under the current Ruby, in the project root", :aggregate_failures do + command.run + + interpreter, working_dir = project_root.join("ready.log").read.lines(chomp: true) + expect(interpreter).to eq(RbConfig.ruby) + expect(working_dir).to eq(project_root.to_s) + end + end + + context "when the rake task fails" do + let(:rakefile) { "task(:ready) { exit 3 }" } + + it "exits, propagating rake's failure status" do + expect { command.run }.to raise_error(an_instance_of(SystemExit).and(having_attributes(status: 3))) + end end end diff --git a/spec/ready/cli_spec.rb b/spec/ready/cli_spec.rb new file mode 100644 index 0000000..0fb63e4 --- /dev/null +++ b/spec/ready/cli_spec.rb @@ -0,0 +1,13 @@ +require "stringio" + +RSpec.describe Ready::CLI do + ["--version", "-V"].each do |flag| + it "prints the version and exits 0 for #{flag}", :aggregate_failures do + stdout = StringIO.new + status = described_class.main([flag], stdout: stdout, stderr: StringIO.new) + + expect(stdout.string).to eq("ready #{Ready::VERSION}\n") + expect(status).to eq(0) + end + end +end diff --git a/spec/ready/readyfile_spec.rb b/spec/ready/readyfile_spec.rb index bd43ef3..0826850 100644 --- a/spec/ready/readyfile_spec.rb +++ b/spec/ready/readyfile_spec.rb @@ -1,24 +1,22 @@ +require "fileutils" require "tempfile" require "tmpdir" RSpec.describe Ready::Readyfile do - around do |example| - Dir.mktmpdir do |dir| - @build_dir = dir - example.run - end - end + let(:build_dir) { Dir.mktmpdir } + + after { FileUtils.remove_entry(build_dir) } def readyfile(content) file = Tempfile.new("readyfile") file.write(content) if content file.close - described_class.open(file.path, build_dir: @build_dir) + described_class.open(file.path, build_dir: build_dir) end describe ".open" do - it "treats a missing readyfile as empty" do - config = described_class.open("/no/such/readyfile", build_dir: @build_dir) + it "treats a missing readyfile as empty", :aggregate_failures do + config = described_class.open("/no/such/readyfile", build_dir: build_dir) expect(config.gem_names).to eq([]) expect(config.executable_names).to eq([]) end @@ -31,7 +29,7 @@ def readyfile(content) expect(readyfile("---\n").gem_names).to eq([]) end - it "parses declared gems and executables" do + it "parses declared gems and executables", :aggregate_failures do config = readyfile("gems:\n - rails\nexecutables:\n - irb\n") expect(config.gem_names).to eq(["rails"]) expect(config.executable_names).to eq(["irb"]) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index edb1393..be66414 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,9 +1,24 @@ require "ready" +require "zeitwerk" + +# Autoload the e2e/benchmark harness the Zeitwerk way, into the Ready namespace +# (spec/support/pty_shell.rb -> Ready::PtyShell, support/bench/span.rb -> +# Ready::Bench::Span, ...). A second loader may share a namespace owned by the +# gem's for_gem loader. +Zeitwerk::Loader.new.tap do |loader| + loader.inflector.inflect("cli" => "CLI") + loader.push_dir(Pathname(__dir__) / "support", namespace: Ready) + loader.setup +end RSpec.configure do |config| config.example_status_persistence_file_path = ".rspec_status" config.disable_monkey_patching! + # E2E specs need zsh + a real ~10s+ readyup build; keep them out of the fast + # loop unless explicitly opted in (the `spec:e2e` rake task sets READY_E2E). + config.filter_run_excluding :e2e unless ENV["READY_E2E"] + config.expect_with :rspec do |c| c.syntax = :expect end diff --git a/spec/support/bench.rb b/spec/support/bench.rb new file mode 100644 index 0000000..53b1834 --- /dev/null +++ b/spec/support/bench.rb @@ -0,0 +1,13 @@ +module Ready + ## + # The cold-vs-hot startup benchmark. Entities live under bench/; this file + # holds the namespace-level helpers they share. + module Bench + # Evaluated by constructors that let rbenv default. The cold arm cannot + # resolve rubygems stubs without rbenv, so a missing rbenv fails fast at + # construction rather than deep inside the first round. + def self.rbenv_available? + system("command -v rbenv >/dev/null 2>&1", exception: true) + end + end +end diff --git a/spec/support/bench/arm_result.rb b/spec/support/bench/arm_result.rb new file mode 100644 index 0000000..ec2bd50 --- /dev/null +++ b/spec/support/bench/arm_result.rb @@ -0,0 +1,43 @@ +module Ready + module Bench + ## + # One arm's accumulated measurements (:cold boots fresh processes, :hot + # dispatches to the warm server). Records a Measurement per run, excludes + # the warmup runs from every number it reports, and summarizes on demand. + class ArmResult + attr_reader :name + + def initialize(name:, warmups:) + @name = name + @warmups = warmups + @measurements = [] + end + + def record(measurement) + @measurements << measurement + end + + def summary + Waterfall.summarizing(measured.map(&:waterfall)) + end + + def duration_of(label) + summary.duration_of(label) + end + + def samples_of(label) + measured.filter_map { it.waterfall.duration_of(label) } + end + + def median_wall_clock_milliseconds + Stats.median(measured.map(&:wall_clock_seconds)) * 1000.0 + end + + private + + def measured + @measurements.drop(@warmups) + end + end + end +end diff --git a/spec/support/bench/cli.rb b/spec/support/bench/cli.rb new file mode 100644 index 0000000..41a2200 --- /dev/null +++ b/spec/support/bench/cli.rb @@ -0,0 +1,171 @@ +require "command_kit/command" +require "English" +require "rbconfig" +require "tmpdir" + +module Ready + module Bench + ## + # Development-only ergonomic front door for the benchmark rake tasks. It + # owns no benchmark logic: flags become the BENCH_* environment the tasks + # already read, each command gets its own proven `rake bench` run, and + # --plot charts the collected results with youplot afterwards. Only what + # the user chose is exported, so the rake task's own defaults cover + # everything else. + class CLI < CommandKit::Command + command_name "bench" + + usage "[options] [COMMAND [ARGUMENT ...]] [-- COMMAND [ARGUMENT ...]]..." + + option :readyfile, value: { type: String, usage: "PATH" }, + desc: "Readyfile whose gems: the warm server preloads; with no " \ + "COMMANDs given, every executable it declares is benched" + + option :rounds, value: { type: Integer, usage: "N" }, + desc: "Measured rounds per arm (default: 15)" + + option :warmups, value: { type: Integer, usage: "N" }, + desc: "Warmup rounds, recorded but excluded (default: 3)" + + option :verbose, short: "-v", + desc: "Append the span legend and the terms table" + + option :plot, value: { + type: { "stacked" => :stacked, "youplot" => :youplot }, + }, + desc: "Chart cold vs hot per command after the runs: stacked draws one " \ + "bar per command (hot segment + what ready eliminates), youplot " \ + "draws stock side-by-side pairs" + + argument :command, required: false, + repeats: true, + usage: "COMMAND [ARGUMENT ...]", + desc: "Command to benchmark, exactly as you would type it; " \ + "separate several with -- (default: ri TCPServer)" + + description "Benchmark CLIs cold (fresh boot) vs hot (ready dispatch); needs zsh + by-server + rbenv" + + examples [ + "", + "ri TCPServer", + "--readyfile readyfile ri TCPServer -- ronin help -- kamal version", + "--readyfile readyfile --plot stacked", + ] + + # + # Splits argv into command segments on `--` before options are parsed + # (OptionParser would otherwise eat the first separator). Flags belong + # in the first segment; later segments are commands, verbatim. + # + def self.command_segments(argv) + argv.each_with_object([[]]) do |word, segments| + word == "--" ? segments << [] : segments.last << word + end + end + + def main(argv = []) + first_segment, *rest = self.class.command_segments(argv) + @extra_command_segments = rest + super(first_segment || []) + end + + # + # One proven rake run per command, then the optional chart. + # + def run(*command_words) + segments = [command_words, *@extra_command_segments].reject(&:empty?) + invocations_under_test(segments).each do |invocation| + run_rake(environment_for(invocation), task_name) + end + plot_results if options[:plot] + end + + # + # The commands to bench: the typed segments, else everything the + # readyfile declares (bare), else [nil] -- one run on the rake task's + # default. + # + def invocations_under_test(segments) + return segments.map { Invocation.parse(it) } unless segments.empty? + return [nil] unless readyfile + + readyfile.executable_names.map { Invocation.bare(it) } + end + + # + # The BENCH_* environment for one rake run: only the knobs the user + # actually turned. A typed command is exported verbatim -- executable + # and arguments both -- so what you typed is what runs. + # + def environment_for(invocation) + flag_environment.merge(command_environment(invocation)).compact + end + + # + # Which of the two proven tasks to run. + # + def task_name + options[:verbose] ? "bench:verbose" : "bench" + end + + # + # The renderer for the chosen --plot style: our stacked bars, or the + # stock youplot pairs. + # + def plotter_for(style, comparisons) + case style + in :stacked then Plot::Stacked.new(comparisons) + in :youplot then Plot::Youplot.new(comparisons) + end + end + + private + + def flag_environment + { + "BENCH_READYFILE" => options[:readyfile], + "BENCH_RUNS" => options[:rounds]&.to_s, + "BENCH_WARMUPS" => options[:warmups]&.to_s, + "BENCH_RESULTS" => (results_path.to_s if options[:plot]), + } + end + + def command_environment(invocation) + return {} unless invocation + + { + "BENCH_EXE" => invocation.executable_name, + "BENCH_ARGS" => invocation.arguments.join(" "), + } + end + + # Only gem/executable names are read; Readyfile demands an existing + # build_dir regardless, so the readyfile's own parent satisfies it. + def readyfile + return nil unless options[:readyfile] + + @readyfile ||= begin + path = Pathname(options[:readyfile]) + Ready::Readyfile.open(path, build_dir: path.expand_path.parent) + end + end + + # Where each rake run appends its "executable,arm,full_ms" rows. + def results_path + @results_path ||= Pathname(Dir.mktmpdir("bench")) / "results.csv" + end + + # Reads back every run's results and renders them in the chosen style. + def plot_results + comparisons = ResultsLog.new(results_path).comparisons + plotter_for(options[:plot], comparisons).render + end + + def run_rake(environment, task) + return if system(environment, RbConfig.ruby, "-S", "rake", task, chdir: Ready.root.to_s) + + exit($CHILD_STATUS&.exitstatus || 1) + end + end + end +end diff --git a/spec/support/bench/cold_arm.rb b/spec/support/bench/cold_arm.rb new file mode 100644 index 0000000..19d4105 --- /dev/null +++ b/spec/support/bench/cold_arm.rb @@ -0,0 +1,78 @@ +module Ready + module Bench + ## + # Prepares the cold arm: faithful, mark-instrumented COPIES of the files a + # cold invocation runs through, so every layer can be attributed without + # touching any real file. instrument! writes four artifacts into the + # workdir: + # + # prelude.rb - marks ruby_up; armed via RUBYOPT -r, so it is the first + # Ruby user code to run + # stub - the real rubygems stub for the executable, copied and + # instrumented at its standard boundaries (RubygemsStub) + # two shims - RbenvShim (the real `rbenv exec` chain) and DirectShim + # (execs the stub copy under this Ruby); see each class + class ColdArm + def initialize(executable_name:, workdir:, marks_log:, rbenv: true) + @executable_name = executable_name + @workdir = Pathname(workdir) + @marks_log = marks_log + @rbenv = rbenv + end + + def rbenv_shim + @rbenv_shim ||= RbenvShim.new(executable_name:, workdir:, prelude_path:) + end + + def direct_shim + @direct_shim ||= DirectShim.new(executable_name:, workdir:, prelude_path:, + stub_path: instrumented_stub_path) + end + + def instrument! + workdir.mkpath + write_prelude + write_instrumented_stub + shims.each(&:write!) + end + + # Environment a shell exports so marks land in the log and the shim + # copies shadow the real commands on PATH. + def environment_for(run_id:) + { + "READY_MARKS" => @marks_log.path.to_s, + "READY_RUN_ID" => run_id, + "PATH" => "#{workdir}:#{ENV.fetch("PATH", nil)}", + }.merge(Ready::Sandbox::NON_INTERACTIVE_ENV) + end + + private + + attr_reader :executable_name, :workdir + + # The rbenv shim exists only to measure the real `rbenv exec` overhead, + # which the runner probes only when rbenv is present. Without rbenv the + # clean DirectShim is the whole cold arm. + def shims + @rbenv ? [rbenv_shim, direct_shim] : [direct_shim] + end + + def write_prelude + prelude_path.write("#{MarkHelper.definition}#{MarkHelper.record(:ruby_up)}\n") + end + + def write_instrumented_stub + stub = RubygemsStub.for_executable(executable_name, rbenv: @rbenv) + instrumented_stub_path.write(stub.instrumented_source) + end + + def prelude_path + workdir / "prelude.rb" + end + + def instrumented_stub_path + workdir / "stub" + end + end + end +end diff --git a/spec/support/bench/comparison.rb b/spec/support/bench/comparison.rb new file mode 100644 index 0000000..fa5a3fc --- /dev/null +++ b/spec/support/bench/comparison.rb @@ -0,0 +1,19 @@ +module Ready + module Bench + ## + # One command's headline verdict: its cold and hot full startup times, in + # milliseconds, side by side. Identified by the command as typed + # ("ri TCPServer"), so the same executable on two inputs stays two + # comparisons. + class Comparison < Data.define(:command, :cold_milliseconds, :hot_milliseconds) + def speedup + cold_milliseconds / hot_milliseconds + end + + # What ready removes from every invocation -- the pitch, as a number. + def eliminated_milliseconds + cold_milliseconds - hot_milliseconds + end + end + end +end diff --git a/spec/support/bench/direct_shim.rb b/spec/support/bench/direct_shim.rb new file mode 100644 index 0000000..a3fee9f --- /dev/null +++ b/spec/support/bench/direct_shim.rb @@ -0,0 +1,28 @@ +require "rbconfig" + +module Ready + module Bench + ## + # The shim variant that skips rbenv and execs the instrumented rubygems + # stub copy directly under this Ruby -- its :launch span is pure + # interpreter boot, and the stub copy's own marks split every layer after + # it. + class DirectShim < Shim + def command_word + "#{executable_name}_direct" + end + + private + + attr_reader :stub_path + + def post_initialize(stub_path:) + @stub_path = stub_path + end + + def exec_line + %(exec "#{RbConfig.ruby}" "#{stub_path}" "$@") + end + end + end +end diff --git a/spec/support/bench/harness_log.rb b/spec/support/bench/harness_log.rb new file mode 100644 index 0000000..2bff892 --- /dev/null +++ b/spec/support/bench/harness_log.rb @@ -0,0 +1,30 @@ +module Ready + module Bench + ## + # The append-only log every measured command tees its stdout+stderr into, + # so a tool that fails silently leaves evidence in ./log/bench.log instead + # of being timed as a fast "success". Persisted in the project tree so it + # survives across runs; each run's output is headed by its run id. + class HarnessLog + attr_reader :path + + def initialize(invocation:) + @invocation = invocation + @path = Ready.root / "log" / "bench.log" + end + + # Creates ./log and heads this run's section so appended runs stay legible. + def open! + path.dirname.mkpath + path.open("a") { it.puts("\n===== #{@invocation} =====") } + self + end + + # Shell fragment teeing a command's stdout+stderr here under a run-id + # header; the pty still sees nothing (>/dev/null), so marker sync holds. + def tee(run_id, command) + "{ print -r -- '### #{run_id} ###'; #{command}; } 2>&1 | tee -a #{path} >/dev/null" + end + end + end +end diff --git a/spec/support/bench/hot_arm.rb b/spec/support/bench/hot_arm.rb new file mode 100644 index 0000000..a723d52 --- /dev/null +++ b/spec/support/bench/hot_arm.rb @@ -0,0 +1,61 @@ +require "shellwords" + +module Ready + module Bench + ## + # Runs the executable through the warm by-server, instrumenting the REAL + # Ready::Executable render output (a copy -- no gem runtime is edited). + # The zsh stub and the instrumented source mark: + # + # command_start - first statement of the zsh stub function, closing the + # :shell span (hot pays a function call there, no fork, + # no exec -- the same boundary the cold shim marks) + # server_entry - first statement of the eval'd source; command_start + # to here is :dispatch_overhead -- client boot, socket + # round-trip, and the server forking a worker + # pre_tool - immediately before the tool's entry call, where the + # preloaded requires end and :server_tool_run begins + class HotArm + TOOL_ENTRY_CALL = /^(\s*)([A-Z][\w:]*\.(?:start|run)\b|main\b)/ + + def self.instrument_source(rendered_source) + prologue = "#{MarkHelper.definition}#{MarkHelper.record(:server_entry)}\n" + "#{prologue}#{rendered_source}".sub(TOOL_ENTRY_CALL) do + indent = Regexp.last_match(1) + entry_call = Regexp.last_match(2) + "#{indent}#{MarkHelper.record(:pre_tool)}\n#{indent}#{entry_call}" + end + end + + def initialize(executable_name:, rendered_source:, sandbox:, marks_log:) + @executable_name = executable_name + @rendered_source = rendered_source + @sandbox = sandbox + @marks_log = marks_log + end + + # A faithful ready_ zsh function (mirrors fn.zsh.erb) whose + # inlined source carries the marks. +rendered_source+ is the production + # render, resolved by NAME exactly as `ready gem ` does. The + # command_start mark (bench_mark comes from prof.zsh, sourced first) + # closes :shell and opens :dispatch_overhead. + def stub_function + instrumented = self.class.instrument_source(@rendered_source) + <<~ZSH + ready_#{@executable_name}() { + emulate -L zsh + bench_mark command_start + autoload -Uz ready_by + BY_SOCKET=#{@sandbox.sock_path} ready_by -e #{Shellwords.escape(instrumented)} "$@" + } + ZSH + end + + # Exported in the pty shell so the server worker's marks land in our log + # (the worker inherits the client's environment). + def shell_setup(run_id:) + "export READY_MARKS=#{@marks_log.path} READY_RUN_ID=#{run_id}" + end + end + end +end diff --git a/spec/support/bench/invocation.rb b/spec/support/bench/invocation.rb new file mode 100644 index 0000000..1c6ec51 --- /dev/null +++ b/spec/support/bench/invocation.rb @@ -0,0 +1,16 @@ +module Ready + module Bench + ## + # One command line to benchmark, exactly as the user would type it: the + # executable and the arguments that make it do real work. + class Invocation < Data.define(:executable_name, :arguments) + def self.parse(words) + new(executable_name: words.first, arguments: words[1..]) + end + + def self.bare(executable_name) + new(executable_name:, arguments: []) + end + end + end +end diff --git a/spec/support/bench/mark_helper.rb b/spec/support/bench/mark_helper.rb new file mode 100644 index 0000000..4cdc592 --- /dev/null +++ b/spec/support/bench/mark_helper.rb @@ -0,0 +1,30 @@ +module Ready + module Bench + ## + # The Ruby an instrumented process runs to append one mark line to the + # marks log. Kept as literal source (never interpolated at generation + # time) because it executes inside the measured process, which knows its + # run id and log path only through the environment. One definition serves + # every injection site: the cold arm's prelude, the instrumented rubygems + # stub, and the hot arm's server-eval'd source. + module MarkHelper + DEFINITION = <<~'RUBY'.freeze + def ready_bench_mark(mark_name) + marks_log_path = ENV.fetch("READY_MARKS") + run_id = ENV.fetch("READY_RUN_ID") + instant = Process.clock_gettime(Process::CLOCK_REALTIME) + File.write(marks_log_path, "#{run_id} #{mark_name} #{instant}\n", mode: "a") + end + RUBY + + def self.definition + DEFINITION + end + + # A statement recording +mark_name+, for splicing in after .definition. + def self.record(mark_name) + %(ready_bench_mark("#{mark_name}")) + end + end + end +end diff --git a/spec/support/bench/marks_log.rb b/spec/support/bench/marks_log.rb new file mode 100644 index 0000000..798472e --- /dev/null +++ b/spec/support/bench/marks_log.rb @@ -0,0 +1,43 @@ +module Ready + module Bench + ## + # The shared append-only file every instrumented process writes marks + # into, one line per mark: " ". + # zsh's $EPOCHREALTIME and Ruby's Process::CLOCK_REALTIME read the same + # wall clock, so lines from either side subtract cleanly. + class MarksLog + attr_reader :path + + def initialize(path) + @path = Pathname(path) + end + + def run(id) + lines = lines_for(id) + Run.new(id:, mark_times: mark_times(lines), exit_status: exit_status(lines)) + end + + private + + def lines_for(id) + return [] unless path.exist? + + path.readlines.map(&:split).select { |line| line.length == 3 && line.first == id } + end + + # A malformed line (e.g. a mark whose shell left the timestamp empty) is + # skipped rather than crashing the run: the span that needed it simply + # goes unmeasured. The exit_status line is not a timestamp, so it is + # excluded here and read separately. + def mark_times(lines) + lines.reject { |line| line[1] == "exit_status" } + .to_h { |_run_id, mark_name, seconds| [mark_name.to_sym, Float(seconds)] } + end + + def exit_status(lines) + status = lines.find { |line| line[1] == "exit_status" } + status && Integer(status[2]) + end + end + end +end diff --git a/spec/support/bench/measurement.rb b/spec/support/bench/measurement.rb new file mode 100644 index 0000000..683907c --- /dev/null +++ b/spec/support/bench/measurement.rb @@ -0,0 +1,10 @@ +module Ready + module Bench + ## + # Everything one run yielded: the Waterfall of its marked spans, plus the + # wall clock (a Float of seconds, CLOCK_MONOTONIC) the pty driver observed + # from OUTSIDE the shell -- the independent cross-check on the in-shell + # marks. + Measurement = Data.define(:waterfall, :wall_clock_seconds) + end +end diff --git a/spec/support/bench/plot/stacked.rb b/spec/support/bench/plot/stacked.rb new file mode 100644 index 0000000..689fced --- /dev/null +++ b/spec/support/bench/plot/stacked.rb @@ -0,0 +1,53 @@ +module Ready + module Bench + module Plot + ## + # One bar per command on a shared scale: the leading segment is what you + # still pay with ready, the rest is what ready eliminates -- together, the + # cold total. + class Stacked + WIDTH = 40 + READY_CELL = "#".freeze + ELIMINATED_CELL = ".".freeze + + def initialize(comparisons) + @comparisons = comparisons + end + + def render + puts "full startup (ms) -- #{READY_CELL} ready, #{ELIMINATED_CELL} eliminated by ready" + @comparisons.each { render_bar(it) } + end + + private + + # Cells per millisecond, sized so the slowest measurement of either arm + # spans WIDTH -- so a regression (ready slower than cold) can't overflow. + def scale + @scale ||= WIDTH / @comparisons.flat_map { [it.cold_milliseconds, it.hot_milliseconds] }.max + end + + # The command column widens to the longest command so no label + # overflows and shoves the numbers out of line. + def command_width + @command_width ||= @comparisons.map { it.command.length }.max + end + + def render_bar(comparison) + row_format = "%-#{command_width}s %-#{WIDTH}s " \ + "%7.1f ready / %7.1f cold (%.1fx)" + puts format(row_format, + command: comparison.command, bar: bar_for(comparison), + ready: comparison.hot_milliseconds, cold: comparison.cold_milliseconds, + speedup: comparison.speedup) + end + + def bar_for(comparison) + ready_cells = [(comparison.hot_milliseconds * scale).round, 1].max + cold_cells = [(comparison.cold_milliseconds * scale).round, ready_cells].max + (READY_CELL * ready_cells) + (ELIMINATED_CELL * (cold_cells - ready_cells)) + end + end + end + end +end diff --git a/spec/support/bench/plot/youplot.rb b/spec/support/bench/plot/youplot.rb new file mode 100644 index 0000000..9155b65 --- /dev/null +++ b/spec/support/bench/plot/youplot.rb @@ -0,0 +1,46 @@ +require "rbconfig" + +module Ready + module Bench + module Plot + ## + # Delegates to the stock youplot barplot: cold vs ready. With a single + # command the command names the plot (title) and the bars are just + # "cold" / "ready"; with several, each bar carries its command so they + # stay distinct. + class Youplot + def initialize(comparisons) + @comparisons = comparisons + end + + def render + uplot = Gem.bin_path("youplot", "uplot") + IO.popen([RbConfig.ruby, uplot, "bar", "-d", ",", "-t", title], "w") do |pipe| + @comparisons.each { write_bars(pipe, it) } + end + end + + private + + def title + single? ? "full startup (ms): #{@comparisons.first.command}" : "full startup (ms): cold vs ready" + end + + def write_bars(pipe, comparison) + pipe.puts "#{label(comparison, "cold")},#{comparison.cold_milliseconds.round(1)}" + pipe.puts "#{label(comparison, "ready")},#{comparison.hot_milliseconds.round(1)}" + end + + # One command -> bare arm labels (the command is the title); several -> + # prefix each with its command so the bars are distinguishable. + def label(comparison, arm) + single? ? arm : "#{comparison.command} #{arm}" + end + + def single? + @comparisons.one? + end + end + end + end +end diff --git a/spec/support/bench/progress.rb b/spec/support/bench/progress.rb new file mode 100644 index 0000000..e86dd16 --- /dev/null +++ b/spec/support/bench/progress.rb @@ -0,0 +1,49 @@ +module Ready + module Bench + ## + # Narrates a run to stderr so a long, otherwise-silent benchmark never + # looks hung: it announces the one-time sandbox build, then the warmup and + # the measured rounds as separate phases (so the measured count matches + # what the user asked for), ticking once per round. Output goes to stderr, + # leaving stdout for the report itself. + class Progress + def initialize(command:, io: $stderr) + @command = command + @io = io + end + + def building + @io.print "#{@command}: building sandbox -- compiling stubs, booting the server (one-time setup)..." + @io.flush + end + + def warming_up(rounds) + start_phase("warming up (#{count(rounds)})") + end + + def measuring(rounds) + start_phase("measuring #{count(rounds)}") + end + + def tick + @io.print "." + @io.flush + end + + def done + @io.puts " done" + end + + private + + def start_phase(label) + @io.print "\n#{@command}: #{label} " + @io.flush + end + + def count(rounds) + "#{rounds} #{rounds == 1 ? "round" : "rounds"}" + end + end + end +end diff --git a/spec/support/bench/protocol.rb b/spec/support/bench/protocol.rb new file mode 100644 index 0000000..74029a0 --- /dev/null +++ b/spec/support/bench/protocol.rb @@ -0,0 +1,22 @@ +module Ready + module Bench + ## + # What one benchmark actually tested and how: the executable under test, + # the arguments that make it do real work, the gems the hot server + # preloads, and the round counts (measured rounds per arm, plus warmups + # that are recorded but excluded from every reported number). + class Protocol < Data.define(:executable_name, :arguments, :preload_gems, :rounds, :warmups) + # The standard target: ri rendering real documentation -- a genuine CLI + # workload, present on every machine. + def self.default + new(executable_name: "ri", arguments: ["TCPServer"], preload_gems: ["rdoc"], + rounds: 15, warmups: 3) + end + + # The exact command line the harness times, minus redirections. + def invocation + [executable_name, *arguments].join(" ") + end + end + end +end diff --git a/spec/support/bench/rbenv_shim.rb b/spec/support/bench/rbenv_shim.rb new file mode 100644 index 0000000..286c216 --- /dev/null +++ b/spec/support/bench/rbenv_shim.rb @@ -0,0 +1,20 @@ +module Ready + module Bench + ## + # The shim variant that runs the real `rbenv exec` chain, exactly as the + # user's PATH does -- its :launch span therefore carries the rbenv cost. + # It shares the tool's own name, so nothing downstream can tell it apart + # from the real shim. + class RbenvShim < Shim + def command_word + executable_name + end + + private + + def exec_line + %(exec rbenv exec "#{executable_name}" "$@") + end + end + end +end diff --git a/spec/support/bench/report.rb b/spec/support/bench/report.rb new file mode 100644 index 0000000..c30d1c7 --- /dev/null +++ b/spec/support/bench/report.rb @@ -0,0 +1,183 @@ +module Ready + module Bench + ## + # Renders the benchmark: a headline verdict (the caveat-less cold/hot times + # and the speedup), a preamble saying exactly what ran, a waterfall table + # with one statistic per column plus a per-layer delta, a note reconciling + # the table's cold with the rbenv shim, a cross-check of the in-shell + # numbers against the pty driver's independently observed wall clock, and + # -- when verbose -- a legend and the harness vocabulary. + class Report + ROW = "%-18s %12s %12s " \ + "%12s %12s %12s".freeze + LEGEND_ROW = "%-18s %-41s %-8s %s".freeze + TERM_ROW = "%-13s %s".freeze + # The fast arm is the product: display it as "ready", not "hot". + ARM_LABEL = { cold: "cold", hot: "ready" }.freeze + + TERMS = { + arm: "one side of the comparison -- cold boots a fresh process per run, " \ + "hot dispatches to the warm by-server", + round: "one interleaved pass timing both arms once (a cold run and a hot " \ + "run), order alternating per round to cancel drift", + run: "a single timed invocation inside an arm, identified . " \ + "(cold.3 is round 3's cold run)", + delta: "cold minus hot for that layer: what ready saves there " \ + "(negative = ready's own overhead)", + "cross-check": "the pty driver's outside wall clock against the in-shell " \ + "marks; the delta is driver overhead", + }.freeze + + def initialize(cold:, hot:, protocol:, rbenv_shim_overhead: nil, verbose: false) + @cold = cold + @hot = hot + @arms = [cold, hot] + @protocol = protocol + @rbenv_shim_overhead = rbenv_shim_overhead + @verbose = verbose + end + + def render + render_headline + render_slide_summary + render_preamble + render_table + render_rbenv_note + @arms.each { render_wall_clock_check(it) } + return unless @verbose + + render_legend + render_terms + end + + private + + # The final, caveat-less answer up top: what a real cold run costs, what + # ready's warm dispatch costs, and how many times faster that is. + def render_headline + cold = real_cold_full + hot = @hot.duration_of(:full) + puts "ready startup benchmark -- #{@protocol.invocation}" + puts + puts format(" cold %8.1f ms %s", cold:, note: cold_note) + puts format(" ready %8.1f ms warm dispatch", hot:) + puts format(" ready is %.1fx faster, saving %.1f ms per run", x: cold / hot, saved: cold - hot) + puts + end + + # What a real cold invocation costs: the measured cold full plus the + # rbenv shim it goes through (the table measures cold without it, to keep + # the per-layer marks clean). No rbenv probe -> the two are the same. + def real_cold_full + @cold.duration_of(:full) + (@rbenv_shim_overhead || 0) + end + + def cold_note + @rbenv_shim_overhead ? "fresh boot, through the rbenv shim" : "fresh boot" + end + + # The one-slide version of the waterfall: the cold path collapsed to the + # handful of layers, in plain words, with the real end-to-end total. + def render_slide_summary + puts SlideSummary.new(cold: @cold, rbenv_shim_overhead: @rbenv_shim_overhead).render + puts + end + + def render_preamble + puts "tool under test: #{@protocol.executable_name}, invoked as: #{@protocol.invocation}" + puts "cold arm: a fresh Ruby boot per run, through an instrumented copy of its rubygems stub" + puts "ready arm: #{ready_invocation} dispatching to a warm by-server " \ + "(#{@protocol.preload_gems.join(", ")} preloaded)" + puts "protocol: #{@protocol.rounds} rounds, each timing both arms once " \ + "(order alternating), after #{warmup_phrase}" + puts "choose the target: bin/bench [--readyfile PATH] [COMMAND ...] [-- COMMAND ...]" + puts "all durations in milliseconds" + puts + end + + def ready_invocation + ["ready_#{@protocol.executable_name}", *@protocol.arguments].join(" ") + end + + def warmup_phrase + @protocol.warmups == 1 ? "1 warmup round" : "#{@protocol.warmups} warmup rounds" + end + + def render_table + puts format(ROW, span: "span", cold_median: "cold median", cold_minimum: "cold minimum", + hot_median: "ready median", hot_minimum: "ready minimum", delta: "delta") + Span.table.each { render_row(it) } + end + + def render_row(span) + puts format(ROW, span: span.label, + cold_median: median_cell(@cold, span), cold_minimum: minimum_cell(@cold, span), + hot_median: median_cell(@hot, span), hot_minimum: minimum_cell(@hot, span), + delta: delta_cell(span)) + end + + def median_cell(arm, span) + samples = arm.samples_of(span.label) + return "-" if samples.empty? + + format("%.1f", Stats.median(samples)) + end + + def minimum_cell(arm, span) + samples = arm.samples_of(span.label) + return "-" if samples.empty? + + format("%.1f", samples.min) + end + + # cold - hot for the layer: positive where ready eliminates cold work, + # negative for the layers ready adds (its dispatch). "-" when neither arm + # measured the layer. + def delta_cell(span) + return "-" if @cold.samples_of(span.label).empty? && @hot.samples_of(span.label).empty? + + format("%+.1f", median_or_zero(@cold, span) - median_or_zero(@hot, span)) + end + + def median_or_zero(arm, span) + samples = arm.samples_of(span.label) + samples.empty? ? 0.0 : Stats.median(samples) + end + + def render_rbenv_note + return unless @rbenv_shim_overhead + + puts format("\ncold full in the table excludes the rbenv shim; a real cold run pays " \ + "+%.1f ms more (folded into the headline above)", + overhead: @rbenv_shim_overhead) + end + + def render_wall_clock_check(arm) + in_shell = arm.duration_of(:full) + observed = arm.median_wall_clock_milliseconds + puts format("pty cross-check %-5s in-shell %7.1fms " \ + "pty-observed %7.1fms (delta %.1fms driver overhead)", + arm: ARM_LABEL.fetch(arm.name, arm.name.to_s), in_shell:, observed:, + delta: observed - in_shell) + end + + def render_legend + puts "\nlegend" + puts format(LEGEND_ROW, span: "span", interval: "interval (opening mark -> closing mark)", + summary: "summary", description: "what it measures") + Span.table.each { render_legend_row(it) } + end + + def render_legend_row(span) + interval = "#{span.opening_mark} -> #{span.closing_mark}" + puts format(LEGEND_ROW, span: span.label, interval:, summary: span.summary, + description: span.description) + end + + def render_terms + puts "\nterms" + TERMS.each { |term, meaning| puts format(TERM_ROW, term:, meaning:) } + end + end + end +end diff --git a/spec/support/bench/results_log.rb b/spec/support/bench/results_log.rb new file mode 100644 index 0000000..0a4d5af --- /dev/null +++ b/spec/support/bench/results_log.rb @@ -0,0 +1,46 @@ +module Ready + module Bench + ## + # The shared file each `rake bench` run appends its headline numbers to, + # one line per arm: ",,". It bridges the + # separate rake processes and the plotting CLI, which reads the + # accumulated rows back as one Comparison per command. + class ResultsLog + ## + # One parsed line: which command, which arm, and its full startup time. + Row = Data.define(:command, :arm, :full_milliseconds) + + attr_reader :path + + def initialize(path) + @path = Pathname(path) + end + + def append(command:, arm:, full_milliseconds:) + path.write("#{command},#{arm},#{full_milliseconds}\n", mode: "a") + end + + def comparisons + rows.group_by(&:command).map { |command, command_rows| comparison_for(command, command_rows) } + end + + private + + def rows + path.readlines(chomp: true).map { parse_row(it) } + end + + def parse_row(line) + command, arm, full_milliseconds = line.split(",") + Row.new(command:, arm: arm.to_sym, full_milliseconds: Float(full_milliseconds)) + end + + def comparison_for(command, command_rows) + by_arm = command_rows.to_h { [it.arm, it.full_milliseconds] } + Comparison.new(command:, + cold_milliseconds: by_arm.fetch(:cold), + hot_milliseconds: by_arm.fetch(:hot)) + end + end + end +end diff --git a/spec/support/bench/round.rb b/spec/support/bench/round.rb new file mode 100644 index 0000000..73e2db0 --- /dev/null +++ b/spec/support/bench/round.rb @@ -0,0 +1,24 @@ +module Ready + module Bench + ## + # One interleaved pass of the benchmark: it times both arms once, in an + # order that alternates each round to cancel run-order drift. Warmup rounds + # run identically but are excluded from the reported numbers. A round owns + # the run ids its arms report under -- "cold.3" is round 3's cold run. + class Round < Data.define(:number, :warmup) + def warmup? + warmup + end + + # Cold-first on even rounds, ready-first on odd, so run-order drift + # cancels across the session. + def arm_order + number.even? ? %i[cold hot] : %i[hot cold] + end + + def run_id(arm) + "#{arm}.#{number}" + end + end + end +end diff --git a/spec/support/bench/rubygems_stub.rb b/spec/support/bench/rubygems_stub.rb new file mode 100644 index 0000000..001f2e8 --- /dev/null +++ b/spec/support/bench/rubygems_stub.rb @@ -0,0 +1,74 @@ +require "rbconfig" + +module Ready + module Bench + ## + # The rubygems stub a cold invocation runs after the shim -- resolved the + # way rbenv itself resolves it -- plus the mark surgery that makes a COPY + # of it measurable. The instrumented copy marks: + # + # rubygems_ready - after `Gem.use_gemdeps`, closing the :rubygems span + # bin_path_resolved - after `Gem.activate_bin_path`, separating + # dependency activation (:activation) from the tool + # itself (:tool_run) + # ruby_exit - at_exit, closing :tool_run + class RubygemsStub + # The two forms a standard stub uses to activate-and-run the tool; both + # capture (indent, arguments) so the rewrite can split the combined call. + ACTIVATION_CALLS = [ + /^(\s*)Gem\.activate_and_load_bin_path\((.*)\)/, + /^(\s*)load Gem\.activate_bin_path\((.*)\)/, + ].freeze + + def self.for_executable(executable_name, rbenv: true) + stub_path = resolve_stub_path(executable_name, rbenv:) + raise "could not resolve #{executable_name.inspect} to a rubygems stub" unless stub_path&.file? + + new(stub_path.read) + end + + # rbenv resolves an executable to the rubygems stub in the active Ruby's + # bindir, bypassing its shims. Without rbenv that same stub is just + # / -- exactly what `rbenv which` returns anyway. + def self.resolve_stub_path(executable_name, rbenv:) + return Pathname(RbConfig::CONFIG["bindir"]) / executable_name unless rbenv + + resolved = `rbenv which #{executable_name}`.strip + Pathname(resolved) unless resolved.empty? + end + private_class_method :resolve_stub_path + + def initialize(source) + @source = source + end + + def instrumented_source + @source + .sub(/\A(#!.*\n)?/) { "#{Regexp.last_match(1)}#{prologue}" } + .sub(/(Gem\.use_gemdeps.*\n)/) { "#{Regexp.last_match(1)}#{MarkHelper.record(:rubygems_ready)}\n" } + .then { split_activation_calls(it) } + end + + private + + # The mark helper plus the exit mark, inserted after any shebang line. + def prologue + "#{MarkHelper.definition}at_exit { #{MarkHelper.record(:ruby_exit)} }\n" + end + + def split_activation_calls(source) + ACTIVATION_CALLS.reduce(source) do |rewritten, call| + rewritten.gsub(call) { split_activation(Regexp.last_match(1), Regexp.last_match(2)) } + end + end + + # One combined activate-and-load call becomes activate, mark, load -- + # the mark between them is what separates :activation from :tool_run. + def split_activation(indent, arguments) + "#{indent}activated_bin_path = Gem.activate_bin_path(#{arguments}); " \ + "#{MarkHelper.record(:bin_path_resolved)}; " \ + "load activated_bin_path" + end + end + end +end diff --git a/spec/support/bench/run.rb b/spec/support/bench/run.rb new file mode 100644 index 0000000..4082a0d --- /dev/null +++ b/spec/support/bench/run.rb @@ -0,0 +1,36 @@ +module Ready + module Bench + ## + # One timed invocation of the tool under one arm, identified by its run id + # ("cold.3" is round 3's cold invocation). A run holds the wall-clock + # instant of every mark its instrumented layers recorded; Spans measure + # themselves by asking the run for the time between two marks. + class Run + attr_reader :id, :exit_status + + def initialize(id:, mark_times:, exit_status: nil) + @id = id + @mark_times = mark_times.dup.freeze + @exit_status = exit_status + end + + # True only when the invocation ran to completion and exited 0. A missing + # status (the process was killed before recording one) counts as failure. + def succeeded? + @exit_status&.zero? || false + end + + def failure_reason + @exit_status.nil? ? "recorded no exit status (killed before finishing)" : "exited #{@exit_status}" + end + + def recorded?(mark_name) + @mark_times.key?(mark_name) + end + + def milliseconds_between(opening_mark, closing_mark) + (@mark_times.fetch(closing_mark) - @mark_times.fetch(opening_mark)) * 1000.0 + end + end + end +end diff --git a/spec/support/bench/runner.rb b/spec/support/bench/runner.rb new file mode 100644 index 0000000..3c5d6bc --- /dev/null +++ b/spec/support/bench/runner.rb @@ -0,0 +1,224 @@ +require "bundler" +require "English" +require "fileutils" +require "tmpdir" +require "rbconfig" + +module Ready + module Bench + ## + # Orchestrates the interleaved cold/hot layer benchmark. Each round runs + # both arms (order alternating to cancel drift), every run's marks land in + # one MarksLog, and each arm accumulates its runs' waterfalls in an + # ArmResult. When rbenv is present, each round also probes the real rbenv + # shim to derive the shim overhead the hot path eliminates. + class Runner + attr_reader :rbenv_shim_overhead + + def initialize(protocol: Protocol.default, rbenv: Bench.rbenv_available?, progress: nil) + @protocol = protocol + @rbenv = rbenv + @progress = progress || Progress.new(command: protocol.invocation) + @harness_log = HarnessLog.new(invocation: protocol.invocation) + @cold_result = ArmResult.new(name: :cold, warmups: protocol.warmups) + @hot_result = ArmResult.new(name: :hot, warmups: protocol.warmups) + @rbenv_launch_samples = [] + end + + def executable_name + @protocol.executable_name + end + + # The command as typed ("ri TCPServer") -- the plot's per-command + # identity, so the same executable on two inputs stays two bars. + def invocation + @protocol.invocation + end + + def call + @harness_log.open! + @progress.building + build + run_schedule + derive_rbenv_shim_overhead + self + ensure + teardown + end + + def cold_summary + @cold_result.summary + end + + def hot_summary + @hot_result.summary + end + + def report(verbose: false) + Report.new(cold: @cold_result, hot: @hot_result, protocol: @protocol, + rbenv_shim_overhead:, verbose:) + end + + def render(verbose: false) + report(verbose:).render + end + + def teardown + @sandbox&.teardown + FileUtils.rm_rf(@tmp) if @tmp + end + + private + + # Warmup rounds run first (and are dropped from the results by ArmResult); + # the measured rounds the user asked for follow. Reporting them as + # separate phases keeps `--rounds 4` from showing as 7. + def run_schedule + warmup_rounds, measured_rounds = schedule.partition(&:warmup?) + @progress.warming_up(warmup_rounds.size) + warmup_rounds.each { run_and_tick(it) } + @progress.measuring(measured_rounds.size) + measured_rounds.each { run_and_tick(it) } + @progress.done + end + + def schedule + total = @protocol.rounds + @protocol.warmups + (1..total).map { |number| Round.new(number:, warmup: number <= @protocol.warmups) } + end + + def run_and_tick(round) + run_round(round) + @progress.tick + end + + def build + @tmp = Pathname(Dir.mktmpdir("bench")) + @marks_log = MarksLog.new(@tmp / "marks") + @sandbox = Ready::Sandbox.build(executables: ["rake"], gems: @protocol.preload_gems) + @cold_arm = ColdArm.new(executable_name:, workdir: @tmp / "cold", marks_log: @marks_log, rbenv: @rbenv) + @cold_arm.instrument! + @hot_arm = HotArm.new(executable_name:, rendered_source: render_production_source, + sandbox: @sandbox, marks_log: @marks_log) + (@tmp / "stub.zsh").write(@hot_arm.stub_function) + end + + # Renders the hot stub source the way production `ready gem ` + # does: by NAME, so Executable resolves via Gem.bin_path and reads the + # real file whether the binstub lives in bin/ or exe/. Runs in a + # scrubbed, unbundled subprocess because the bench itself runs under + # this project's bundle, where the target (e.g. ronin) is not a bundled + # gem. + def render_production_source + script = "require \"ready\"; print Ready::Executable.new(#{executable_name.inspect}).render" + output = Bundler.with_unbundled_env do + IO.popen({ "GEM_HOME" => nil, "GEM_PATH" => nil }, + [RbConfig.ruby, "-I", (Ready.root / "lib").to_s, "-e", script], &:read) + end + raise "cannot render #{executable_name}: #{output}" unless $CHILD_STATUS.success? && !output.empty? + + output + end + + def run_round(round) + round.arm_order.each { |arm| run_arm(arm, round) } + probe_rbenv_shim(round.run_id(:rbenv)) if @rbenv + end + + def run_arm(arm, round) + arm == :cold ? run_cold(round.run_id(:cold)) : run_hot(round.run_id(:hot)) + end + + def run_cold(run_id) + measurement = cold_invocation(run_id, shim: @cold_arm.direct_shim) + @cold_result.record(measurement) + end + + # The rbenv variant exists only to isolate the real shim's cost: its + # :launch also carries the `rbenv exec` chain, so min(rbenv launch) + # minus the direct arm's launch floor is the shim overhead. + def probe_rbenv_shim(run_id) + measurement = cold_invocation(run_id, shim: @cold_arm.rbenv_shim) + launch = measurement.waterfall.duration_of(:launch) + @rbenv_launch_samples << launch if launch + end + + def cold_invocation(run_id, shim:) + Bundler.with_unbundled_env do + drop_stale_gem_home + shell = Ready::PtyShell.new(@cold_arm.environment_for(run_id:)) + shell.run("source #{profiler_path}") + harness_run = shell.run(harness_command(run_id, shim.command_word)) + measurement_for(run_id, harness_run) + ensure + shell&.close + end + end + + # Dispatches from a bundler-free shell (a real user terminal is + # unbundled) so the by client isn't slowed by a bundler/setup require + # inherited through RUBYOPT. + def run_hot(run_id) + Bundler.with_unbundled_env do + drop_stale_gem_home + shell = Ready::PtyShell.new(@sandbox.shell_env) + shell.run(hot_setup(run_id)) + harness_run = shell.run(harness_command(run_id, "ready_#{executable_name}")) + measurement = measurement_for(run_id, harness_run) + @hot_result.record(measurement) + ensure + shell&.close + end + end + + # The measured command, teed through the harness log so a tool that fails + # silently leaves evidence instead of being timed as a fast "success". + def harness_command(run_id, command_word) + workload = [command_word, *@protocol.arguments].join(" ") + @harness_log.tee(run_id, "bench_harness #{run_id} -- #{workload}") + end + + def hot_setup(run_id) + ["source #{@sandbox.plugin_path}", + "source #{profiler_path}", + @hot_arm.shell_setup(run_id:), + "source #{@tmp / "stub.zsh"}"].join("; ") + end + + def profiler_path + Ready.root / "bench" / "prof.zsh" + end + + # Pairs the run's in-shell waterfall with the wall clock the pty driver + # observed for the harness command. + def measurement_for(run_id, harness_run) + run = @marks_log.run(run_id) + verify_succeeded!(run) + Measurement.new(waterfall: Waterfall.of(run), wall_clock_seconds: harness_run.wall_clock_seconds) + end + + # A run whose tool crashed (non-zero exit) or vanished (no status) aborts + # the whole benchmark instead of being timed as a fast "success". The + # failing command's output is in log/bench.log. + def verify_succeeded!(run) + return if run.succeeded? + + raise "benchmark run #{run.id} #{run.failure_reason} -- see #{@harness_log.path}" + end + + def derive_rbenv_shim_overhead + return unless @rbenv + + samples = @rbenv_launch_samples.drop(@protocol.warmups) + direct_launch = cold_summary.duration_of(:launch) + return if samples.empty? || direct_launch.nil? + + @rbenv_shim_overhead = samples.min - direct_launch + end + + def drop_stale_gem_home + %w[GEM_HOME GEM_PATH].each { ENV.delete(it) if ENV[it] && !File.directory?(ENV[it]) } + end + end + end +end diff --git a/spec/support/bench/shim.rb b/spec/support/bench/shim.rb new file mode 100644 index 0000000..d676b68 --- /dev/null +++ b/spec/support/bench/shim.rb @@ -0,0 +1,71 @@ +module Ready + module Bench + ## + # A faithful, instrumented stand-in for the rbenv shim a cold invocation + # hits first. Every shim marks command_start (closing the :shell span), + # arms the Ruby-side prelude via RUBYOPT, boots with --disable-gems, then + # execs its variant's target. Subclasses answer what the shim is called + # on PATH (#command_word) and what it execs (#exec_line). + # + # Why --disable-gems: + # + # default boot: MRI eagerly requires rubygems at interpreter startup, + # BEFORE any RUBYOPT -r runs -- so ~45ms hides inside + # :launch and the stub's own `require "rubygems"` becomes + # a no-op + # --disable-gems: that implicit require is skipped, so the SAME require + # runs at the stub's explicit call site instead -- same + # code, same cost, now inside the markable :rubygems span + # + # (This is the interpreter's default `gems` feature, not Kernel#autoload.) + class Shim + def initialize(executable_name:, workdir:, prelude_path:, **options) + @executable_name = executable_name + @workdir = Pathname(workdir) + @prelude_path = prelude_path + post_initialize(**options) + end + + # The word a shell types to invoke this shim (its filename on PATH). + def command_word + raise NotImplementedError, "#{self.class} must name its command word" + end + + def write! + path = @workdir / command_word + path.write(script) + path.chmod(0o755) + end + + private + + attr_reader :executable_name, :prelude_path + + # Hook for a variant's extra construction state; overriding it never + # requires calling super. + def post_initialize(**) + nil + end + + def exec_line + raise NotImplementedError, "#{self.class} must supply its exec line" + end + + # A zsh script, not bash: it marks command_start with $EPOCHREALTIME, + # which zsh always provides via zsh/datetime. macOS still ships bash 3.2, + # where $EPOCHREALTIME is empty -- that produced a timestamp-less mark and + # crashed the parser. zsh is the shell the whole harness already requires. + def script + <<~SH + #!/usr/bin/env zsh + set -e + zmodload zsh/datetime + print -r -- "$READY_RUN_ID command_start $EPOCHREALTIME" >> "$READY_MARKS" + export RUBYOPT="--disable-gems -r#{prelude_path}${RUBYOPT:+ $RUBYOPT}" + export RBENV_ROOT="$HOME/.rbenv" + #{exec_line} + SH + end + end + end +end diff --git a/spec/support/bench/slide_summary.rb b/spec/support/bench/slide_summary.rb new file mode 100644 index 0000000..5e84ab2 --- /dev/null +++ b/spec/support/bench/slide_summary.rb @@ -0,0 +1,105 @@ +module Ready + module Bench + ## + # The cold-startup breakdown condensed to the few layers an audience can + # hold at once -- the version meant for a slide. Each line pairs a plain + # label with the cold arm's floor (minimum) milliseconds for the matching + # span. The Ruby VM boot is its own line, kept separate from "rubygems": both + # arms pay the boot (ready's by client boots a VM too), but only the cold arm + # loads rubygems -- folding them would falsely imply ready eliminates the + # boot. Using the floor -- the same statistic the headline's + # cold total uses -- keeps the layers summing to that total rather than + # overshooting it (median layers can exceed the whole once a tool drags in a + # big dependency graph); the tiny reap and the slack between the layer + # floors and the full floor fall under the tilde on the total. + class SlideSummary + TITLE = "where the cold startup goes (slide summary)".freeze + + # One labeled row, its raw milliseconds and the prefix its value prints + # with: "~" marks the total as approximate, layers carry none. + Line = Data.define(:label, :milliseconds, :prefix) do + def rounded_milliseconds + milliseconds.round + end + + def value + "#{prefix}#{rounded_milliseconds}" + end + end + + # A friendly slide label over the cold span(s) whose durations it sums. + Layer = Data.define(:label, :spans) + + LAYERS = [ + Layer.new(label: "shell", spans: [:shell]), + Layer.new(label: "ruby vm boot", spans: [:launch]), + Layer.new(label: "rubygems", spans: [:rubygems]), + Layer.new(label: "activate deps", spans: [:activation]), + Layer.new(label: "the tool", spans: [:tool_run]), + ].freeze + + def initialize(cold:, rbenv_shim_overhead: nil) + @cold = cold + @rbenv_shim_overhead = rbenv_shim_overhead + end + + # The layer lines in the order a cold invocation pays them, with the + # rbenv shim slotted in after the shell when a real shim was measured. + def lines + layers = LAYERS.map { layer_line(it) } + shim = rbenv_line + layers.insert(1, shim) if shim + layers + end + + def total_line + Line.new(label: "total", milliseconds: total_milliseconds, prefix: "~") + end + + def render + [TITLE, *displayed_lines.map { row(it) }].join("\n") + end + + private + + def displayed_lines + @displayed_lines ||= [*lines, total_line] + end + + def layer_line(layer) + milliseconds = layer.spans.sum { minimum_of(it) } + Line.new(label: layer.label, milliseconds:, prefix: "") + end + + # A span's structural floor: its minimum across measured runs. The slide + # sums floors (not medians) so the layers add up to the min-based cold + # total the headline reports instead of overshooting it. + def minimum_of(span) + samples = @cold.samples_of(span) + samples.empty? ? 0.0 : samples.min + end + + def rbenv_line + return nil unless @rbenv_shim_overhead + + Line.new(label: "rbenv shim", milliseconds: @rbenv_shim_overhead, prefix: "") + end + + def total_milliseconds + @cold.duration_of(:full) + (@rbenv_shim_overhead || 0.0) + end + + def row(line) + " #{line.label.ljust(label_width)} #{line.value.rjust(value_width)} ms" + end + + def label_width + @label_width ||= displayed_lines.map { it.label.length }.max + end + + def value_width + @value_width ||= displayed_lines.map { it.value.length }.max + end + end + end +end diff --git a/spec/support/bench/span.rb b/spec/support/bench/span.rb new file mode 100644 index 0000000..946c762 --- /dev/null +++ b/spec/support/bench/span.rb @@ -0,0 +1,65 @@ +module Ready + module Bench + ## + # A named interval between two marks of a Run -- the unit every waterfall + # row is built from. Each span declares the statistic that collapses many + # runs' samples into one representative number (:minimum for + # process-creation spans, where scheduler jitter only ever adds time, so + # the floor is the structural cost; :median for in-process spans, where + # the typical case is the honest number) and a description the report's + # legend prints. + class Span < Data.define(:label, :opening_mark, :closing_mark, :summary, :description) + TABLE = [ + new(label: :shell, opening_mark: :harness_start, closing_mark: :command_start, + summary: :minimum, + description: "the shell starting the command -- cold fork/execs the shim found on " \ + "PATH, ready calls the ready_ function (no fork, no exec)"), + new(label: :launch, opening_mark: :command_start, closing_mark: :ruby_up, + summary: :minimum, description: "(cold only) Ruby interpreter boot, rubygems disabled"), + new(label: :rubygems, opening_mark: :ruby_up, closing_mark: :rubygems_ready, + summary: :median, description: "(cold only) the stub's require of rubygems plus Gem.use_gemdeps"), + new(label: :activation, opening_mark: :rubygems_ready, closing_mark: :bin_path_resolved, + summary: :median, description: "(cold only) resolving and activating the tool's gem dependencies"), + new(label: :tool_run, opening_mark: :bin_path_resolved, closing_mark: :ruby_exit, + summary: :median, description: "(cold only) loading and executing the tool itself"), + new(label: :reap, opening_mark: :ruby_exit, closing_mark: :harness_end, + summary: :median, description: "(cold only) interpreter exit and process reap, back to the shell"), + new(label: :dispatch_overhead, opening_mark: :command_start, closing_mark: :server_entry, + summary: :minimum, + description: "(ready only) by client boot (a real Ruby process, the floor), socket " \ + "round-trip, server fork"), + new(label: :server_tool_run, opening_mark: :server_entry, closing_mark: :harness_end, + summary: :median, description: "(ready only) the preloaded tool executing inside the warm server"), + new(label: :full, opening_mark: :harness_start, closing_mark: :harness_end, + summary: :minimum, description: "everything between the harness clock reads; what a user feels"), + ].freeze + + def self.table + TABLE + end + + # Milliseconds between this span's marks, or nil when the run did not + # record them both (a hot run has no shim marks, a cold run no server + # marks). + def measure(run) + return nil unless measured_by?(run) + + run.milliseconds_between(opening_mark, closing_mark) + end + + def measured_by?(run) + run.recorded?(opening_mark) && run.recorded?(closing_mark) + end + + # Collapses many runs' samples of this span into one representative + # number, using the statistic the span declared for itself. An unknown + # statistic raises rather than silently falling back. + def summarize(samples) + case summary + in :minimum then samples.min + in :median then Stats.median(samples) + end + end + end + end +end diff --git a/spec/support/bench/stats.rb b/spec/support/bench/stats.rb new file mode 100644 index 0000000..b7e256d --- /dev/null +++ b/spec/support/bench/stats.rb @@ -0,0 +1,13 @@ +module Ready + module Bench + ## + # Median helper over span samples. + module Stats + def self.median(values) + sorted = values.sort + middle = sorted.size / 2 + sorted.size.odd? ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2.0 + end + end + end +end diff --git a/spec/support/bench/waterfall.rb b/spec/support/bench/waterfall.rb new file mode 100644 index 0000000..d5845df --- /dev/null +++ b/spec/support/bench/waterfall.rb @@ -0,0 +1,38 @@ +module Ready + module Bench + ## + # The measured spans of one run, as span label to milliseconds; the + # summarizing constructor instead collapses many runs' waterfalls into one + # using each span's own summary statistic. Only spans whose marks were + # actually recorded appear: a hot run has no :launch row, a cold run no + # :dispatch_overhead row. + class Waterfall + def self.of(run) + Span.table + .select { it.measured_by?(run) } + .to_h { [it.label, it.measure(run)] } + .then { new(it) } + end + + def self.summarizing(waterfalls) + Span.table + .map { |span| [span, waterfalls.filter_map { it.duration_of(span.label) }] } + .reject { |_span, samples| samples.empty? } + .to_h { |span, samples| [span.label, span.summarize(samples)] } + .then { new(it) } + end + + def initialize(durations) + @durations = durations.dup.freeze + end + + def duration_of(label) + @durations[label] + end + + def measured?(label) + @durations.key?(label) + end + end + end +end diff --git a/spec/support/pty_shell.rb b/spec/support/pty_shell.rb new file mode 100644 index 0000000..154f305 --- /dev/null +++ b/spec/support/pty_shell.rb @@ -0,0 +1,88 @@ +require "pty" +require "expect" +require "shellwords" + +module Ready + ## + # Drives a real interactive zsh under a pty. Commands are sent as keystrokes; + # completion is detected by a marker printed after the command, using zsh's + # `''` quote-splitting so the keystroke echo can never match the marker output. + class PtyShell + ## + # What one #run produced: the output captured up to the completion marker, + # and the seconds (a Float, read from CLOCK_MONOTONIC) the driver observed + # from outside the shell between sending the command and seeing the marker. + Result = Data.define(:output, :wall_clock_seconds) + + PROMPT = "@@P> ".freeze + # IO#expect's timeout is the total seconds to wait for the pattern, so it + # bounds a hung shell on its own -- no outside watchdog needed. Generous: + # benchmarked commands finish in well under a second, so this only trips on + # a genuinely stuck shell (and is roomy enough not to false-alarm on a + # loaded CI box). + EXPECT_TIMEOUT = 30 + + def initialize(env = {}) + assignments = env.map { |k, v| "#{k}=#{Shellwords.escape(v.to_s)}" }.join(" ") + command = ["env", assignments, "zsh", "-f", "-i"].reject(&:empty?).join(" ") + @out, @in, @pid = PTY.spawn(command) + @seq = 0 + send_line("PS1='@@''P> '") + expect!(PROMPT) + rescue StandardError + close + raise + end + + ## + # Runs +cmd+ to completion and returns a Result. + def run(cmd) + @seq += 1 + marker = "DONE#{@seq}" + started_at = monotonic_clock + send_line("#{cmd}; print #{marker[0, 2]}''#{marker[2..]}") + output = expect!(marker) + Result.new(output:, wall_clock_seconds: monotonic_clock - started_at) + end + + # TERM the shell's whole process group (the '-' prefix signals the group, + # reaching forked children so none are orphaned), then hand the shell to + # Process.detach, which reaps it in a background thread -- never blocking on + # a wait, never leaving a zombie. + def close + return unless @pid + + terminate_group + Process.detach(@pid) + end + + private + + def terminate_group + Process.kill("-TERM", Process.getpgid(@pid)) + rescue Errno::ESRCH, Errno::EPERM, Errno::ECHILD + nil + end + + def send_line(line) + @in.puts(line) + end + + def monotonic_clock + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def expect!(pattern) + result = @out.expect(pattern, EXPECT_TIMEOUT) + raise "no #{pattern.inspect} within #{EXPECT_TIMEOUT}s; pty tail: #{tail.inspect}" unless result + + result.first + rescue Errno::EIO + raise "pty closed waiting for #{pattern.inspect}; pty tail: #{tail.inspect}" + end + + def tail + @out.instance_variable_get(:@unusedBuf).to_s[-400..] + end + end +end diff --git a/spec/support/sandbox.rb b/spec/support/sandbox.rb new file mode 100644 index 0000000..8ee7a92 --- /dev/null +++ b/spec/support/sandbox.rb @@ -0,0 +1,128 @@ +require "tmpdir" +require "fileutils" +require "shellwords" +require "bundler" + +module Ready + ## + # Builds an isolated `ready` runtime in a temp prefix: writes a readyfile, + # runs `ready up` once (compile stubs + boot by-server), exposes the env a pty + # needs to attach, and tears it all down (stop server + remove tree). + class Sandbox + attr_reader :prefix, :readyfile, :sock_path + + PLUGIN_PATH = Ready.root / "zsh" / "ready" / "ready.plugin.zsh" + EXE_READY = Ready.root / "exe" / "ready" + + # Tools run through the harness must never page: a pager grabs the pty and + # can leave a suspended child that wedges shell teardown. Neutralize the + # common pagers so every benched tool runs to completion non-interactively. + NON_INTERACTIVE_ENV = { "PAGER" => "cat", "RI_PAGER" => "cat", "GIT_PAGER" => "cat" }.freeze + + def self.build(executables:, gems: []) + new(executables:, gems:).tap(&:up) + end + + def initialize(executables:, gems: []) + @executables = executables + @gems = gems + @prefix = Pathname(Dir.mktmpdir("ready-e2e")) + @readyfile = @prefix / ".readyfile" + @sock_path = @prefix / "ready.sock" + end + + # Env a pty must set so the plugin attaches to THIS sandbox's live server. + # Carries the no-pager env, which build_env inherits, so the by-server + # (and its forked hot-arm workers) never launch a pager. + def shell_env + { + "READY_PREFIX" => prefix.to_s, + "READY_SOCK_PATH" => sock_path.to_s, + "READY_LOG_PATH" => (prefix / "ready.log").to_s, + "READY_DEBUG" => "0", + }.merge(NON_INTERACTIVE_ENV) + end + + def plugin_path + PLUGIN_PATH + end + + def up + FileUtils.mkdir_p(prefix / "builds") + write_readyfile + run_up + assert_built! + self + end + + def teardown + stop_server + FileUtils.rm_rf(prefix) + end + + private + + def write_readyfile + lines = [] + lines += ["gems:", *@gems.map { |g| " - #{g}" }] unless @gems.empty? + lines += ["executables:", *@executables.map { |e| " - #{e}" }] + readyfile.write("#{lines.join("\n")}\n") + end + + def run_up + log = prefix / "up.log" + ok = Bundler.with_unbundled_env do + system(build_env, RbConfig.ruby, EXE_READY.to_s, "up", + chdir: Ready.root.to_s, out: log.to_s, err: %i[child out]) + end + @build_output = log.read + raise "ready up failed:\n#{@build_output}" unless ok + end + + # Runs inside with_unbundled_env, where ENV holds the pre-bundler values. A + # removed version manager (rvm) can leave GEM_HOME/GEM_PATH pointing at a + # deleted directory; propagating it breaks gem resolution in the build. Drop + # such stale vars so exe/ready's bundler/setup resolves from the project + # bundle (mirrors the working `unset GEM_HOME ...` recipe). + def build_env + env = shell_env.merge( + "READY_READYFILE" => readyfile.to_s, + "RUBY_HOME" => nil, + "MY_RUBY_HOME" => nil, + ) + gem_home = ENV.fetch("GEM_HOME", nil) + if gem_home && !File.directory?(gem_home) + env["GEM_HOME"] = nil + env["GEM_PATH"] = nil + end + env + end + + def assert_built! + raise "no live socket at #{sock_path}\n#{@build_output}" unless sock_path.socket? + raise "no builds.zwc in #{prefix}\n#{@build_output}" unless (prefix / "builds.zwc").file? + end + + # Stop the by-server daemon and every worker it forked, without `by-server + # stop` (which can itself block). The daemon has no pidfile, but its argv + # carries the unique temp prefix, so pgrep finds it; TERMing its whole + # process group reaps the workers too -- they setproctitle to the tool name + # and so are invisible to a prefix search, but they share the daemon's + # group, and the '-' prefix signals the group so none are orphaned. + def stop_server + daemon_pids.each { |pid| terminate_process_group(pid) } + end + + def daemon_pids + `pgrep -f #{Shellwords.escape(prefix.to_s)}`.split.map(&:to_i).reject { |pid| pid == Process.pid } + end + + def terminate_process_group(pid) + group = Process.getpgid(pid) + # Never signal our own group; fall back to the lone pid if it shares ours. + group == Process.getpgrp ? Process.kill("TERM", pid) : Process.kill("-TERM", group) + rescue Errno::ESRCH, Errno::EPERM + nil + end + end +end diff --git a/zsh/ready/functions/ready/__ready_debug b/zsh/ready/functions/ready/__ready_debug index db555bf..4feffad 100644 --- a/zsh/ready/functions/ready/__ready_debug +++ b/zsh/ready/functions/ready/__ready_debug @@ -3,21 +3,20 @@ # emulate -L zsh -setopt errreturn warncreateglobal warnnestedvar +setopt extendedglob pipefail errreturn warncreateglobal warnnestedvar zmodload zsh/datetime autoload -Uz __ready_datetime __ready_print -local temp local level local configured_level=${READY_LOGLEVEL:-info} local -a rest local -A mapping=( - [debug]=yellow - [info]=cyan - [warn]=magenta - [error]=red + debug yellow + info cyan + warn magenta + error red ) if (( $+mapping[$1] )); then @@ -28,7 +27,7 @@ else rest=("${(@)@}") fi -if [[ -z "${(j::)rest}" ]]; then +if [[ -z ${(j::)rest} ]]; then print -u2 "must enter a debug message" return 1 fi @@ -39,14 +38,14 @@ fi local -A rank=(debug 0 info 1 warn 2 error 3) -if (( ${rank[$level]:-1} <= ${rank[$configured_level]:-1} )); then +if (( ${rank[$level]:-1} >= ${rank[$configured_level]:-1} )); then + local REPLY __ready_datetime $EPOCHREALTIME -{ - if [[ $READY_LOG_PATH = /dev/stdin ]] || [[ $READY_LOG_PATH = /dev/stderr ]] || [[ $READY_LOG_PATH = /dev/tty ]] && [[ -t 1 ]] - then - __ready_print -c ${mapping[$level]} "[%B${REPLY}%b]" "[%B${(U)level}%b]" $rest - else - __ready_print "[${REPLY}]" "[${(U)level}]" $rest - fi -} >> $READY_LOG_PATH + { + if [[ $READY_LOG_PATH = /dev/(stdin|stderr|tty) && -t 1 ]]; then + __ready_print -c $mapping[$level] "[%B${REPLY}%b]" "[%B${(U)level}%b]" $rest + else + __ready_print "[${REPLY}]" "[${(U)level}]" $rest + fi + } >> $READY_LOG_PATH fi diff --git a/zsh/ready/functions/readyinit b/zsh/ready/functions/readyinit index dc836f9..c90eea0 100644 --- a/zsh/ready/functions/readyinit +++ b/zsh/ready/functions/readyinit @@ -3,7 +3,9 @@ emulate -L zsh -setopt extendedglob errreturn warncreateglobal warnnestedvar +setopt extendedglob warncreateglobal warnnestedvar localtraps + +trap 'local rc=$?; local str="$0: Something went wrong initialzing ready"; __ready_debug error $str; return $rc' ZERR autoload -Uz __ready_debug autoload -Uz compdef @@ -11,7 +13,8 @@ autoload -Uz compdef local fn local compiledpath=$READY_PREFIX/builds.zwc local possible_basename -local possible_declared_completion_fn +local possible_declared_completion_fn +local -a noncompletions compiledpath=${compiledpath:P} @@ -20,11 +23,34 @@ if ! [[ -f $compiledpath ]]; then return 1 fi -local -a noncompletions=(${${${${${(f@)"$(builtin zcompile -t $compiledpath)"}[2,-1]}:t}:#_*}:#ready}) +# Not exposed publicly yet, but this decides during the `autoload -Uz -w` of the zwc wordcode file whether or not to add the +X flag. +# If enabled, pay a (small) penalty on `ready init`, for consistent performance on the first command call +# If disabled, `ready init` is faster, but lazy load the source from the zwc on the first command call +typeset -gx READY_ZSH_EAGER_LOAD=1 + +# Point the `by` client at the ready server's socket. Gem stubs set BY_SOCKET +# inline per call, but the bare `by`/`ready_by` command relies on the env; the +# by client otherwise defaults to ~/.by_socket and can't reach the server. +typeset -gx BY_SOCKET=$READY_SOCK_PATH + +noncompletions=(${${${${${(f@)"$(builtin zcompile -t $compiledpath)"}[2,-1]}:t}:#_*}:#ready}) +__ready_debug debug "Found the following functions (non-completion)" +__ready_debug debug "${(F)noncompletions}" + +__ready_debug debug "Compiled path is $compiledpath. Adding to fpath" fpath+=($compiledpath) -# eager load everything in the compiled binary -builtin autoload -w +X -Uz $compiledpath +__ready_debug debug "Now running zcompile..." +# this loads the entire ready.zwc package in one call, and creates autoload stubs +# for each of the ready_* functions. They will be fully loaded upon being called for the first +# time. +if (( READY_ZSH_EAGER_LOAD == 1 )); then + builtin autoload -w +X -Uz $compiledpath +else + builtin autoload -w -Uz $compiledpath +fi + +__ready_debug debug "Zcompile done" for fn in $noncompletions; do possible_basename=${fn#ready_} diff --git a/zsh/ready/functions/readyup b/zsh/ready/functions/readyup index 0efd0b9..afb3d33 100644 --- a/zsh/ready/functions/readyup +++ b/zsh/ready/functions/readyup @@ -55,7 +55,7 @@ tounalias=(${${tounfun:t}/#*ready_}) rc=$? if (( rc != 0 )); then - string="Encountered error(%B$rc%b) while running compilation task. Cannot continue." + string="Encountered error(%B$rc%b) while running compilation task. Cannot continue. Run %Bready clobber%b to reset the build, then open a new shell to rebuild." __ready_print -c red $string __ready_debug error $string return $rc