Add end-to-end rspec harness + benchmark analysis library

.github/workflows/ci.yml

@@ -2,9 +2,10 @@ name: CI
 
 on:
   push:
-    branches: [main, master]
+    branches: [main, master, e2e-rspec]
   pull_request:
     branches: [main, master]
+  workflow_dispatch:
 
 permissions:
   contents: read
@@ -26,3 +27,72 @@ jobs:
       run: bundle exec rake spec
     - name: Run RuboCop
       run: bundle exec rake rubocop
+
+  e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ['3.4.7', '4.0.1']
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install zsh
+      run: sudo apt-get update && sudo apt-get install -y zsh
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby-version }}
+        bundler-cache: true
+    # The benchmark deliberately runs two scrubbed, unbundled subprocesses that
+    # resolve gems the way a real (non-bundler) install does: `ready up` spawns
+    # by-server, and render_production_source does `require "ready"` (which
+    # needs zeitwerk). With bundler-cache's path install those gems live only in
+    # the bundle, so the scrubbed processes can't see them. Put ready's runtime
+    # deps on the base PATH at their locked versions -- where rbenv has them
+    # locally and on the macos-e2e job.
+    - name: Install ready's runtime deps on the base PATH
+      run: gem install by:1.1.0 zeitwerk:2.8.2
+    - name: Run E2E specs
+      run: bundle exec rake spec:e2e
+
+  # macOS is where the harness actually runs for users (zsh is the default
+  # shell, executables resolve through rbenv). This job reproduces the
+  # process-hang/leak reports on a real Mac and fails loudly if the harness
+  # leaves any by-server or sandbox process behind.
+  macos-e2e:
+    runs-on: macos-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v4
+    - name: Cache the rbenv-built Ruby (first run compiles it, later runs restore)
+      uses: actions/cache@v4
+      with:
+        path: ~/.rbenv/versions
+        key: rbenv-versions-4.0.1-${{ runner.os }}
+    - name: Install rbenv + Ruby 4.0.1 (matches a real user's setup)
+      run: |
+        brew install rbenv ruby-build coreutils
+        rbenv versions --bare | grep -qx 4.0.1 || rbenv install 4.0.1
+        rbenv global 4.0.1
+        echo "$HOME/.rbenv/shims" >> "$GITHUB_PATH"
+    - name: Bundle
+      run: |
+        gem install bundler
+        bundle install
+    # A hard gtimeout wraps each step so a regression hangs the job for
+    # seconds, not the 6h GitHub ceiling. The whole point of this job is that
+    # the harness must run to completion on macOS (default zsh, rbenv).
+    - name: Run the e2e suite
+      run: gtimeout -s KILL 300 bundle exec rake spec:e2e
+    - name: Run the real benchmark (what bin/bench runs)
+      run: gtimeout -s KILL 200 bundle exec rake bench BENCH_EXE=ri BENCH_ARGS=TCPServer BENCH_RUNS=2 BENCH_WARMUPS=1
+    - name: Fail if the harness leaked any process
+      if: always()
+      run: |
+        pgrep -fl by-server || echo "no by-server"
+        pgrep -fl ready-e2e || echo "no ready-e2e"
+        leaked=$(pgrep -f 'by-server|ready-e2e' | wc -l | tr -d ' ')
+        echo "leaked process count: $leaked"
+        if [ "$leaked" != "0" ]; then
+          echo "::error::harness leaked $leaked process(es)"
+          exit 1
+        fi

.gitignore

@@ -1,4 +1,11 @@
 /.bundle/
+/vendor/
+/.idea/
+/talk.tar.gz
+# Local skill caches dropped by the assistant tooling; regenerated on reload.
+/_claude/poodr/
+/_claude/rspec3/
+/_claude/zsh/
 /.yardoc
 /_yardoc/
 /coverage/
@@ -9,4 +16,6 @@
 Gemfile.lock
 *.gem
 .rspec_status
-references
+references
+# benchmark run logs
+/log/

.rubocop.yml

@@ -17,10 +17,23 @@ plugins:
 
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 4.0
+  # Lint at the gem's supported floor (ready.gemspec's required_ruby_version),
+  # which CI also exercises -- not the 4.0.1 dev pin -- so accidental
+  # 4.0-only syntax is caught rather than shipped to 3.4 users.
+  TargetRubyVersion: 3.4
   Exclude:
-    - bin/*
+    # Stock bundler scripts; bin/bench is ours and stays linted.
+    - bin/console
+    - bin/setup
     - vendor/**/*
+    # Vendored read-only reference copy of an external gem; not this project's
+    # code (it is also gitignored, so CI never sees it).
+    - references/**/*
+    # Standalone runtime helper scripts -- a doc formatter and the rdoc `ri`
+    # patch -- with their own conventions, not the gem's authored library code.
+    - extra/**/*
+    # Gitignored scratch: benchmark run logs and throwaway report-parsing scripts.
+    - log/**/*
     - lib/core_ext/**/*
     - rakelib/project.rb
     - rakelib/project_version.rb
@@ -128,6 +141,12 @@ Claude/MysteryRegex:
 Style/MutableConstant:
   EnforcedStyle: literals
 
+# Data.define subclassing keeps the class body a real class body, so constants
+# holding instances of the class (e.g. Span::TABLE) live where they belong;
+# the block form would bind them lexically to the enclosing namespace.
+Style/DataInheritance:
+  Enabled: false
+
 # Shared test contexts legitimately define many helpers.
 RSpec/MultipleMemoizedHelpers:
   Max: 10
@@ -151,6 +170,35 @@ Style/Documentation:
   Exclude:
     - "spec/**/*"
 
+# E2E specs share one expensive by-server across examples via before(:all) and
+# hold it in an instance variable; that is intentional here.
+RSpec/BeforeAfterAll:
+  Exclude:
+    - "spec/e2e/**/*"
+
+RSpec/InstanceVariable:
+  Exclude:
+    - "spec/e2e/**/*"
+
+# E2E examples are longer by nature (shell/server setup + several aggregated
+# structural assertions per example).
+RSpec/ExampleLength:
+  Exclude:
+    - "spec/e2e/**/*"
+
+# The e2e and benchmark suites are grouped by test type, not class namespace,
+# so their paths deliberately do not mirror the described constant.
+RSpec/SpecFilePathFormat:
+  Exclude:
+    - "spec/e2e/**/*"
+    - "spec/bench/**/*"
+
+# Ready::Bench::Report is a waterfall renderer whose job is to write to stdout;
+# it lives under spec/support only so Zeitwerk autoloads it.
+RSpec/Output:
+  Exclude:
+    - "spec/support/bench/**/*"
+
 # Trailing commas in multiline literals and arguments.
 Style/TrailingCommaInArrayLiteral:
   EnforcedStyleForMultiline: comma

CLAUDE.md

@@ -19,6 +19,7 @@ The project has two halves that meet at generated code:
 `rakelib/ready.rake` is the orchestrator that drives half 1 to produce artifacts
 that half 2 consumes.
 
+It uses zeitwerk
 ## Commands
 
 ```bash
@@ -111,13 +112,22 @@ socket. Top-level `rake ready` = restart server + compile + clean.
   (`lib/ready/foo_bar.rb` → `Ready::FooBar`). A spec eager-loads everything and
   `rake zeitwerk:validate` checks naming — run it after adding/moving files.
 - **Ruby 4.0.1.** The code leans on modern syntax (`case/in` pattern matching,
-  the `it` block param, endless methods). CI pins 4.0.1.
+  the `it` block param). **No endless methods** (`def x = y`) — always regular
+  `def`/`end` blocks. CI pins 4.0.1.
 - **RuboCop is heavily customized** (see `.rubocop.yml`) via `rubocop-claude`
   (AI guardrails). Match the house style: double quotes, **no** frozen-string
   comment, trailing commas in multiline literals/args, dot-aligned multiline
   method chains, pipeline/`.then`-chaining style, short blocks
   (`Metrics/BlockLength` max 8), and every class carries an rdoc `##` comment
   (`Style/Documentation` is on).
+- **Design rulings (owner review, binding):** name the domain — real objects
+  over primitive hashes; never return an array/tuple (use a `Data.define`
+  value object whose readers carry type and unit, e.g. `PtyShell::Result`);
+  no abbreviations in names; polymorphism over boolean/type flags; symbols
+  over string keys; `Pathname` everywhere (never `File.expand_path` or
+  `File.join`); `system` calls pass `exception: true` unless the result is
+  explicitly checked; intermediate variables over nested work-doing calls;
+  multi-variant docs as tables/bullets, never paragraphs.
 - **`references/command_kit.rb/` is a vendored, read-only reference copy** of an
   external gem (its own git repo). It is not part of this project — don't edit it
   or count it when reasoning about the codebase.

Gemfile

@@ -11,3 +11,4 @@ gem "rubocop-performance"
 gem "rubocop-rake"
 gem "rubocop-rspec"
 gem "ruby-lsp", "~> 0.26.10"
+gem "youplot"

Rakefile

@@ -1,9 +1,100 @@
+# Everything below develops the ready gem itself (gem build/release, specs,
+# rubocop, benchmarks) and depends on dev-only gems. None of it is needed to RUN
+# ready: the runtime `ready`/`compile`/`clobber`/`ready:*` tasks live in
+# rakelib/ready.rake, which rake auto-loads independently of this file. When
+# ready runs as an installed gem the gemspec and dev gems are absent, so loading
+# this file would crash `ready up|compile|clobber` (e.g. bundler/gem_tasks
+# raising "Unable to determine name from existing gemspec"). Load the dev tasks
+# only in a source checkout, detected by the gemspec's presence.
+return unless (Pathname(__dir__) / "ready.gemspec").exist?
+
 require "bundler/gem_tasks"
 
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
+namespace :spec do
+  desc "Run the end-to-end (:e2e) specs (needs zsh + by/by-server)"
+  task :e2e do
+    ok = system({ "READY_E2E" => "1" }, RbConfig.ruby, "-S", "rspec", "--tag", "e2e")
+    abort("e2e specs failed") unless ok
+  end
+end
+
+def bench_protocol
+  executable = ENV.fetch("BENCH_EXE", "ri")
+  # TCPServer is the default workload for the default tool only; any other
+  # executable runs bare unless BENCH_ARGS says otherwise.
+  default_arguments = executable == "ri" ? "TCPServer" : ""
+  Ready::Bench::Protocol.new(
+    executable_name: executable,
+    arguments: ENV.fetch("BENCH_ARGS", default_arguments).split,
+    preload_gems: bench_preload_gems,
+    rounds: Integer(ENV.fetch("BENCH_RUNS", "15")),
+    warmups: Integer(ENV.fetch("BENCH_WARMUPS", "3")),
+  )
+end
+
+# The hot server preloads the gems a readyfile declares (BENCH_READYFILE),
+# falling back to rdoc, which ships the default tool. The build_dir is
+# irrelevant here -- only gem names are read -- but Readyfile requires an
+# existing directory, so the readyfile's own parent satisfies it.
+def bench_preload_gems
+  readyfile_path = ENV.fetch("BENCH_READYFILE", nil)
+  return ["rdoc"] if readyfile_path.nil?
+
+  readyfile_path = Pathname(readyfile_path)
+  Ready::Readyfile.open(readyfile_path, build_dir: readyfile_path.expand_path.parent).gem_names
+end
+
+def bench_runner
+  require "ready"
+  require "zeitwerk"
+  Zeitwerk::Loader.new.tap do |loader|
+    loader.inflector.inflect("cli" => "CLI")
+    loader.push_dir(Pathname(__dir__) / "spec/support", namespace: Ready)
+    loader.setup
+  end
+  Ready::Bench::Runner.new(protocol: bench_protocol)
+end
+
+# Appends this run's headline numbers so a caller sequencing several
+# benchmarks (bin/bench --plot) can chart them afterwards.
+def export_bench_results(runner, results_path)
+  results = Ready::Bench::ResultsLog.new(results_path)
+  results.append(command: runner.invocation, arm: :cold,
+                 full_milliseconds: runner.cold_summary.duration_of(:full))
+  results.append(command: runner.invocation, arm: :hot,
+                 full_milliseconds: runner.hot_summary.duration_of(:full))
+end
+
+# Plot mode (BENCH_RESULTS set by bin/bench --plot) exports the headline
+# numbers for the CLI to chart and stays silent on stdout, so the plot is the
+# only output; otherwise the full waterfall report is the output. Either way
+# the run narrates its progress to stderr.
+def run_bench(verbose:)
+  runner = bench_runner.call
+  results_path = ENV.fetch("BENCH_RESULTS", nil)
+  if results_path
+    export_bench_results(runner, results_path)
+  else
+    runner.render(verbose:)
+  end
+end
+
+desc "Print the cold-vs-hot startup waterfall (needs zsh + by-server + rbenv); bin/bench is the front door"
+task :bench do
+  run_bench(verbose: false)
+end
+
+namespace :bench do
+  desc "rake bench plus a legend table explaining every span row"
+  task :verbose do
+    run_bench(verbose: true)
+  end
+end
+
 require "rubocop/rake_task"
 RuboCop::RakeTask.new