From f94ae58d91927d02c6433d9d3200df82f12b0db7 Mon Sep 17 00:00:00 2001 From: Colin McDonnell <3084745+colinhacks@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:01:40 -0700 Subject: [PATCH 1/9] loader: standalone tsx-style npm loader entries, packages, and version plumbing Adds runtime/loader-*.{mjs,cjs}: slim entrypoints that arm only the shared resolve+transpile hooks (transform-core + preload-common) for node --import / --require / /esm, with the addon resolved from a per-platform npm package via __NUB_ADDON_PATH (probed last by transform-core). Adds npm/loader + npm/loader- package manifests, the staging script, and version/lockstep/gitignore plumbing. --- .gitignore | 10 ++ Makefile | 14 ++ npm/loader-darwin-arm64/package.json | 16 ++ npm/loader-darwin-x64/package.json | 16 ++ npm/loader-linux-arm64-musl/package.json | 19 +++ npm/loader-linux-arm64/package.json | 19 +++ npm/loader-linux-x64-musl/package.json | 19 +++ npm/loader-linux-x64/package.json | 19 +++ npm/loader-win32-arm64/package.json | 16 ++ npm/loader-win32-x64/package.json | 16 ++ npm/loader/README.md | 48 ++++++ npm/loader/package.json | 61 +++++++ runtime/loader-addon-env.mjs | 24 +++ runtime/loader-entry.mjs | 193 +++++++++++++++++++++++ runtime/loader-esm.mjs | 6 + runtime/loader-platform.cjs | 95 +++++++++++ runtime/loader-register.cjs | 30 ++++ runtime/loader-register.mjs | 5 + runtime/preload-common.cjs | 4 + runtime/transform-core.mjs | 10 ++ scripts/build-loader-npm.mjs | 88 +++++++++++ scripts/check-oxc-lockstep.mjs | 19 ++- scripts/set-version.mjs | 11 ++ 23 files changed, 751 insertions(+), 7 deletions(-) create mode 100644 npm/loader-darwin-arm64/package.json create mode 100644 npm/loader-darwin-x64/package.json create mode 100644 npm/loader-linux-arm64-musl/package.json create mode 100644 npm/loader-linux-arm64/package.json create mode 100644 npm/loader-linux-x64-musl/package.json create mode 100644 npm/loader-linux-x64/package.json create mode 100644 npm/loader-win32-arm64/package.json create mode 100644 npm/loader-win32-x64/package.json create mode 100644 npm/loader/README.md create mode 100644 npm/loader/package.json create mode 100644 runtime/loader-addon-env.mjs create mode 100644 runtime/loader-entry.mjs create mode 100644 runtime/loader-esm.mjs create mode 100644 runtime/loader-platform.cjs create mode 100644 runtime/loader-register.cjs create mode 100644 runtime/loader-register.mjs create mode 100644 scripts/build-loader-npm.mjs diff --git a/.gitignore b/.gitignore index 2984d0172..79065fea3 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,16 @@ npm/nub-*/bin/ npm/nub/*.tgz npm/nub-*/*.tgz +# Standalone-loader packages (npm/loader, npm/loader-): the JS is a +# verbatim slice of runtime/ staged in by scripts/build-loader-npm.mjs, and the +# addon comes from the platform build — only package.json + README are source. +npm/loader/*.mjs +npm/loader/*.cjs +npm/loader/LICENSE +npm/loader/*.tgz +npm/loader-*/nub-native.node +npm/loader-*/*.tgz + # nub's localStorage/cache leaking into cwd as `0/--…` when # XDG_CACHE_HOME is unset and the store path resolves relative (a real nub bug — # the cache should always live under ~/.cache/nub). Ignore the stray dirs until diff --git a/Makefile b/Makefile index 341ad7107..ce2a25b20 100644 --- a/Makefile +++ b/Makefile @@ -207,6 +207,20 @@ version-check: const types = JSON.parse(fs.readFileSync('npm/nub-types/package.json', 'utf8')); \ if (types.version !== v) errors.push('npm/nub-types/package.json has ' + types.version + ', expected ' + v); \ } catch { errors.push('missing or unreadable npm/nub-types/package.json'); } \ + try { \ + const loader = JSON.parse(fs.readFileSync('npm/loader/package.json', 'utf8')); \ + if (loader.version !== v) errors.push('npm/loader/package.json has ' + loader.version + ', expected ' + v); \ + for (const [dep, ver] of Object.entries(loader.optionalDependencies || {})) { \ + if (ver !== v) errors.push(dep + ' optionalDependency pinned at ' + ver + ', expected ' + v); \ + const pkg = 'npm/' + dep.replace('@nubjs/', '') + '/package.json'; \ + try { \ + const p = JSON.parse(fs.readFileSync(pkg, 'utf8')); \ + if (p.version !== v) errors.push(pkg + ' has ' + p.version + ', expected ' + v); \ + } catch { errors.push('missing or unreadable ' + pkg); } \ + } \ + const lrt = (loader.dependencies || {})['@oxc-project/runtime']; \ + if (!lrt) errors.push('npm/loader/package.json: @oxc-project/runtime missing from dependencies'); \ + } catch { errors.push('missing or unreadable npm/loader/package.json'); } \ const cargo = fs.readFileSync('Cargo.toml', 'utf8'); \ const cm = cargo.match(/^version = \x22([^\x22]*)\x22/m); \ if (!cm) errors.push('Cargo.toml: workspace version line not found'); \ diff --git a/npm/loader-darwin-arm64/package.json b/npm/loader-darwin-arm64/package.json new file mode 100644 index 000000000..aa6817160 --- /dev/null +++ b/npm/loader-darwin-arm64/package.json @@ -0,0 +1,16 @@ +{ + "name": "@nubjs/loader-darwin-arm64", + "version": "0.8.0", + "description": "Nub loader native addon for macOS ARM64 (Apple Silicon)", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "files": [ + "nub-native.node" + ] +} diff --git a/npm/loader-darwin-x64/package.json b/npm/loader-darwin-x64/package.json new file mode 100644 index 000000000..e22218dcb --- /dev/null +++ b/npm/loader-darwin-x64/package.json @@ -0,0 +1,16 @@ +{ + "name": "@nubjs/loader-darwin-x64", + "version": "0.8.0", + "description": "Nub loader native addon for macOS x64 (Intel)", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "files": [ + "nub-native.node" + ] +} diff --git a/npm/loader-linux-arm64-musl/package.json b/npm/loader-linux-arm64-musl/package.json new file mode 100644 index 000000000..be782a155 --- /dev/null +++ b/npm/loader-linux-arm64-musl/package.json @@ -0,0 +1,19 @@ +{ + "name": "@nubjs/loader-linux-arm64-musl", + "version": "0.8.0", + "description": "Nub loader native addon for Linux ARM64 (musl)", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "files": [ + "nub-native.node" + ], + "libc": [ + "musl" + ] +} diff --git a/npm/loader-linux-arm64/package.json b/npm/loader-linux-arm64/package.json new file mode 100644 index 000000000..afcf9c3f2 --- /dev/null +++ b/npm/loader-linux-arm64/package.json @@ -0,0 +1,19 @@ +{ + "name": "@nubjs/loader-linux-arm64", + "version": "0.8.0", + "description": "Nub loader native addon for Linux ARM64 (glibc)", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "files": [ + "nub-native.node" + ], + "libc": [ + "glibc" + ] +} diff --git a/npm/loader-linux-x64-musl/package.json b/npm/loader-linux-x64-musl/package.json new file mode 100644 index 000000000..15931ad4c --- /dev/null +++ b/npm/loader-linux-x64-musl/package.json @@ -0,0 +1,19 @@ +{ + "name": "@nubjs/loader-linux-x64-musl", + "version": "0.8.0", + "description": "Nub loader native addon for Linux x64 (musl)", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "files": [ + "nub-native.node" + ], + "libc": [ + "musl" + ] +} diff --git a/npm/loader-linux-x64/package.json b/npm/loader-linux-x64/package.json new file mode 100644 index 000000000..7e1b1285e --- /dev/null +++ b/npm/loader-linux-x64/package.json @@ -0,0 +1,19 @@ +{ + "name": "@nubjs/loader-linux-x64", + "version": "0.8.0", + "description": "Nub loader native addon for Linux x64 (glibc)", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "files": [ + "nub-native.node" + ], + "libc": [ + "glibc" + ] +} diff --git a/npm/loader-win32-arm64/package.json b/npm/loader-win32-arm64/package.json new file mode 100644 index 000000000..6bb95589c --- /dev/null +++ b/npm/loader-win32-arm64/package.json @@ -0,0 +1,16 @@ +{ + "name": "@nubjs/loader-win32-arm64", + "version": "0.8.0", + "description": "Nub loader native addon for Windows ARM64", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "win32" + ], + "cpu": [ + "arm64" + ], + "files": [ + "nub-native.node" + ] +} diff --git a/npm/loader-win32-x64/package.json b/npm/loader-win32-x64/package.json new file mode 100644 index 000000000..94dd0957d --- /dev/null +++ b/npm/loader-win32-x64/package.json @@ -0,0 +1,16 @@ +{ + "name": "@nubjs/loader-win32-x64", + "version": "0.8.0", + "description": "Nub loader native addon for Windows x64", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "files": [ + "nub-native.node" + ] +} diff --git a/npm/loader/README.md b/npm/loader/README.md new file mode 100644 index 000000000..e32d08a04 --- /dev/null +++ b/npm/loader/README.md @@ -0,0 +1,48 @@ +# nubjs + +Standalone TypeScript loader for Node.js, from the [Nub](https://nubjs.com) project. Register it the way tsx or ts-node is registered, and TypeScript works in `import`, in `require()`, and in worker threads — powered by the same native oxc-based transform the Nub CLI uses. + +```sh +npm install --save-dev nubjs +node --import nubjs app.ts +``` + +Any way Node accepts a preload works: + +```sh +node --import nubjs app.ts # one run +NODE_OPTIONS="--import nubjs" vitest # tools that spawn node themselves +node --require nubjs app.ts # CommonJS delivery (see below) +``` + +## What it does + +- Transpiles `.ts` / `.tsx` / `.mts` / `.cts` / `.jsx` on the fly — full TypeScript, including enums, namespaces, and legacy decorators, not just type stripping. +- Resolves TypeScript conventions: tsconfig `paths` and `baseUrl`, extensionless imports, the `.js` → `.ts` emit-convention swap, directory index files. +- Augments CommonJS `require()` with the same resolution and transpile, not only `import`. +- Loads data formats as modules: `.yaml`, `.toml`, `.json5`, `.jsonc`, `.txt`, and `with { type: "text" }` imports. +- Lowers `using` / `await using` and other syntax newer than the running Node. +- Inline source maps, on for every transpiled file. +- Applies inside worker threads automatically (Node inherits the preload). + +Dependencies under `node_modules` are never transpiled, and files Node handles natively load byte-for-byte unchanged — the loader adds behavior, it does not modify Node's. + +## Entry points + +```sh +node --import nubjs app.ts # ESM hooks + CommonJS require() augmentation +node --require nubjs app.ts # same, delivered as a CommonJS preload (Node 20.19+) +node --import nubjs/esm app.ts # ESM hooks only +``` + +Module formats follow Node's own rules: a `.cts` file is CommonJS and a `.mts` file is an ES module, and the loader transpiles types and syntax without converting one format into the other. + +## Node support + +Node 18.19 and newer. On Node 22.15+ hooks register synchronously in-thread (`module.registerHooks`); older versions run them in Node's loader worker (`module.register`). The `--require` delivery needs `require(esm)` (Node 20.19+ / 22.12+); below that use `--import`. + +## Relationship to the Nub CLI + +The [`@nubjs/nub`](https://www.npmjs.com/package/@nubjs/nub) CLI is a complete TypeScript-first toolchain — runner, package manager, Node version management — and does everything this loader does without any flags. This package is the loader alone, for cases where the `node` invocation itself is fixed: existing tooling, test runners, other CLIs that spawn `node`. + +Platform binaries ship as `optionalDependencies` (`@nubjs/loader-*`) for macOS, Linux (glibc and musl), and Windows, on x64 and arm64. diff --git a/npm/loader/package.json b/npm/loader/package.json new file mode 100644 index 000000000..2501d8678 --- /dev/null +++ b/npm/loader/package.json @@ -0,0 +1,61 @@ +{ + "name": "nubjs", + "version": "0.8.0", + "description": "Standalone TypeScript loader for Node.js from the Nub project — TypeScript, JSX, tsconfig paths, and data-format imports through a native transform, registered the way tsx and ts-node are", + "license": "MIT", + "repository": "https://github.com/nubjs/nub", + "homepage": "https://nubjs.com", + "bugs": { + "url": "https://github.com/nubjs/nub/issues" + }, + "keywords": [ + "typescript", + "loader", + "esm", + "nodejs", + "tsx", + "ts-node", + "transpiler", + "jsx" + ], + "engines": { + "node": ">=18.19.0" + }, + "exports": { + ".": { + "import": "./loader-register.mjs", + "require": "./loader-register.cjs" + }, + "./esm": "./loader-esm.mjs", + "./package.json": "./package.json" + }, + "files": [ + "loader-register.mjs", + "loader-register.cjs", + "loader-esm.mjs", + "loader-entry.mjs", + "loader-addon-env.mjs", + "loader-platform.cjs", + "transform-core.mjs", + "preload-common.cjs", + "preload-async-hooks.mjs", + "pnp-util.cjs", + "floor-builtin.mjs", + "cache-evict.mjs", + "README.md", + "LICENSE" + ], + "dependencies": { + "@oxc-project/runtime": "0.140.0" + }, + "optionalDependencies": { + "@nubjs/loader-darwin-arm64": "0.8.0", + "@nubjs/loader-darwin-x64": "0.8.0", + "@nubjs/loader-linux-x64": "0.8.0", + "@nubjs/loader-linux-x64-musl": "0.8.0", + "@nubjs/loader-linux-arm64": "0.8.0", + "@nubjs/loader-linux-arm64-musl": "0.8.0", + "@nubjs/loader-win32-x64": "0.8.0", + "@nubjs/loader-win32-arm64": "0.8.0" + } +} diff --git a/runtime/loader-addon-env.mjs b/runtime/loader-addon-env.mjs new file mode 100644 index 000000000..2d8e18a9f --- /dev/null +++ b/runtime/loader-addon-env.mjs @@ -0,0 +1,24 @@ +// Standalone-loader addon plumbing — MUST evaluate before transform-core.mjs. +// +// transform-core loads the `nub-native` N-API addon at its own module evaluation +// (fast tier: eagerly, the moment the module body runs), probing a sibling +// `./addons/nub-native.node` first. Under the nub CLI that sibling always exists +// (the extracted runtime dir); in the standalone loader package the addon rides a +// per-platform npm package instead, so this module resolves it and hands the +// absolute path over via the internal `__NUB_ADDON_PATH` plumbing var — see +// ensureAddonEnv in loader-platform.cjs for the probe-order and worker-thread +// rationale. +// +// Why a separate side-effect module: ESM evaluates imports in source order, so the +// entry importing THIS file before transform-core is what guarantees the env var is +// set in time (the same ordering trick compile-cache-restore.mjs uses for +// NODE_COMPILE_CACHE). The createRequire import MUST come from `node:module` +// directly, NOT from floor-builtin.mjs: floor-builtin statically imports +// transform-core (to thread the floor's createRequire into it), so reaching +// createRequire through it would evaluate transform-core — and run its addon +// probe — before this module's body sets the env var. That ordering bug shipped +// in the first cut and only the dev tree's sibling addons/ dir masked it. +import { createRequire } from "node:module"; + +const __require = createRequire(import.meta.url); +__require("./loader-platform.cjs").ensureAddonEnv(__require); diff --git a/runtime/loader-entry.mjs b/runtime/loader-entry.mjs new file mode 100644 index 000000000..95f64cb25 --- /dev/null +++ b/runtime/loader-entry.mjs @@ -0,0 +1,193 @@ +// Standalone Nub loader — the arming logic behind `node --import ` / +// `node --require `, consumed the way tsx/ts-node are. Slim by design: it +// arms ONLY the resolve + transpile surface (TS/JSX/`using`-lowering, tsconfig +// `paths`, extension probing, data-format imports) from the shared +// transform-core / preload-common machinery, and none of the CLI runtime's +// process augmentation — no polyfills, no Temporal/Worker/navigator globals, no +// watch IPC, no user preload chain, no version marker. A file that runs under +// `node --import ` must behave identically minus TS-just-works. +// +// Import order is load-bearing (ESM evaluates imports in source order): +// 1. loader-addon-env.mjs — resolves the per-platform addon package and sets +// the internal `__NUB_ADDON_PATH` plumbing var BEFORE transform-core's +// module body probes for the addon. +// 2. floor-builtin.mjs — threads `createRequire` into transform-core on the +// narrow pre-`process.getBuiltinModule` floor (18.19.x, 20.11–20.15, +// 22.0–22.2); a no-op elsewhere. +// 3. transform-core.mjs — the tier-agnostic resolve+transpile core, shared +// verbatim with the nub CLI. It has ZERO static imports by construction, so +// routing it through a user's loader chain leaks nothing (R11). +import "./loader-addon-env.mjs"; +import { createRequire } from "./floor-builtin.mjs"; +import * as core from "./transform-core.mjs"; + +const __require = createRequire(import.meta.url); +const module_ = __require("node:module"); +const { fileURLToPath } = __require("node:url"); +const { dirname, isAbsolute, resolve: resolvePath, sep } = __require("node:path"); +const common = __require("./preload-common.cjs"); + +// One arming record per module instance (= per realm: the main thread and each +// user worker thread evaluate this module separately via inherited execArgv). +// `esmMode` records which hook surface the ESM side took, because the CJS side's +// classic-transpile decision depends on it. +const armed = { esm: false, cjs: false, esmMode: null }; + +const OWN_DIR = dirname(fileURLToPath(import.meta.url)) + sep; + +// The loader package's own published name, for recognizing our own `--import` +// token in the foreign-loader scan. In the published package this file sits next +// to package.json; in the dev tree (runtime/) there is none, and path-prefix +// matching covers that case. +const OWN_PKG_NAME = (() => { + try { + const raw = __require("node:fs").readFileSync( + fileURLToPath(new URL("./package.json", import.meta.url)), + "utf8", + ); + const name = JSON.parse(raw).name; + return typeof name === "string" && name.length > 0 ? name : null; + } catch { + return null; + } +})(); + +// nub's own preload chainer rides `--import` too; same marker preload-common uses. +const NUB_CHAIN_MARKER = /[\\/]\.nub[\\/]preload-chain\./; + +// Is this `--import`/`--loader` value one of OUR OWN entrypoints (or nub's +// chainer), as opposed to a genuinely foreign async loader (tsx, ts-node, an OTel +// attach)? The distinction preload-common's own scan does not need to make — the +// CLI's fast tier is delivered by `--require`, so for it ANY `--import` is +// foreign — but the standalone loader IS an `--import`, so a value-blind scan +// would classify the loader itself as foreign and force the async tier on every +// run in the broken-compose band. +function isOwnLoaderToken(value) { + if (!value) return false; + if (NUB_CHAIN_MARKER.test(value)) return true; + if (OWN_PKG_NAME && (value === OWN_PKG_NAME || value.startsWith(`${OWN_PKG_NAME}/`))) { + return true; + } + try { + const p = value.startsWith("file:") ? fileURLToPath(value) : value; + if (isAbsolute(p) && (resolvePath(p) + sep).startsWith(OWN_DIR)) return true; + // A relative dev-tree form (`--import ./runtime/loader-register.mjs`) + // resolves from the CWD, matching how Node resolved it. + if (p.startsWith(".") && (resolvePath(process.cwd(), p) + sep).startsWith(OWN_DIR)) { + return true; + } + } catch { + // Unparseable value — treat as foreign; over-selection of the async tier is + // safe (it is always correct, just slower to start). + } + return false; +} + +// A foreign async ESM loader riding THIS process's startup flags, via either +// delivery channel (execArgv or NODE_OPTIONS). Same two-channel scan as +// preload-common's computeForeignAsyncLoaderFlagPresent, but value-aware so our +// own token is excluded. +function foreignAsyncLoaderPresent() { + const tokens = []; + const argv = Array.isArray(process.execArgv) ? process.execArgv : []; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (typeof a !== "string") continue; + for (const flag of ["--import", "--loader", "--experimental-loader"]) { + if (a === flag) { + if (typeof argv[i + 1] === "string") tokens.push(argv[i + 1]); + } else if (a.startsWith(`${flag}=`)) { + tokens.push(a.slice(flag.length + 1)); + } + } + } + const opts = process.env.NODE_OPTIONS; + if (typeof opts === "string" && opts !== "") { + const re = /(?:^|\s)--(?:experimental-)?(?:import|loader)(?:=|\s)("[^"]*"|\S*)/g; + for (const match of opts.matchAll(re)) { + tokens.push((match[1] || "").replace(/^"|"$/g, "")); + } + } + return tokens.some((t) => t && !isOwnLoaderToken(t)); +} + +// Arm the loader. `esm` = the ESM hook surface (module.registerHooks on the fast +// tier, the module.register loader worker on the compat tier); `cjs` = the +// CommonJS require() surface (Module._resolveFilename + the classic transpile +// shim where the tier needs it). Idempotent per surface, so `--import ` plus +// `--require /cjs` in one invocation arms each exactly once. +export function arm({ esm = true, cjs = true } = {}) { + // Electron: the sync load hook deadlocks Electron's main-process module + // bootstrap, and its app JS is pre-bundled (the bundler owns TS) — same bail, + // same reason as the CLI preload (issue #246). + if (process.versions.electron) return; + + const wantEsm = esm && !armed.esm; + const wantCjs = cjs && !armed.cjs; + if (!wantEsm && !wantCjs) return; + + const [major = 0, minor = 0] = process.versions.node + .split(".") + .map((n) => parseInt(n, 10)); + if (major < 18 || (major === 18 && minor < 19)) { + process.stderr.write( + `The Nub loader requires Node 18.19 or newer; got ${process.versions.node}. Hooks are inactive.\n`, + ); + return; + } + + // The loader ships no polyfill packages, so the clobber map's synthetic + // modules (re-exports of globals the CLI runtime installs) would hand users + // `undefined`. A user who installed @js-temporal/polyfill or urlpattern-polyfill + // themselves must get the real package — clear the map before any hook runs. + core.CLOBBER_MAP.clear(); + + // No-op unless the nub CLI's watch mode spawned this process (it sets + // WATCH_REPORT_DEPENDENCIES); wiring it keeps `nub watch` restarts correct when + // the loader runs under it. + const watchReporting = common.installWatchReporting(core); + + const hasSyncHooks = typeof module_.registerHooks === "function"; + // On 22.15.0–24.11.0 an async `module.register` loader's resolveSync/loadSync + // are unimplemented stubs, so nub's sync hooks composing with a foreign async + // loader (tsx, ts-node, an OTel ESM attach) would crash resolution. Register + // via the async path there instead so both loaders compose all-async — the same + // tier decision the CLI preload makes, minus counting ourselves as foreign. + const forceAsync = common.nodeHookComposeBroken() && foreignAsyncLoaderPresent(); + + if (wantEsm) { + if (hasSyncHooks && !forceAsync) { + const { resolve, load } = common.makeHooks(core, watchReporting); + module_.registerHooks({ resolve, load }); + armed.esmMode = "sync"; + } else { + // Compat tier (18.19–22.14, 23.0–23.4) or forced-async composition: hooks + // run in a dedicated loader worker. It imports transform-core statically in + // its own thread and finds the addon via the inherited __NUB_ADDON_PATH. + common.registerLoaderWorker("./preload-async-hooks.mjs", import.meta.url); + armed.esmMode = "worker"; + } + armed.esm = true; + } + + if (wantCjs) { + // Classic require.extensions transpile is needed only where the sync + // registerHooks load hook does not already transpile require()'d TS: on the + // sync tier it does (and the classic shim would shadow native require(esm), + // throwing bogus ERR_REQUIRE_ESM on ESM `.ts` — see preload.cjs); elsewhere + // install it unless Node has native TypeScript (mirrors preload.mjs). + const classic = armed.esmMode === "sync" ? false : !process.features?.typescript; + common.installCjsRequireHooks(core, classic); + armed.cjs = true; + } + + // Bounded transpile-cache eviction, same cheap-probe shape as the CLI entries: + // schedule the deferred sweep only on the once-a-day run where one is due. + if (core.sweepDue()) { + setImmediate(() => { + try { + core.maybeSweepCache(); + } catch {} + }); + } +} diff --git a/runtime/loader-esm.mjs b/runtime/loader-esm.mjs new file mode 100644 index 000000000..8f8d4c7aa --- /dev/null +++ b/runtime/loader-esm.mjs @@ -0,0 +1,6 @@ +// `node --import /esm` — arms the ESM hook surface only (tsx/esm's shape). +// `import` of TS/JSX/data formats works; a bare `require()` of the same files is +// left to Node. +import { arm } from "./loader-entry.mjs"; + +arm({ esm: true, cjs: false }); diff --git a/runtime/loader-platform.cjs b/runtime/loader-platform.cjs new file mode 100644 index 000000000..0988d3e98 --- /dev/null +++ b/runtime/loader-platform.cjs @@ -0,0 +1,95 @@ +"use strict"; +// Standalone-loader addon location: platform → `@nubjs/loader-` package +// selection, plus the resolver that turns the selected package into an absolute +// `nub-native.node` path. Mirrors npm/nub/platform.js (same musl detection, same +// platform matrix) but for the loader's per-platform addon packages, which carry +// the ~6 MB N-API addon instead of the full CLI binary. CommonJS so both the ESM +// side-effect module (loader-addon-env.mjs) and any CJS entry can share it. + +const PLATFORMS = { + "darwin-arm64": "@nubjs/loader-darwin-arm64", + "darwin-x64": "@nubjs/loader-darwin-x64", + "linux-x64": "@nubjs/loader-linux-x64", + "linux-x64-musl": "@nubjs/loader-linux-x64-musl", + "linux-arm64": "@nubjs/loader-linux-arm64", + "linux-arm64-musl": "@nubjs/loader-linux-arm64-musl", + "win32-x64": "@nubjs/loader-win32-x64", + "win32-arm64": "@nubjs/loader-win32-arm64", +}; + +// True on a musl Linux (Alpine, etc.). Primary signal: Node's own diagnostic +// report — `header.glibcVersionRuntime` is present on glibc and absent on musl. +// Fallback: `ldd --version`, whose merged output contains "musl" there (the +// stderr-only read shipped wrong once in npm/nub — check the merged output). +function isMusl() { + if (process.platform !== "linux") return false; + try { + const report = process.report.getReport(); + const header = (typeof report === "string" ? JSON.parse(report) : report).header; + if (header && "glibcVersionRuntime" in header) { + return !header.glibcVersionRuntime; + } + } catch { + // process.report unavailable — fall through to ldd. + } + try { + const out = require("child_process").execSync("ldd --version 2>&1", { encoding: "utf8" }); + return out.includes("musl"); + } catch (e) { + const out = `${(e && e.stdout) || ""}${(e && e.stderr) || ""}`; + return out.includes("musl"); + } +} + +function platformKey() { + const base = `${process.platform}-${process.arch}`; + return isMusl() ? `${base}-musl` : base; +} + +// Absolute path to this platform's `nub-native.node`, resolved from the loader +// package's own dependency tree via the caller-supplied `require` (created from a +// file inside the package, so the node_modules walk starts next to the platform +// packages regardless of hoisting). Returns null when the platform package is not +// installed — an unsupported platform, or optionalDependencies pruned. +function resolveAddonPath(requireFromPackage) { + const pkg = PLATFORMS[platformKey()]; + if (!pkg) return null; + try { + return requireFromPackage.resolve(`${pkg}/nub-native.node`); + } catch { + return null; + } +} + +// Make the addon reachable for every transform-core instance in this process tree +// by setting the internal `__NUB_ADDON_PATH` plumbing var (probed LAST by +// transform-core, after its relative candidates, so a nested nub CLI always wins +// with its own bundled addon). Worker threads inherit process.env, which is what +// carries the path into the compat tier's loader worker. A sibling +// `addons/nub-native.node` (the dev tree, or a bundled layout) means the relative +// probe wins anyway and no env is needed. Idempotent; safe to call from both the +// ESM side-effect module and the CJS `--require` fallback. +function ensureAddonEnv(requireFromPackage) { + if (process.env.__NUB_ADDON_PATH) return true; + try { + const { statSync } = require("node:fs"); + const sibling = require("node:path").join(__dirname, "addons", "nub-native.node"); + const s = statSync(sibling, { throwIfNoEntry: false }); + if (s !== undefined && s.isFile()) return true; + } catch { + // fall through to the platform-package probe + } + const resolved = resolveAddonPath(requireFromPackage); + if (resolved) { + process.env.__NUB_ADDON_PATH = resolved; + return true; + } + process.stderr.write( + `The Nub loader could not find its native addon for ${process.platform}-${process.arch}` + + ` — the platform package may be missing (optionalDependencies pruned, or an` + + ` unsupported platform). TypeScript transpilation is inactive.\n`, + ); + return false; +} + +module.exports = { PLATFORMS, platformKey, resolveAddonPath, ensureAddonEnv }; diff --git a/runtime/loader-register.cjs b/runtime/loader-register.cjs new file mode 100644 index 000000000..6993db947 --- /dev/null +++ b/runtime/loader-register.cjs @@ -0,0 +1,30 @@ +"use strict"; +// `node --require ` — the CommonJS delivery of the full loader (ESM hooks + +// CommonJS require() augmentation). On Node 22.15+ this is strictly the better +// consumption shape: a `--require` CJS preload keeps Node's synchronous CJS entry +// path (the mere presence of `--import` forces eager async ESM-loader init that +// routes even a CJS entry through the async module-job — see preload.cjs, R1), +// which is exactly why the nub CLI injects its own fast-tier preload this way. +// +// require(esm) loads the shared ES-module arming logic synchronously (TLA-free by +// construction). Where require(esm) is unavailable — `--no-experimental-require- +// module`, or a compat-tier Node below 22.12/20.19 — fall back to registering the +// loader-worker hooks directly (preload-common is CommonJS, so that registration +// needs no require(esm)): `import`-side TS still transpiles through the worker, +// and only require()'d TS is inactive, matching the CLI preload's own degradation +// under that flag. +try { + require("./loader-entry.mjs").arm({ esm: true, cjs: true }); +} catch (err) { + if (!err || err.code !== "ERR_REQUIRE_ESM") throw err; + if (process.versions.electron) return; + // The loader worker's transform-core needs the addon path in the inherited env + // — loader-entry.mjs (which normally sets it) could not load here. + require("./loader-platform.cjs").ensureAddonEnv(require); + const { pathToFileURL } = require("node:url"); + const common = require("./preload-common.cjs"); + common.registerLoaderWorker( + "./preload-async-hooks.mjs", + pathToFileURL(__filename).href, + ); +} diff --git a/runtime/loader-register.mjs b/runtime/loader-register.mjs new file mode 100644 index 000000000..e3c5a95c5 --- /dev/null +++ b/runtime/loader-register.mjs @@ -0,0 +1,5 @@ +// `node --import ` — arms BOTH module systems (ESM hooks + CommonJS +// require() augmentation), tsx's default-entry shape. +import { arm } from "./loader-entry.mjs"; + +arm({ esm: true, cjs: true }); diff --git a/runtime/preload-common.cjs b/runtime/preload-common.cjs index 8072168ff..1d259e95d 100644 --- a/runtime/preload-common.cjs +++ b/runtime/preload-common.cjs @@ -1709,6 +1709,10 @@ module.exports = { registerLoaderWorker, makeHooks, shouldAutoAsyncTierAtPreload, + // Consumed by the standalone loader entry (loader-entry.mjs), whose foreign- + // loader scan must be value-aware (its own delivery IS an `--import`) and so + // cannot reuse shouldAutoAsyncTierAtPreload directly. + nodeHookComposeBroken, installCjsRequireHooks, preloadPolyfillPackages, installTemporalGlobal, diff --git a/runtime/transform-core.mjs b/runtime/transform-core.mjs index 9ae4b891d..a26e54537 100644 --- a/runtime/transform-core.mjs +++ b/runtime/transform-core.mjs @@ -144,6 +144,16 @@ function __ensureBuiltins() { for (const rel of ["./addons/nub-native.node", "../runtime/addons/nub-native.node"]) { try { nubNative = __require(fileURLToPath(new URL(rel, import.meta.url))); break; } catch {} } + // Standalone-loader distribution (`node --import `): the addon rides a + // per-platform npm package rather than a sibling addons/ dir; the loader entry + // resolves it and hands the absolute path over via internal env plumbing + // (loader-platform.cjs ensureAddonEnv). LAST in probe order, deliberately: a + // nub-CLI process nested under the standalone loader inherits the env var, and + // probing it first would load the outer loader's (possibly differently- + // versioned) addon over the CLI's own bundled one. + if (!nubNative && process.env.__NUB_ADDON_PATH) { + try { nubNative = __require(process.env.__NUB_ADDON_PATH); } catch {} + } } // Fast tier: getBuiltinModule is present, so acquire everything now (preserves the // original eager-at-eval behavior). The floor defers to first-use — see above. diff --git a/scripts/build-loader-npm.mjs b/scripts/build-loader-npm.mjs new file mode 100644 index 000000000..f6e2f5930 --- /dev/null +++ b/scripts/build-loader-npm.mjs @@ -0,0 +1,88 @@ +// Stage the standalone-loader npm packages from the runtime/ sources. +// +// The loader package ships a curated slice of runtime/ verbatim — the loader +// entrypoints plus the shared resolve/transpile machinery (transform-core and +// friends) — laid out FLAT at the package root so every relative require works +// unchanged. This script copies that slice into npm/loader/ and, when a built +// addon is present (or --addon points at one), places it in the current +// platform's npm/loader-/ package. Release CI runs it once per +// platform leg; locally it stages whatever the dev tree has built. +// +// node scripts/build-loader-npm.mjs [--addon ] [--pack] +// +// --pack additionally runs `npm pack` in each staged package dir, leaving +// versioned tarballs in place (the local e2e installs from these). +import { cpSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repo = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + +// The exact file closure of the loader entrypoints. Additions to the entries' +// relative-import graph must be mirrored here; the existence check below turns a +// forgotten file into a hard failure rather than a broken tarball. +const RUNTIME_FILES = [ + "loader-register.mjs", + "loader-register.cjs", + "loader-esm.mjs", + "loader-entry.mjs", + "loader-addon-env.mjs", + "loader-platform.cjs", + "transform-core.mjs", + "preload-common.cjs", + "preload-async-hooks.mjs", + "pnp-util.cjs", + "floor-builtin.mjs", + "cache-evict.mjs", +]; + +const args = process.argv.slice(2); +const pack = args.includes("--pack"); +const addonFlag = args.indexOf("--addon"); +const addonPath = + addonFlag !== -1 ? resolve(args[addonFlag + 1]) : join(repo, "runtime", "addons", "nub-native.node"); + +const loaderDir = join(repo, "npm", "loader"); +for (const f of RUNTIME_FILES) { + const src = join(repo, "runtime", f); + if (!existsSync(src)) { + console.error(`missing runtime file: ${src}`); + process.exit(1); + } + cpSync(src, join(loaderDir, f)); +} +cpSync(join(repo, "LICENSE"), join(loaderDir, "LICENSE")); +console.log(`staged ${RUNTIME_FILES.length} runtime files into npm/loader/`); + +// Verify the staged file list matches the manifest's `files` allowlist, so a file +// added to RUNTIME_FILES but not to package.json (or vice versa) fails here. +const manifest = JSON.parse(readFileSync(join(loaderDir, "package.json"), "utf8")); +const missing = RUNTIME_FILES.filter((f) => !manifest.files.includes(f)); +if (missing.length) { + console.error(`package.json files[] is missing: ${missing.join(", ")}`); + process.exit(1); +} + +const packed = [loaderDir]; + +// The current platform's addon package. Cross-platform staging is CI's job (one +// leg per platform); locally only the host's package can be staged. +if (existsSync(addonPath)) { + const { platformKey } = await import(join(repo, "runtime", "loader-platform.cjs")).then( + (m) => m.default ?? m, + ); + const platDir = join(repo, "npm", `loader-${platformKey()}`); + mkdirSync(platDir, { recursive: true }); + cpSync(addonPath, join(platDir, "nub-native.node")); + console.log(`staged addon → npm/loader-${platformKey()}/nub-native.node`); + packed.push(platDir); +} else { + console.log(`no addon at ${addonPath} — platform package not staged`); +} + +if (pack) { + for (const dir of packed) { + execFileSync("npm", ["pack"], { cwd: dir, stdio: "inherit" }); + } +} diff --git a/scripts/check-oxc-lockstep.mjs b/scripts/check-oxc-lockstep.mjs index 772b2efbf..3a9ff8532 100644 --- a/scripts/check-oxc-lockstep.mjs +++ b/scripts/check-oxc-lockstep.mjs @@ -80,13 +80,18 @@ if (smRoot && smNative && smRoot !== smNative) { // release as the transformer compiled into nub-native — a floating range here // would let the helpers drift from the emit that imports them, and the pin // doubles as the A12 transpile-cache-key proxy. -const rt = (JSON.parse(read("package.json")).dependencies ?? {})["@oxc-project/runtime"]; -if (!rt) { - errors.push("package.json: @oxc-project/runtime missing from dependencies"); -} else if (!/^\d+\.\d+\.\d+$/.test(rt)) { - errors.push(`package.json: @oxc-project/runtime must be an EXACT version, got "${rt}"`); -} else if (canonical && rt !== canonical) { - errors.push(`package.json: @oxc-project/runtime is ${rt}, expected ${canonical} (the Cargo.toml oxc pin)`); +// The standalone loader package declares the same helpers as a real dependency +// (its emitted code imports them, and a published tarball cannot carry a nested +// node_modules), so it is held to the same pin. +for (const manifest of ["package.json", "npm/loader/package.json"]) { + const rt = (JSON.parse(read(manifest)).dependencies ?? {})["@oxc-project/runtime"]; + if (!rt) { + errors.push(`${manifest}: @oxc-project/runtime missing from dependencies`); + } else if (!/^\d+\.\d+\.\d+$/.test(rt)) { + errors.push(`${manifest}: @oxc-project/runtime must be an EXACT version, got "${rt}"`); + } else if (canonical && rt !== canonical) { + errors.push(`${manifest}: @oxc-project/runtime is ${rt}, expected ${canonical} (the Cargo.toml oxc pin)`); + } } // The repo carries an npm and a bun lockfile (cross-PM dogfooding), and only diff --git a/scripts/set-version.mjs b/scripts/set-version.mjs index 1f3087ad0..d16fe1a58 100644 --- a/scripts/set-version.mjs +++ b/scripts/set-version.mjs @@ -54,6 +54,17 @@ const pkgs = [ "npm/nub-linux-arm64-musl/package.json", "npm/nub-win32-x64/package.json", "npm/nub-win32-arm64/package.json", + // The standalone loader and its per-platform addon packages track the binary's + // version in lockstep: the addon bakes the version into the transpile-cache key. + "npm/loader/package.json", + "npm/loader-darwin-arm64/package.json", + "npm/loader-darwin-x64/package.json", + "npm/loader-linux-x64/package.json", + "npm/loader-linux-x64-musl/package.json", + "npm/loader-linux-arm64/package.json", + "npm/loader-linux-arm64-musl/package.json", + "npm/loader-win32-x64/package.json", + "npm/loader-win32-arm64/package.json", ]; for (const f of pkgs) { const p = JSON.parse(fs.readFileSync(f, "utf8")); From 48673d2e583f35bd19c433f982168f7f3b5d2038 Mon Sep 17 00:00:00 2001 From: Colin McDonnell <3084745+colinhacks@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:06:14 -0700 Subject: [PATCH 2/9] loader: release wiring, docs page, and the tests/loader matrix harness release.yml stages a loader- addon artifact per build leg and publishes the 8 addon packages then the root loader package (stable + canary), mirroring the CLI's ordering. Adds site/content/docs/loader.mdx and a tarball-install matrix under tests/loader/ (--import on every tier, --require where require(esm) exists, optional tsx differential). --- .github/workflows/release.yml | 108 ++++++++++++++++++++++++++ site/content/docs/loader.mdx | 62 +++++++++++++++ site/content/docs/meta.json | 1 + tests/loader/README.md | 28 +++++++ tests/loader/fixtures/conf.yaml | 4 + tests/loader/fixtures/expected.txt | 5 ++ tests/loader/fixtures/main.ts | 6 ++ tests/loader/fixtures/package.json | 1 + tests/loader/fixtures/paths.ts | 5 ++ tests/loader/fixtures/req.cts | 4 + tests/loader/fixtures/src/alias.ts | 1 + tests/loader/fixtures/tsconfig.json | 1 + tests/loader/fixtures/using.ts | 3 + tests/loader/fixtures/util-cjs.cts | 3 + tests/loader/fixtures/util.ts | 3 + tests/loader/fixtures/worker-child.ts | 3 + tests/loader/fixtures/worker-main.ts | 4 + tests/loader/run-matrix.sh | 97 +++++++++++++++++++++++ 18 files changed, 339 insertions(+) create mode 100644 site/content/docs/loader.mdx create mode 100644 tests/loader/README.md create mode 100644 tests/loader/fixtures/conf.yaml create mode 100644 tests/loader/fixtures/expected.txt create mode 100644 tests/loader/fixtures/main.ts create mode 100644 tests/loader/fixtures/package.json create mode 100644 tests/loader/fixtures/paths.ts create mode 100644 tests/loader/fixtures/req.cts create mode 100644 tests/loader/fixtures/src/alias.ts create mode 100644 tests/loader/fixtures/tsconfig.json create mode 100644 tests/loader/fixtures/using.ts create mode 100644 tests/loader/fixtures/util-cjs.cts create mode 100644 tests/loader/fixtures/util.ts create mode 100644 tests/loader/fixtures/worker-child.ts create mode 100644 tests/loader/fixtures/worker-main.ts create mode 100755 tests/loader/run-matrix.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ef53df35..969f08262 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -907,6 +907,23 @@ jobs: name: nub-${{ matrix.platform }} path: npm/nub-${{ matrix.platform }} + - name: Assemble loader platform package (addon only) + shell: bash + env: + MATRIX_PLATFORM: ${{ matrix.platform }} + run: | + # The standalone loader (npm/loader) ships the SAME nub-native addon + # staged above, as its own per-platform package — the addon is the only + # platform-specific part of the loader; its JS is a verbatim slice of + # runtime/ that the publish job stages once (scripts/build-loader-npm.mjs). + cp runtime/addons/nub-native.node "npm/loader-$MATRIX_PLATFORM/nub-native.node" + + - name: Upload loader platform package + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: loader-${{ matrix.platform }} + path: npm/loader-${{ matrix.platform }} + glibc-floor-guard: # The real glibc-floor gate. test-install runs on glibc-2.39 hosts and would # NOT catch a floor regression (0.0.44 shipped a GLIBC_2.39-requiring @@ -1650,6 +1667,54 @@ jobs: (cd npm/nub-types && npm publish --access public) fi + - name: Prepare and publish loader packages (idempotent) + run: | + # The standalone loader: 8 addon-only platform packages first, the root + # package LAST — the same ordering safety as the CLI's packages, since the + # root exact-pins its platform deps and must never resolve to a version + # whose addon packages have not all published. Same idempotent skip on an + # already-published version. Each platform leg uploaded its addon package + # (loader- artifact); the root's JS is staged here from this + # checkout by build-loader-npm.mjs (no addon on this runner, by design). + for platform in darwin-arm64 darwin-x64 linux-x64 linux-x64-musl linux-arm64 linux-arm64-musl win32-x64 win32-arm64; do + PKG="artifacts/loader-$platform" + if [ ! -d "$PKG" ]; then + echo "⚠️ Skipping loader-$platform (not built)" + continue + fi + cp "npm/loader-$platform/package.json" "$PKG/package.json" + node -e " + const fs = require('fs'); + const p = JSON.parse(fs.readFileSync('$PKG/package.json', 'utf8')); + p.version = process.env.VERSION; + fs.writeFileSync('$PKG/package.json', JSON.stringify(p, null, 2)); + " + NAME="@nubjs/loader-$platform" + if [ "$(npm view "$NAME@$VERSION" version 2>/dev/null)" = "$VERSION" ]; then + echo "✓ $NAME@$VERSION already published — skipping" + continue + fi + echo "→ $NAME@$VERSION" + (cd "$PKG" && npm publish --access public) + done + node scripts/build-loader-npm.mjs --addon /nonexistent + node -e " + const fs = require('fs'); + const p = JSON.parse(fs.readFileSync('npm/loader/package.json', 'utf8')); + p.version = process.env.VERSION; + for (const k of Object.keys(p.optionalDependencies || {})) { + p.optionalDependencies[k] = process.env.VERSION; + } + fs.writeFileSync('npm/loader/package.json', JSON.stringify(p, null, 2) + '\n'); + " + NAME="$(node -p "require('./npm/loader/package.json').name")" + if [ "$(npm view "$NAME@$VERSION" version 2>/dev/null)" = "$VERSION" ]; then + echo "✓ $NAME@$VERSION already published — skipping" + else + echo "→ $NAME@$VERSION" + (cd npm/loader && npm publish --access public) + fi + stable-immutable-release: name: Stage immutable stable release (+ verify assets) # Publish the exact versioned release and verify its complete 32-asset set @@ -2322,6 +2387,49 @@ jobs: (cd npm/nub-types && npm publish --access public --tag canary) fi + - name: Prepare and publish loader packages (idempotent) + run: | + # Mirrors publish-npm's loader step (keep the two in sync): platform + # addon packages first, root last, every publish tagged canary. + for platform in darwin-arm64 darwin-x64 linux-x64 linux-x64-musl linux-arm64 linux-arm64-musl win32-x64 win32-arm64; do + PKG="artifacts/loader-$platform" + if [ ! -d "$PKG" ]; then + echo "⚠️ Skipping loader-$platform (not built)" + continue + fi + cp "npm/loader-$platform/package.json" "$PKG/package.json" + node -e " + const fs = require('fs'); + const p = JSON.parse(fs.readFileSync('$PKG/package.json', 'utf8')); + p.version = process.env.VERSION; + fs.writeFileSync('$PKG/package.json', JSON.stringify(p, null, 2)); + " + NAME="@nubjs/loader-$platform" + if [ "$(npm view "$NAME@$VERSION" version 2>/dev/null)" = "$VERSION" ]; then + echo "✓ $NAME@$VERSION already published — skipping" + continue + fi + echo "→ $NAME@$VERSION (canary)" + (cd "$PKG" && npm publish --access public --tag canary) + done + node scripts/build-loader-npm.mjs --addon /nonexistent + node -e " + const fs = require('fs'); + const p = JSON.parse(fs.readFileSync('npm/loader/package.json', 'utf8')); + p.version = process.env.VERSION; + for (const k of Object.keys(p.optionalDependencies || {})) { + p.optionalDependencies[k] = process.env.VERSION; + } + fs.writeFileSync('npm/loader/package.json', JSON.stringify(p, null, 2) + '\n'); + " + NAME="$(node -p "require('./npm/loader/package.json').name")" + if [ "$(npm view "$NAME@$VERSION" version 2>/dev/null)" = "$VERSION" ]; then + echo "✓ $NAME@$VERSION already published — skipping" + else + echo "→ $NAME@$VERSION (canary)" + (cd npm/loader && npm publish --access public --tag canary) + fi + canary-release: name: Update rolling canary release diff --git a/site/content/docs/loader.mdx b/site/content/docs/loader.mdx new file mode 100644 index 000000000..193fb3758 --- /dev/null +++ b/site/content/docs/loader.mdx @@ -0,0 +1,62 @@ +--- +title: Loader +description: Run TypeScript under plain Node with the Nub loader — a standalone npm package registered the way tsx and ts-node are, for test runners and tools that spawn Node themselves. +--- + +Some invocations of `node` are not yours to change: a test runner, a framework CLI, a deploy script that shells out to `node` directly. The Nub loader brings Nub's TypeScript runtime to those, as a standalone npm package you register with a Node flag. + +```sh +npm install --save-dev nubjs +node --import nubjs app.ts +``` + +Any way Node accepts a preload works: + +```sh +node --import nubjs app.ts # one run +NODE_OPTIONS="--import nubjs" vitest # a tool that spawns node itself +node --require nubjs app.ts # CommonJS delivery +``` + +```json +{ + "scripts": { + "test": "mocha --import nubjs 'test/**/*.test.ts'" + } +} +``` + +## What it adds + +The loader arms the same resolve and transpile hooks the `nub` command uses, and nothing else — no polyfills, no globals, no `.env` loading: + +- Full TypeScript — `enum`, `namespace`, parameter properties, `emitDecoratorMetadata` decorators — not just the erasable subset Node's built-in stripping accepts +- JSX / TSX with the automatic runtime +- Editor-style resolution — extensionless imports, `.js → .ts` rewriting, `tsconfig.json#paths` +- Data-file imports — `.yaml`, `.toml`, `.json5`, `.jsonc`, `.txt`, and `with { type: "text" }` +- Syntax newer than the running Node, such as `using`, downleveled by the transpiler +- Inline source maps on every transpiled file + +Both module systems are covered: `import` and `require()` of a `.ts` file resolve and transpile the same way, and worker threads pick the loader up automatically because Node passes `--import` down to them. + +Dependencies under `node_modules` are never transpiled, and files Node handles natively load byte-for-byte unchanged. + +## Entry points + +```sh +node --import nubjs app.ts # ESM hooks + CommonJS require() augmentation +node --require nubjs app.ts # same, delivered as a CommonJS preload (Node 20.19+) +node --import nubjs/esm app.ts # ESM hooks only +``` + +Module formats follow Node's own rules: a `.cts` file is CommonJS and a `.mts` file is an ES module. The loader transpiles types and syntax; it does not convert one format into the other, so a `.cts` file uses `module.exports`, exactly as it would under plain Node. + +## Node support + +Node 18.19 and newer. On Node 22.15+ the hooks register synchronously in-thread through `module.registerHooks()`; on older versions they run in Node's loader worker through `module.register()`. The `--require` delivery needs `require(esm)`, so below Node 20.19 / 22.12 use `--import`. + +Platform binaries ship as `optionalDependencies` (`@nubjs/loader-*`) for macOS, Linux (glibc and musl), and Windows, on x64 and arm64. + +## The loader and the `nub` command + +Running `nub app.ts` gives you everything on this page plus [the rest of the runtime](/docs/runtime) — polyfilled web APIs, `.env` loading, Node version provisioning — with no flags. Reach for the loader when the `node` invocation itself is fixed; reach for `nub` everywhere else. The two share one transpiler and one resolver, so a file behaves the same under either. diff --git a/site/content/docs/meta.json b/site/content/docs/meta.json index c6bbfb8cb..ada5e2c6e 100644 --- a/site/content/docs/meta.json +++ b/site/content/docs/meta.json @@ -3,6 +3,7 @@ "pages": [ "index", "runtime", + "loader", "watch", "runner", "init", diff --git a/tests/loader/README.md b/tests/loader/README.md new file mode 100644 index 000000000..5e287d47f --- /dev/null +++ b/tests/loader/README.md @@ -0,0 +1,28 @@ +# Standalone loader harness + +End-to-end checks for the standalone loader package (`npm/loader`): the packed tarballs are installed into a throwaway project and each fixture runs under `node --import ` (plus `--require ` where the Node has `require(esm)`), with stdout compared to `fixtures/expected.txt`. Install-from-tarball is the point — the loader's addon discovery, its `@oxc-project/runtime` dependency, and the flat package layout are only exercised through a real install, never from the dev tree (where a sibling `runtime/addons/` masks addon-resolution bugs; one shipped that way in the first cut). + +```sh +make addon-fast # or any built runtime/addons/nub-native.node +tests/loader/run-matrix.sh # host node +NODE_VERSIONS="18.19.0 22.14.0 22.15.0 26.7.0" tests/loader/run-matrix.sh # nvm-installed versions +TSX=1 tests/loader/run-matrix.sh # differential: same fixtures under tsx +``` + +## Fixtures + +| Fixture | Exercises | +| --- | --- | +| `main.ts` | non-erasable TS (`enum`) and a type-only import — fails under plain `node`, so a pass proves the loader transpiled | +| `paths.ts` | tsconfig `paths`, an extensionless import, a YAML data import | +| `req.cts` | CommonJS `require()` of a CommonJS-content `.cts` with an `enum` | +| `using.ts` | `using` lowering — resolves the `@oxc-project/runtime` helpers from the package's real dependency | +| `worker-main.ts` | a worker thread inheriting the preload and transpiling its own `.ts` entry | + +The fixture project is `"type": "module"`, so `.ts` files with `import`/`export` are ES modules; `.cts` content must be CommonJS (`module.exports`) because the loader transpiles syntax without converting module formats. + +## Tiers + +The `--import` column is expected green on every supported Node (18.19+): 22.15+ arms sync `module.registerHooks`, older versions the `module.register` loader worker (`preload-async-hooks.mjs`). The `--require` delivery loads the arming logic through `require(esm)` and is skipped below 20.19 / 22.12. + +Known, inherited from the CLI: `require()` of an ESM-syntax `.ts` from a `.cts` crashes on 22.15–22.17 inside Node's translator (`cjsCache.get(...)`, fixed upstream in Node #60380); the CLI fails identically, so the fixtures avoid that shape. diff --git a/tests/loader/fixtures/conf.yaml b/tests/loader/fixtures/conf.yaml new file mode 100644 index 000000000..f4e431b81 --- /dev/null +++ b/tests/loader/fixtures/conf.yaml @@ -0,0 +1,4 @@ +name: demo +items: + - a + - b diff --git a/tests/loader/fixtures/expected.txt b/tests/loader/fixtures/expected.txt new file mode 100644 index 000000000..dc21f5802 --- /dev/null +++ b/tests/loader/fixtures/expected.txt @@ -0,0 +1,5 @@ +main.ts=hello world red 42 +paths.ts=via-paths hello x demo 2 +req.cts=cjs: 14 a +using.ts=in scope\ndisposed +worker-main.ts=from worker: hello thread blue diff --git a/tests/loader/fixtures/main.ts b/tests/loader/fixtures/main.ts new file mode 100644 index 000000000..1f34d281a --- /dev/null +++ b/tests/loader/fixtures/main.ts @@ -0,0 +1,6 @@ +// Non-erasable TS (enum) + a type-only import: fails under plain node, must +// transpile under the loader. +import { greet, Color } from "./util.ts"; +import type { Foo } from "./util.ts"; +const x: Foo = { n: 41 }; +console.log(greet("world"), Color.Red, x.n + 1); diff --git a/tests/loader/fixtures/package.json b/tests/loader/fixtures/package.json new file mode 100644 index 000000000..7713f546d --- /dev/null +++ b/tests/loader/fixtures/package.json @@ -0,0 +1 @@ +{ "name": "nub-loader-fixture", "private": true, "type": "module" } diff --git a/tests/loader/fixtures/paths.ts b/tests/loader/fixtures/paths.ts new file mode 100644 index 000000000..e285fc51d --- /dev/null +++ b/tests/loader/fixtures/paths.ts @@ -0,0 +1,5 @@ +// tsconfig paths alias, extensionless import, and a YAML data import. +import { fromAlias } from "@u/alias"; +import { greet } from "./util"; +import cfg from "./conf.yaml"; +console.log(fromAlias, greet("x"), cfg.name, cfg.items.length); diff --git a/tests/loader/fixtures/req.cts b/tests/loader/fixtures/req.cts new file mode 100644 index 000000000..bed4f5825 --- /dev/null +++ b/tests/loader/fixtures/req.cts @@ -0,0 +1,4 @@ +// CommonJS require() of a CommonJS-content .cts with non-erasable TS. +const { twice, Mode } = require("./util-cjs.cts"); +const v: number = 7; +console.log("cjs:", twice(v), Mode.A); diff --git a/tests/loader/fixtures/src/alias.ts b/tests/loader/fixtures/src/alias.ts new file mode 100644 index 000000000..385cc47b0 --- /dev/null +++ b/tests/loader/fixtures/src/alias.ts @@ -0,0 +1 @@ +export const fromAlias = "via-paths"; diff --git a/tests/loader/fixtures/tsconfig.json b/tests/loader/fixtures/tsconfig.json new file mode 100644 index 000000000..c1c10de20 --- /dev/null +++ b/tests/loader/fixtures/tsconfig.json @@ -0,0 +1 @@ +{ "compilerOptions": { "baseUrl": ".", "paths": { "@u/*": ["./src/*"] } } } diff --git a/tests/loader/fixtures/using.ts b/tests/loader/fixtures/using.ts new file mode 100644 index 000000000..5b3f8a4a5 --- /dev/null +++ b/tests/loader/fixtures/using.ts @@ -0,0 +1,3 @@ +// `using` lowering — exercises the @oxc-project/runtime helper resolution. +class R { [Symbol.dispose]() { console.log("disposed"); } } +{ using r = new R(); console.log("in scope"); } diff --git a/tests/loader/fixtures/util-cjs.cts b/tests/loader/fixtures/util-cjs.cts new file mode 100644 index 000000000..39d11b9b7 --- /dev/null +++ b/tests/loader/fixtures/util-cjs.cts @@ -0,0 +1,3 @@ +enum Mode { A = "a" } +const twice = (n: number): number => n * 2; +module.exports = { twice, Mode }; diff --git a/tests/loader/fixtures/util.ts b/tests/loader/fixtures/util.ts new file mode 100644 index 000000000..271b9453f --- /dev/null +++ b/tests/loader/fixtures/util.ts @@ -0,0 +1,3 @@ +export enum Color { Red = "red", Blue = "blue" } +export interface Foo { n: number } +export function greet(name: string): string { return `hello ${name}`; } diff --git a/tests/loader/fixtures/worker-child.ts b/tests/loader/fixtures/worker-child.ts new file mode 100644 index 000000000..df32cb16d --- /dev/null +++ b/tests/loader/fixtures/worker-child.ts @@ -0,0 +1,3 @@ +import { parentPort } from "node:worker_threads"; +import { greet, Color } from "./util.ts"; +parentPort!.postMessage(greet("thread") + " " + Color.Blue); diff --git a/tests/loader/fixtures/worker-main.ts b/tests/loader/fixtures/worker-main.ts new file mode 100644 index 000000000..6f9a8ac73 --- /dev/null +++ b/tests/loader/fixtures/worker-main.ts @@ -0,0 +1,4 @@ +// A worker thread inherits the preload flag and transpiles its own .ts entry. +import { Worker } from "node:worker_threads"; +const w = new Worker(new URL("./worker-child.ts", import.meta.url)); +w.on("message", (m: string) => { console.log("from worker:", m); }); diff --git a/tests/loader/run-matrix.sh b/tests/loader/run-matrix.sh new file mode 100755 index 000000000..d2f80683c --- /dev/null +++ b/tests/loader/run-matrix.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Standalone-loader matrix: pack the loader npm packages from this checkout, +# install them into a throwaway project, and run every fixture under +# `node --import ` (and `--require ` where the Node supports it) on +# each requested Node, comparing stdout to fixtures/expected.txt. +# +# tests/loader/run-matrix.sh # host node only +# NODE_VERSIONS="18.19.0 22.14.0 26.7.0" tests/loader/run-matrix.sh +# # nvm-installed versions +# TSX=1 tests/loader/run-matrix.sh # also run each fixture under tsx +# +# Needs a built addon at runtime/addons/nub-native.node (`make addon-fast`, or +# `cd crates/nub-native && cargo build --release` + copy). See README.md. +set -euo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +fixtures="$repo/tests/loader/fixtures" +addon="$repo/runtime/addons/nub-native.node" +[[ -f "$addon" ]] || { echo "missing $addon — build the addon first"; exit 2; } + +pkg_name="$(node -p "require('$repo/npm/loader/package.json').name")" +pkg_version="$(node -p "require('$repo/npm/loader/package.json').version")" + +echo "== packing $pkg_name@$pkg_version" +node "$repo/scripts/build-loader-npm.mjs" --addon "$addon" --pack >/dev/null +platform="$(node -p "require('$repo/runtime/loader-platform.cjs').platformKey()")" +root_tgz="$repo/npm/loader/$(echo "$pkg_name" | tr -d '@' | tr '/' '-')-$pkg_version.tgz" +plat_tgz="$repo/npm/loader-$platform/nubjs-loader-$platform-$pkg_version.tgz" +[[ -f "$root_tgz" && -f "$plat_tgz" ]] || { echo "pack produced no tarballs ($root_tgz, $plat_tgz)"; exit 2; } + +work="$(mktemp -d "${TMPDIR:-/tmp}/nub-loader-matrix.XXXXXX")" +trap 'rm -rf "$work"' EXIT +cp -R "$fixtures/." "$work/" +(cd "$work" && npm install --no-audit --no-fund --silent "$root_tgz" "$plat_tgz") +if [[ "${TSX:-0}" == "1" ]]; then + (cd "$work" && npm install --no-audit --no-fund --silent tsx) +fi + +# --require delivery goes through require(esm), which needs 20.19+ / 22.12+. +supports_require() { + node -e ' + const [a, b] = process.versions.node.split(".").map(Number); + process.exit((a > 22 || (a === 22 && b >= 12) || (a === 20 && b >= 19)) ? 0 : 1); + ' +} + +fail=0 +run_one() { #