Perf/decode response encodings

packages/safe-chain/src/registryProxy/decodeBody.js

@@ -0,0 +1,59 @@
+import { gunzip } from "zlib";
+import { promisify } from "util";
+
+// Guards against decompression bombs; the largest npm packuments are ~32MB.
+export const MAX_DECODED_BODY_BYTES = 256 * 1024 * 1024;
+
+const gunzipAsync = promisify(gunzip);
+
+/**
+ * @param {Buffer} buffer
+ * @returns {Promise<Buffer>}
+ */
+function decodeGzip(buffer) {
+  return gunzipAsync(buffer, { maxOutputLength: MAX_DECODED_BODY_BYTES });
+}
+
+/**
+ * Only gzip: npm's br responses are no smaller, and each decoder is attack surface.
+ * @type {Record<string, (buffer: Buffer) => Promise<Buffer>>}
+ */
+const decoders = {
+  gzip: decodeGzip,
+  "x-gzip": decodeGzip,
+};
+
+/**
+ * Decodes a response body off the event loop.
+ * @param {Buffer} body
+ * @param {string | string[] | undefined} contentEncoding
+ * @returns {Promise<Buffer>}
+ */
+export async function decodeBody(body, contentEncoding) {
+  const encoding = String(contentEncoding ?? "identity").trim().toLowerCase();
+  if (encoding === "identity" || encoding === "") {
+    return body;
+  }
+
+  if (!Object.hasOwn(decoders, encoding)) {
+    throw new Error(`Unsupported content-encoding: ${encoding}`);
+  }
+
+  return decoders[encoding](body);
+}
+
+/**
+ * Narrows a client's Accept-Encoding to the offered encodings decodeBody supports.
+ * @param {string | string[] | undefined} acceptEncoding
+ * @returns {string}
+ */
+export function supportedAcceptEncoding(acceptEncoding) {
+  const offered = String(acceptEncoding ?? "")
+    .split(",")
+    .map((token) => token.trim())
+    .filter(Boolean);
+  const supported = offered.filter((token) =>
+    Object.hasOwn(decoders, token.split(";")[0].trim().toLowerCase())
+  );
+  return supported.length > 0 ? supported.join(", ") : "identity";
+}

packages/safe-chain/src/registryProxy/decodeBody.spec.js

@@ -0,0 +1,81 @@
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import zlib from "node:zlib";
+import {
+  decodeBody,
+  supportedAcceptEncoding,
+  MAX_DECODED_BODY_BYTES,
+} from "./decodeBody.js";
+
+describe("decodeBody", () => {
+  const body = Buffer.from('{"name":"pkg"}');
+
+  for (const [encoding, encode] of [
+    ["gzip", zlib.gzipSync],
+    ["x-gzip", zlib.gzipSync],
+  ]) {
+    it(`decodes ${encoding}`, async () => {
+      const decoded = await decodeBody(encode(body), encoding);
+
+      assert.strictEqual(decoded.toString(), body.toString());
+    });
+  }
+
+  it("returns uncompressed bodies as the same buffer", async () => {
+    assert.strictEqual(await decodeBody(body, undefined), body);
+    assert.strictEqual(await decodeBody(body, "identity"), body);
+  });
+
+  it("rejects encodings it cannot decode, including inherited property names", async () => {
+    await assert.rejects(decodeBody(body, "zstd"), /Unsupported/);
+    await assert.rejects(decodeBody(body, "constructor"), /Unsupported/);
+    await assert.rejects(decodeBody(body, "gzip, br"), /Unsupported/);
+  });
+
+  it("does not decode br or deflate", async () => {
+    await assert.rejects(
+      decodeBody(zlib.brotliCompressSync(body), "br"),
+      /Unsupported/
+    );
+    await assert.rejects(
+      decodeBody(zlib.deflateSync(body), "deflate"),
+      /Unsupported/
+    );
+  });
+
+  it("refuses to expand a body beyond the decoded size limit", async () => {
+    // Concatenated gzip members: a tiny body that decodes past the limit.
+    const oneMegabyte = 1024 * 1024;
+    const member = zlib.gzipSync(Buffer.alloc(oneMegabyte));
+    const bomb = Buffer.concat(
+      Array(MAX_DECODED_BODY_BYTES / oneMegabyte + 1).fill(member)
+    );
+
+    await assert.rejects(decodeBody(bomb, "gzip"), {
+      code: "ERR_BUFFER_TOO_LARGE",
+    });
+  });
+});
+
+describe("supportedAcceptEncoding", () => {
+  it("keeps only offered encodings that can be decoded", () => {
+    assert.strictEqual(
+      supportedAcceptEncoding("gzip, deflate, br, zstd, x-gzip"),
+      "gzip, x-gzip"
+    );
+  });
+
+  it("keeps quality values on the encodings it keeps", () => {
+    assert.strictEqual(
+      supportedAcceptEncoding("zstd;q=1.0, br;q=0.9, gzip;q=0.5"),
+      "gzip;q=0.5"
+    );
+  });
+
+  it("falls back to identity when nothing offered can be decoded", () => {
+    assert.strictEqual(supportedAcceptEncoding("zstd"), "identity");
+    assert.strictEqual(supportedAcceptEncoding("br, deflate"), "identity");
+    assert.strictEqual(supportedAcceptEncoding(undefined), "identity");
+    assert.strictEqual(supportedAcceptEncoding("constructor"), "identity");
+  });
+});

packages/safe-chain/src/registryProxy/mitmRequestHandler.js

@@ -2,7 +2,7 @@ import https from "https";
 import { generateCertForHost } from "./certUtils.js";
 import { HttpsProxyAgent } from "https-proxy-agent";
 import { ui } from "../environment/userInteraction.js";
-import { gunzipSync } from "zlib";
+import { decodeBody, supportedAcceptEncoding } from "./decodeBody.js";
 import { omitHeaders } from "./http-utils.js";
 
 /**
@@ -186,6 +186,11 @@ function createProxyRequest(hostname, port, req, res, requestHandler) {
     delete headers.host;
   }
   headers = requestHandler.modifyRequestHeaders(headers);
+  if (headers && requestHandler.modifiesResponse()) {
+    headers["accept-encoding"] = supportedAcceptEncoding(
+      headers["accept-encoding"]
+    );
+  }
 
   /** @type {import("http").RequestOptions} */
   const options = {
@@ -229,16 +234,15 @@ function createProxyRequest(hostname, port, req, res, requestHandler) {
 
       proxyRes.on("data", (chunk) => chunks.push(chunk));
 
-      proxyRes.on("end", () => {
+      proxyRes.on("end", async () => {
         /** @type {Buffer} */
         const originalBuffer = Buffer.concat(chunks);
 
         try {
-          let decodedBuffer = originalBuffer;
-
-          if (proxyRes.headers["content-encoding"] === "gzip") {
-            decodedBuffer = gunzipSync(originalBuffer);
-          }
+          const decodedBuffer = await decodeBody(
+            originalBuffer,
+            proxyRes.headers["content-encoding"]
+          );
 
           const modifiedBuffer = requestHandler.modifyBody(decodedBuffer, headers);
 

packages/safe-chain/src/registryProxy/mitmRequestHandler.spec.js

@@ -5,6 +5,15 @@ import zlib from "node:zlib";
 describe("mitmRequestHandler", async () => {
   let capturedHandler;
   let capturedOptions;
+  let upstreamEncoding = "gzip";
+  let upstreamBody = Buffer.from("rewritten body");
+
+  const encoders = {
+    gzip: zlib.gzipSync,
+    br: zlib.brotliCompressSync,
+    deflate: zlib.deflateSync,
+    zstd: (buffer) => buffer,
+  };
 
   mock.module("https", {
     defaultExport: {
@@ -22,7 +31,7 @@ describe("mitmRequestHandler", async () => {
         const proxyRes = {
           statusCode: 200,
           headers: {
-            "content-encoding": "gzip",
+            "content-encoding": upstreamEncoding,
             "content-length": "999",
             "transfer-encoding": "chunked",
           },
@@ -37,8 +46,7 @@ describe("mitmRequestHandler", async () => {
           on: () => {},
           write: () => {},
           end: () => {
-            const payload = Buffer.from("rewritten body");
-            listeners["data"]?.(zlib.gzipSync(payload));
+            listeners["data"]?.(encoders[upstreamEncoding](upstreamBody));
             listeners["end"]?.();
           },
           destroy: () => {},
@@ -103,6 +111,8 @@ describe("mitmRequestHandler", async () => {
       body: undefined,
     };
 
+    let responseEnded;
+    const ended = new Promise((resolve) => (responseEnded = resolve));
     const res = {
       headersSent: false,
       writeHead: (statusCode, headers) => {
@@ -111,6 +121,7 @@ describe("mitmRequestHandler", async () => {
       },
       end: (body) => {
         resState.body = body;
+        responseEnded();
       },
     };
 
@@ -126,6 +137,7 @@ describe("mitmRequestHandler", async () => {
     };
 
     await capturedHandler(request, res);
+    await ended;
 
     assert.equal(capturedOptions.hostname, "pypi.org");
     assert.equal(resState.statusCode, 200);
@@ -167,6 +179,8 @@ describe("mitmRequestHandler", async () => {
       body: undefined,
     };
 
+    let responseEnded;
+    const ended = new Promise((resolve) => (responseEnded = resolve));
     const res = {
       headersSent: false,
       writeHead: (statusCode, headers) => {
@@ -175,6 +189,7 @@ describe("mitmRequestHandler", async () => {
       },
       end: (body) => {
         resState.body = body;
+        responseEnded();
       },
     };
 
@@ -190,6 +205,7 @@ describe("mitmRequestHandler", async () => {
     };
 
     await capturedHandler(request, res);
+    await ended;
 
     // Caching/encoding headers are preserved so npm and the registry can keep
     // serving the response from cache instead of re-reading on the next install.
@@ -200,4 +216,90 @@ describe("mitmRequestHandler", async () => {
     // The body is forwarded still-compressed, exactly as received from upstream.
     assert.deepEqual(resState.body, zlib.gzipSync(Buffer.from("rewritten body")));
   });
+
+  /**
+   * Runs one intercepted request whose upstream response uses `encoding`, and
+   * returns the body the interceptor saw plus what was sent to the client.
+   */
+  async function interceptWithEncoding(encoding, modifyBody) {
+    upstreamEncoding = encoding;
+    upstreamBody = Buffer.from('{"name":"pkg"}');
+    let seenByInterceptor;
+    const interceptor = {
+      handleRequest: async () => ({
+        blockResponse: undefined,
+        modifyRequestHeaders: (headers) => headers,
+        modifiesResponse: () => true,
+        modifyBody: (body) => {
+          seenByInterceptor = body;
+          return modifyBody(body);
+        },
+      }),
+    };
+
+    mitmConnect(
+      { url: "registry.npmjs.org:443" },
+      { on: () => {}, write: () => {}, writable: true, end: () => {} },
+      interceptor
+    );
+
+    const sent = {};
+    const ended = new Promise((resolve) => {
+      const res = {
+        headersSent: false,
+        writeHead: (statusCode, headers) => {
+          sent.headers = headers;
+        },
+        end: (body) => {
+          sent.body = body;
+          resolve();
+        },
+      };
+      capturedHandler(
+        {
+          url: "/pkg",
+          headers: {},
+          method: "GET",
+          on: (event, handler) => event === "end" && handler(),
+        },
+        res
+      );
+    });
+    await ended;
+
+    upstreamEncoding = "gzip";
+    upstreamBody = Buffer.from("rewritten body");
+    return { seenByInterceptor, sent };
+  }
+
+  it("decodes a gzip response before the interceptor sees it", async () => {
+    const { seenByInterceptor, sent } = await interceptWithEncoding(
+      "gzip",
+      () => Buffer.from('{"name":"rewritten"}')
+    );
+
+    assert.equal(seenByInterceptor.toString(), '{"name":"pkg"}');
+    assert.equal(sent.headers["content-encoding"], undefined);
+    assert.equal(sent.body.toString(), '{"name":"rewritten"}');
+  });
+
+  for (const encoding of ["zstd", "br", "deflate"]) {
+    it(`forwards a ${encoding} response unmodified since it is not decoded`, async () => {
+      let interceptorCalled = false;
+      const { sent } = await interceptWithEncoding(encoding, (body) => {
+        interceptorCalled = true;
+        return body;
+      });
+
+      assert.equal(interceptorCalled, false);
+      assert.equal(sent.headers["content-encoding"], encoding);
+      assert.deepEqual(sent.body, encoders[encoding](Buffer.from('{"name":"pkg"}')));
+    });
+  }
+
+  it("only offers upstream the encodings it can decode when it may read the response", async () => {
+    await interceptWithEncoding("gzip", (body) => body);
+
+    assert.equal(capturedOptions.headers["accept-encoding"], "identity");
+  });
 });