fix(lb): 可观测头写入不可变响应头不再把健康的 200 变成 503

mark() 往代理响应上写 `x-lb-backend`,但跨域且无 CORS 允许头的响应其
headers guard 是 immutable,`set` 抛 `TypeError: Can't modify immutable
headers.`;异常冒泡到 forwardWithFailover 的 catch 后被判成后端故障
(`backend ... unavailable (TypeError)`),重试耗尽返回 503。

后端一直健康:GET / 实际返回 200。/healthz 走 healthResponse() 自建
响应、不调 mark(),所以不受影响——这也正是它掩盖问题多年的原因。

mark() 现在在响应头不可变时静默放弃观测头,绝不阻塞响应返回;补两条
回归测试(2xx 不可变响应必须原样返回且不触发故障转移;5xx 不可变响应
仍正常故障转移)。

验证:wrangler dev --local 下 / /healthz /ready /api/status /assets/*
全部 200(修复前 / 与 /api/status 为 503);node --test 34/34 通过。
This commit is contained in:
anonsaber 2026-09-30 20:02:18 +08:00
parent 3d6606d0cd
commit 656034d71d
2 changed files with 67 additions and 2 deletions

View File

@ -139,8 +139,21 @@ export async function forwardWithFailover(origins, request, opts) {
});
}
/** 在响应上加轻量可观测头(不改 body/状态语义;origin 非敏感,含在响应头中便于排障)。 */
/**
* 在响应上加轻量可观测头(不改 body/状态语义;origin 非敏感,含在响应头中便于排障)。
*
* ⚠️ 跨域且不带 CORS 允许头的响应,其 `headers` 的 guard 是 immutable,`set` 会抛
* `TypeError: Can't modify immutable headers.`(workerd 实测:后端 `/` 返回 200 但无
* `access-control-allow-origin` 即触发)。LB 代理的响应几乎都属于这种——若让该异常冒泡到
* `forwardWithFailover` 的 catch,一个健康的 200 会被记成 `backend ... unavailable (TypeError)`
* 并耗尽重试、返回 503(2026-09-30 生产事故根因)。观测头是尽力而为的元数据,
* 不可变时静默放弃,绝不阻塞响应返回。
*/
function mark(resp, origin) {
resp.headers.set("x-lb-backend", origin);
try {
resp.headers.set("x-lb-backend", origin);
} catch {
/* 响应 headers 不可变(跨域且无 CORS)→ 放弃观测头,照常返回响应。 */
}
return resp;
}

View File

@ -10,6 +10,7 @@ import { forwardWithFailover, joinUrl, isRetryableError } from "../src/lb.js";
* 可控 fetch mock:steps[i] 每次调用消费一个;
* - {status, body?} → 返回该 Response;
* - {error: Error} → 抛出。
* - {response: Response} → 原样返回该对象(用于模拟 workerd 特有行为,如不可变响应头);
*/
function mockFetch(steps) {
const calls = [];
@ -18,6 +19,7 @@ function mockFetch(steps) {
calls.push({ url, init });
const step = steps[Math.min(i, steps.length - 1)];
if (step.error) throw step.error;
if (step.response) return step.response;
return new Response(step.body ?? "ok", { status: step.status, headers: { "content-type": "text/plain" } });
};
return { fetch, calls };
@ -26,6 +28,24 @@ function mockFetch(steps) {
/** 测试用计时器:不真触发超时。 */
const noOpTimers = { setTimeout: () => 0, clearTimeout: () => {} };
/**
* 模拟「不可变响应头」:跨域且不带 CORS 允许头的响应,其 headers guard 是 immutable,
* 任何写操作都抛 `TypeError: Can't modify immutable headers.`(workerd 实测行为)。
* 生产 LB 代理后端 `/` 正是这种响应。
*/
function immutableResponse(status, body = "healthy") {
return {
status,
headers: {
get() { return null; },
set() { throw new TypeError("Can't modify immutable headers."); },
append() { throw new TypeError("Can't modify immutable headers."); },
},
body: undefined,
text: async () => body,
};
}
function req(path = "/reconcile") {
return new Request(`https://lb.example${path}`, {
method: "POST",
@ -70,6 +90,38 @@ test("transparent passthrough forwards headers + body + method (SAF-LB-PASSTHRU)
assert.equal(res.headers.get("x-lb-backend"), "https://a.example");
});
test("immutable response headers never mask a healthy 2xx as 503 (跨域无 CORS)", async () => {
// 回归:2026-09-30 生产事故。后端返回健康的 200,但该响应跨域且无 CORS 头,
// headers guard 为 immutable;mark() 写 `x-lb-backend` 抛 TypeError,被外层 catch
// 记成 `backend ... unavailable (TypeError)` → 重试耗尽 → 503。/healthz 自建响应
// 不调 mark(),所以一直正常,掩盖了问题。
const { fetch, calls } = mockFetch([{ response: immutableResponse(200) }]);
const res = await forwardWithFailover(["https://a.example"], req("/"), {
...noOpTimers,
fetch,
maxAttempts: 2,
timeoutMs: 5000,
rng: () => 0,
});
assert.equal(res.status, 200, "healthy 200 must be returned as-is");
assert.equal(calls.length, 1, "a healthy response must not trigger failover");
assert.equal(await res.text(), "healthy");
});
test("immutable headers are tolerated on the 5xx failover path too", async () => {
const { fetch, calls } = mockFetch([{ response: immutableResponse(503) }, { status: 200, body: "ok" }]);
const res = await forwardWithFailover(["https://a.example", "https://b.example"], req("/"), {
...noOpTimers,
fetch,
maxAttempts: 2,
timeoutMs: 5000,
rng: () => 0,
});
assert.equal(res.status, 200, "503 with immutable headers still fails over");
assert.equal(calls.length, 2);
assert.equal(res.headers.get("x-lb-backend"), "https://b.example", "marker still added on a mutable response");
});
test("GET/HEAD do not forward a body", async () => {
const { fetch, calls } = mockFetch([{ status: 200 }]);
const get = new Request("https://lb.example/ready", { method: "GET", headers: { authorization: "Bearer x" } });