fix(lb): 可观测头写入不可变响应头不再把健康的 200 变成 503
mark() 往代理响应上写 `x-lb-backend`,但跨域且无 CORS 允许头的响应其 headers guard 是 immutable,`set` 抛 `TypeError: Can't modify immutable headers.`;异常冒泡到 forwardWithFailover 的 catch 后被判成后端故障 (`backend ... unavailable (TypeError)`),重试耗尽返回 503。 后端一直健康:GET / 实际返回 200。/healthz 走 healthResponse() 自建 响应、不调 mark(),所以不受影响——这也正是它掩盖问题多年的原因。 mark() 现在在响应头不可变时静默放弃观测头,绝不阻塞响应返回;补两条 回归测试(2xx 不可变响应必须原样返回且不触发故障转移;5xx 不可变响应 仍正常故障转移)。 验证:wrangler dev --local 下 / /healthz /ready /api/status /assets/* 全部 200(修复前 / 与 /api/status 为 503);node --test 34/34 通过。
This commit is contained in:
parent
3d6606d0cd
commit
656034d71d
|
|
@ -139,8 +139,21 @@ export async function forwardWithFailover(origins, request, opts) {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/** 在响应上加轻量可观测头(不改 body/状态语义;origin 非敏感,含在响应头中便于排障)。 */
|
/**
|
||||||
|
* 在响应上加轻量可观测头(不改 body/状态语义;origin 非敏感,含在响应头中便于排障)。
|
||||||
|
*
|
||||||
|
* ⚠️ 跨域且不带 CORS 允许头的响应,其 `headers` 的 guard 是 immutable,`set` 会抛
|
||||||
|
* `TypeError: Can't modify immutable headers.`(workerd 实测:后端 `/` 返回 200 但无
|
||||||
|
* `access-control-allow-origin` 即触发)。LB 代理的响应几乎都属于这种——若让该异常冒泡到
|
||||||
|
* `forwardWithFailover` 的 catch,一个健康的 200 会被记成 `backend ... unavailable (TypeError)`
|
||||||
|
* 并耗尽重试、返回 503(2026-09-30 生产事故根因)。观测头是尽力而为的元数据,
|
||||||
|
* 不可变时静默放弃,绝不阻塞响应返回。
|
||||||
|
*/
|
||||||
function mark(resp, origin) {
|
function mark(resp, origin) {
|
||||||
resp.headers.set("x-lb-backend", origin);
|
try {
|
||||||
|
resp.headers.set("x-lb-backend", origin);
|
||||||
|
} catch {
|
||||||
|
/* 响应 headers 不可变(跨域且无 CORS)→ 放弃观测头,照常返回响应。 */
|
||||||
|
}
|
||||||
return resp;
|
return resp;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import { forwardWithFailover, joinUrl, isRetryableError } from "../src/lb.js";
|
||||||
* 可控 fetch mock:steps[i] 每次调用消费一个;
|
* 可控 fetch mock:steps[i] 每次调用消费一个;
|
||||||
* - {status, body?} → 返回该 Response;
|
* - {status, body?} → 返回该 Response;
|
||||||
* - {error: Error} → 抛出。
|
* - {error: Error} → 抛出。
|
||||||
|
* - {response: Response} → 原样返回该对象(用于模拟 workerd 特有行为,如不可变响应头);
|
||||||
*/
|
*/
|
||||||
function mockFetch(steps) {
|
function mockFetch(steps) {
|
||||||
const calls = [];
|
const calls = [];
|
||||||
|
|
@ -18,6 +19,7 @@ function mockFetch(steps) {
|
||||||
calls.push({ url, init });
|
calls.push({ url, init });
|
||||||
const step = steps[Math.min(i, steps.length - 1)];
|
const step = steps[Math.min(i, steps.length - 1)];
|
||||||
if (step.error) throw step.error;
|
if (step.error) throw step.error;
|
||||||
|
if (step.response) return step.response;
|
||||||
return new Response(step.body ?? "ok", { status: step.status, headers: { "content-type": "text/plain" } });
|
return new Response(step.body ?? "ok", { status: step.status, headers: { "content-type": "text/plain" } });
|
||||||
};
|
};
|
||||||
return { fetch, calls };
|
return { fetch, calls };
|
||||||
|
|
@ -26,6 +28,24 @@ function mockFetch(steps) {
|
||||||
/** 测试用计时器:不真触发超时。 */
|
/** 测试用计时器:不真触发超时。 */
|
||||||
const noOpTimers = { setTimeout: () => 0, clearTimeout: () => {} };
|
const noOpTimers = { setTimeout: () => 0, clearTimeout: () => {} };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 模拟「不可变响应头」:跨域且不带 CORS 允许头的响应,其 headers guard 是 immutable,
|
||||||
|
* 任何写操作都抛 `TypeError: Can't modify immutable headers.`(workerd 实测行为)。
|
||||||
|
* 生产 LB 代理后端 `/` 正是这种响应。
|
||||||
|
*/
|
||||||
|
function immutableResponse(status, body = "healthy") {
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
headers: {
|
||||||
|
get() { return null; },
|
||||||
|
set() { throw new TypeError("Can't modify immutable headers."); },
|
||||||
|
append() { throw new TypeError("Can't modify immutable headers."); },
|
||||||
|
},
|
||||||
|
body: undefined,
|
||||||
|
text: async () => body,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function req(path = "/reconcile") {
|
function req(path = "/reconcile") {
|
||||||
return new Request(`https://lb.example${path}`, {
|
return new Request(`https://lb.example${path}`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
|
|
@ -70,6 +90,38 @@ test("transparent passthrough forwards headers + body + method (SAF-LB-PASSTHRU)
|
||||||
assert.equal(res.headers.get("x-lb-backend"), "https://a.example");
|
assert.equal(res.headers.get("x-lb-backend"), "https://a.example");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("immutable response headers never mask a healthy 2xx as 503 (跨域无 CORS)", async () => {
|
||||||
|
// 回归:2026-09-30 生产事故。后端返回健康的 200,但该响应跨域且无 CORS 头,
|
||||||
|
// headers guard 为 immutable;mark() 写 `x-lb-backend` 抛 TypeError,被外层 catch
|
||||||
|
// 记成 `backend ... unavailable (TypeError)` → 重试耗尽 → 503。/healthz 自建响应
|
||||||
|
// 不调 mark(),所以一直正常,掩盖了问题。
|
||||||
|
const { fetch, calls } = mockFetch([{ response: immutableResponse(200) }]);
|
||||||
|
const res = await forwardWithFailover(["https://a.example"], req("/"), {
|
||||||
|
...noOpTimers,
|
||||||
|
fetch,
|
||||||
|
maxAttempts: 2,
|
||||||
|
timeoutMs: 5000,
|
||||||
|
rng: () => 0,
|
||||||
|
});
|
||||||
|
assert.equal(res.status, 200, "healthy 200 must be returned as-is");
|
||||||
|
assert.equal(calls.length, 1, "a healthy response must not trigger failover");
|
||||||
|
assert.equal(await res.text(), "healthy");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("immutable headers are tolerated on the 5xx failover path too", async () => {
|
||||||
|
const { fetch, calls } = mockFetch([{ response: immutableResponse(503) }, { status: 200, body: "ok" }]);
|
||||||
|
const res = await forwardWithFailover(["https://a.example", "https://b.example"], req("/"), {
|
||||||
|
...noOpTimers,
|
||||||
|
fetch,
|
||||||
|
maxAttempts: 2,
|
||||||
|
timeoutMs: 5000,
|
||||||
|
rng: () => 0,
|
||||||
|
});
|
||||||
|
assert.equal(res.status, 200, "503 with immutable headers still fails over");
|
||||||
|
assert.equal(calls.length, 2);
|
||||||
|
assert.equal(res.headers.get("x-lb-backend"), "https://b.example", "marker still added on a mutable response");
|
||||||
|
});
|
||||||
|
|
||||||
test("GET/HEAD do not forward a body", async () => {
|
test("GET/HEAD do not forward a body", async () => {
|
||||||
const { fetch, calls } = mockFetch([{ status: 200 }]);
|
const { fetch, calls } = mockFetch([{ status: 200 }]);
|
||||||
const get = new Request("https://lb.example/ready", { method: "GET", headers: { authorization: "Bearer x" } });
|
const get = new Request("https://lb.example/ready", { method: "GET", headers: { authorization: "Bearer x" } });
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue