telegram: 重试之间加指数退避,429 尊重服务端 retry_after(01a0e75f-f7e3)

现状:send_text 的 for _ in 0..=max_retries 重试循环在两次尝试之间零延迟,
任何 5xx/超时/429 都以最短间隔连续打 API,429 场景下反而更容易触发二次限流。

改动(ROADMAP 缺口②「Telegram 429 退避」):
- backoff_delay_ms(attempt: u8) 纯函数:起步 250ms、每轮 x2、单轮封顶 4s。
  调用方 sleep 只走这一个出口,便于单测直接断言序列而无需真实等待。
- retry_after_ms(body) 纯函数:解析 429 响应体 parameters.retry_after(秒),
  转毫秒后钳到 60s 上限;缺字段/非数字/非正/非有限值一律回退到指数退避。
- 总退避预算上限 60s:耗尽后 delay 归零不再 sleep,避免单次 send_text 长时间挂起。
- 429 单独分支,错误串改为 telegram request rate limited (429) 以便区分限流与一般失败。

语义不变:max_retries 仍 cap 5、每次请求仍受 telegram_timeout_ms 约束、
返回 Result<(), String>、错误串不泄漏 token。

测试:client() 通过 with_sleep_override 注入 no-op sleep,退避逻辑完整走通但不真实等待。
新增 3 个测试:429 端到端(mock 返回 parameters.retry_after=1,断言 2..=3 次请求)、
backoff 序列(250/500/1000/2000/4000 后封顶)、retry_after 解析与钳制。
GATE-P0 全绿:fmt --check、check --locked、clippy --locked --all-targets -D warnings、
test --locked → 52 passed / 0 failed / 1 ignored。
This commit is contained in:
anonsaber 2026-09-28 18:13:08 +08:00
parent 0ffe19cfe6
commit f4cae00adc
1 changed files with 171 additions and 10 deletions

View File

@ -24,13 +24,56 @@ pub mod telegram {
use crate::domain::Notification; use crate::domain::Notification;
use crate::state::RuntimeConfigProvider; use crate::state::RuntimeConfigProvider;
use secrecy::{ExposeSecret, SecretString}; use secrecy::{ExposeSecret, SecretString};
use serde::Serialize; use serde::{Deserialize, Serialize};
/// ⑥-a 重试退避常量。
const TELEGRAM_BACKOFF_BASE_MS: u64 = 250; // 起步
const TELEGRAM_BACKOFF_CAP_MS: u64 = 4_000; // 单轮封顶
const TELEGRAM_RETRY_AFTER_CAP_MS: u64 = 60_000; // 429 retry_after 钳到 ≤60s
const TELEGRAM_TOTAL_BACKOFF_BUDGET_MS: u64 = 60_000; // 总退避预算上限,避免失控
/// 退避序列纯函数:起步 250ms、每轮 ×2、单轮封顶 4s(⑥-a)。sleep 只调用它。
/// 入参为已完成的重试轮次,与 `max_retries` 同为 `u8`。
fn backoff_delay_ms(attempt: u8) -> u64 {
let shift = attempt.min(4);
(TELEGRAM_BACKOFF_BASE_MS << shift).min(TELEGRAM_BACKOFF_CAP_MS)
}
/// 429 响应体 `parameters.retry_after`(秒)→ 毫秒,钳到 ≤60s。
/// 缺字段 / 非数字 / 非正 → None(调用方回退到固定指数退避)。
fn retry_after_ms(body: &str) -> Option<u64> {
#[derive(Deserialize)]
struct Payload {
parameters: Parameters,
}
#[derive(Deserialize)]
struct Parameters {
retry_after: Option<f64>,
}
let payload: Payload = serde_json::from_str(body).ok()?;
let seconds = payload.parameters.retry_after?;
if !seconds.is_finite() || seconds <= 0.0 {
return None;
}
let capped = seconds.min(TELEGRAM_RETRY_AFTER_CAP_MS as f64 / 1000.0);
Some((capped * 1000.0) as u64)
}
/// 可注入的 sleep(生产恒为 tokio::time::sleep;测试覆盖为 no-op,避免真实等待)。
type SleepFn =
fn(std::time::Duration) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>;
fn default_sleep(
d: std::time::Duration,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(tokio::time::sleep(d))
}
pub struct TelegramClient { pub struct TelegramClient {
http: reqwest::Client, http: reqwest::Client,
token: SecretString, token: SecretString,
endpoint: String, endpoint: String,
runtime: RuntimeConfigProvider, runtime: RuntimeConfigProvider,
sleep: SleepFn,
} }
#[derive(Serialize)] #[derive(Serialize)]
@ -54,6 +97,7 @@ pub mod telegram {
token, token,
endpoint, endpoint,
runtime, runtime,
sleep: default_sleep,
} }
} }
@ -67,6 +111,13 @@ pub mod telegram {
Self::with_endpoint_inner(token, runtime, endpoint) Self::with_endpoint_inner(token, runtime, endpoint)
} }
/// Test-only sleep injection:覆盖为 no-op,避免退避测试真实等待(⑥-a)。
#[cfg(test)]
pub fn with_sleep_override(mut self, sleep: SleepFn) -> Self {
self.sleep = sleep;
self
}
/// Sends metadata only; body text is intentionally not accepted here. /// Sends metadata only; body text is intentionally not accepted here.
pub async fn send_notification( pub async fn send_notification(
&self, &self,
@ -87,7 +138,21 @@ pub mod telegram {
.map(|config| (config.telegram_timeout_ms, config.max_retries.min(5))) .map(|config| (config.telegram_timeout_ms, config.max_retries.min(5)))
.unwrap_or((10_000, 3)); .unwrap_or((10_000, 3));
let mut last_error = None; let mut last_error = None;
for _ in 0..=max_retries { let mut backoff_spent_ms: u64 = 0;
let mut next_retry_after_ms: Option<u64> = None;
for attempt in 0..=max_retries {
if attempt > 0 {
// ⑥-a:失败重试之间的退避(指数退避;429 时按服务器指定 retry_after)。
let delay = next_retry_after_ms
.take()
.unwrap_or_else(|| backoff_delay_ms(attempt - 1));
let delay = delay
.min(TELEGRAM_TOTAL_BACKOFF_BUDGET_MS.saturating_sub(backoff_spent_ms));
backoff_spent_ms += delay;
if delay > 0 {
(self.sleep)(std::time::Duration::from_millis(delay)).await;
}
}
let request = self let request = self
.http .http
.post(format!( .post(format!(
@ -103,9 +168,30 @@ pub mod telegram {
) )
.await .await
{ {
Ok(result) => result Ok(result) => match result {
.and_then(|response| response.error_for_status()) Ok(response)
.map_err(|_| "telegram request failed".to_owned()), if response.status() == reqwest::StatusCode::TOO_MANY_REQUESTS =>
{
// 429:读取响应体 parameters.retry_after(秒),钳 ≤60s。
// body 读取同样受 telegram_timeout_ms 约束,避免挂起。
let body = match tokio::time::timeout(
std::time::Duration::from_millis(timeout_ms.max(100)),
response.text(),
)
.await
{
Ok(Ok(body)) => body,
_ => String::new(),
};
next_retry_after_ms = retry_after_ms(&body);
Err("telegram request rate limited (429)".to_owned())
}
Ok(response) => response
.error_for_status()
.map(|_| ())
.map_err(|_| "telegram request failed".to_owned()),
Err(_) => Err("telegram request failed".to_owned()),
},
Err(_) => Err("telegram request timed out".to_owned()), Err(_) => Err("telegram request timed out".to_owned()),
}; };
match result { match result {
@ -131,6 +217,7 @@ pub mod telegram {
async fn mock_server( async fn mock_server(
status: &'static str, status: &'static str,
delay_ms: u64, delay_ms: u64,
body: &'static str,
requests: Arc<AtomicUsize>, requests: Arc<AtomicUsize>,
) -> String { ) -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@ -146,13 +233,28 @@ pub mod telegram {
if delay_ms > 0 { if delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
} }
let response = format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\n\r\n"); if body.is_empty() {
let _ = socket.write_all(response.as_bytes()).await; let response = format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\n\r\n");
let _ = socket.write_all(response.as_bytes()).await;
} else {
let response = format!(
"HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
let _ = socket.write_all(response.as_bytes()).await;
}
} }
}); });
format!("http://{address}") format!("http://{address}")
} }
/// 测试用 no-op sleep:退避逻辑完整走通但不真实等待(⑥-a)。
fn instant_sleep(
_: std::time::Duration,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async {})
}
fn client(endpoint: String, retries: u8, timeout_ms: u64) -> TelegramClient { fn client(endpoint: String, retries: u8, timeout_ms: u64) -> TelegramClient {
let runtime = runtime_provider(OutboundConfig { let runtime = runtime_provider(OutboundConfig {
telegram_timeout_ms: timeout_ms, telegram_timeout_ms: timeout_ms,
@ -160,12 +262,13 @@ pub mod telegram {
..OutboundConfig::default() ..OutboundConfig::default()
}); });
TelegramClient::with_endpoint(SecretString::new("test-token".into()), runtime, endpoint) TelegramClient::with_endpoint(SecretString::new("test-token".into()), runtime, endpoint)
.with_sleep_override(instant_sleep)
} }
#[tokio::test] #[tokio::test]
async fn mock_endpoint_success_sends_message() { async fn mock_endpoint_success_sends_message() {
let requests = Arc::new(AtomicUsize::new(0)); let requests = Arc::new(AtomicUsize::new(0));
let endpoint = mock_server("200 OK", 0, requests.clone()).await; let endpoint = mock_server("200 OK", 0, "", requests.clone()).await;
client(endpoint, 0, 1_000) client(endpoint, 0, 1_000)
.send_text(42, "hello") .send_text(42, "hello")
.await .await
@ -176,7 +279,7 @@ pub mod telegram {
#[tokio::test] #[tokio::test]
async fn error_response_retries_with_bound() { async fn error_response_retries_with_bound() {
let requests = Arc::new(AtomicUsize::new(0)); let requests = Arc::new(AtomicUsize::new(0));
let endpoint = mock_server("500 Internal Server Error", 0, requests.clone()).await; let endpoint = mock_server("500 Internal Server Error", 0, "", requests.clone()).await;
assert!(client(endpoint, 2, 1_000) assert!(client(endpoint, 2, 1_000)
.send_text(42, "hello") .send_text(42, "hello")
.await .await
@ -187,7 +290,7 @@ pub mod telegram {
#[tokio::test] #[tokio::test]
async fn timeout_retries_with_bound() { async fn timeout_retries_with_bound() {
let requests = Arc::new(AtomicUsize::new(0)); let requests = Arc::new(AtomicUsize::new(0));
let endpoint = mock_server("200 OK", 250, requests.clone()).await; let endpoint = mock_server("200 OK", 250, "", requests.clone()).await;
assert!(client(endpoint, 1, 100) assert!(client(endpoint, 1, 100)
.send_text(42, "hello") .send_text(42, "hello")
.await .await
@ -196,5 +299,63 @@ pub mod telegram {
// second connection before the tiny test server accepts it. // second connection before the tiny test server accepts it.
assert!(requests.load(Ordering::Relaxed) >= 1); assert!(requests.load(Ordering::Relaxed) >= 1);
} }
#[tokio::test]
async fn rate_limited_429_retries_until_max_attempts() {
let requests = Arc::new(AtomicUsize::new(0));
let endpoint = mock_server(
"429 Too Many Requests",
0,
r#"{"ok":false,"parameters":{"retry_after":1}}"#,
requests.clone(),
)
.await;
let result = client(endpoint, 2, 1_000).send_text(42, "hello").await;
assert!(result.is_err());
// max_retries=2 → 最多 max_retries+1 = 3 次请求;must have retried at least once.
let n = requests.load(Ordering::Relaxed);
assert!((2..=3).contains(&n), "expected 2..=3 attempts, got {n}");
}
#[test]
fn backoff_delay_sequence_grows_exponentially_then_caps() {
assert_eq!(backoff_delay_ms(0), 250);
assert_eq!(backoff_delay_ms(1), 500);
assert_eq!(backoff_delay_ms(2), 1_000);
assert_eq!(backoff_delay_ms(3), 2_000);
assert_eq!(backoff_delay_ms(4), 4_000);
assert_eq!(backoff_delay_ms(5), 4_000);
assert_eq!(backoff_delay_ms(100), 4_000);
}
#[test]
fn retry_after_parses_and_clamps() {
// 正常值:1s → 1000ms
assert_eq!(
retry_after_ms(r#"{"parameters":{"retry_after":1}}"#),
Some(1_000)
);
// 缺字段
assert_eq!(retry_after_ms(r#"{}"#), None);
assert_eq!(retry_after_ms(r#"{"parameters":{}}"#), None);
assert_eq!(
retry_after_ms(r#"{"parameters":{"retry_after":null}}"#),
None
);
// 非数字
assert_eq!(
retry_after_ms(r#"{"parameters":{"retry_after":"1"}}"#),
None
);
assert_eq!(retry_after_ms(r#"not json"#), None);
// 超大值 → 钳到 60s 上限
assert_eq!(
retry_after_ms(r#"{"parameters":{"retry_after":999999}}"#),
Some(60_000)
);
// 非正 → None(回退到固定指数退避)
assert_eq!(retry_after_ms(r#"{"parameters":{"retry_after":0}}"#), None);
assert_eq!(retry_after_ms(r#"{"parameters":{"retry_after":-3}}"#), None);
}
} }
} }