MessageWeave/src/state.rs

2040 lines
76 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! SPLIT-EVAL: 已评估暂缓拆分——会话令牌、幂等去重、可靠队列与限流共用同一组 Redis key 约定、加密封装与错误类型,拆分会让 key 命名规范分散到多处而失去单点审阅。
//! Redis-only reliability primitives (C-REDIS-ONLY-STATE, MOD-DEDUP, MOD-STREAMS).
//! These APIs are deliberately short-request operations; no subscriptions or polling.
use crate::config::BusinessConfig;
use async_trait::async_trait;
use ring::{
aead,
rand::{SecureRandom, SystemRandom},
};
use std::{
collections::{HashMap, HashSet},
sync::{Arc, Mutex, RwLock},
};
use thiserror::Error;
/// Every TTL literal in the codebase, in seconds.
///
/// The key/value table these back is `docs/opengaps.md`. The contract is pinned
/// by `ttl_contract_is_pinned`, and the three atomic primitives that actually
/// issue `EX`/`EXPIRE` are asserted against real Redis by
/// `real_redis_ttl_tests`.
///
/// An inline literal at a call site would compile and pass the suite if it
/// drifted to a wrong-but-valid number, silently shrinking the dedup, lock or
/// replay window the email → Telegram chain leans on. Naming the intent here
/// turns that drift into a reviewable const change plus a failing test.
pub(crate) mod ttl {
pub const MINUTE: u64 = 60;
pub const HOUR: u64 = 60 * MINUTE;
pub const DAY: u64 = 24 * HOUR;
pub const WEEK: u64 = 7 * DAY;
pub const YEAR: u64 = 365 * DAY;
/// Initial hold on the reconcile job. Kept short: if the process crashes
/// before the first heartbeat fires, the lock expires in ≤60 s instead of
/// the previous 300 s. The heartbeat renews to `RECONCILE_HEARTBEAT_SECONDS`
/// well before this expires.
pub const RECONCILE_LOCK_SECONDS: u64 = 60;
/// Heartbeat renewal TTL. Must stay well under `RECONCILE_LOCK_SECONDS` and
/// be at least `RECONCILE_HEARTBEAT_INTERVAL_SECONDS × 2` (one missed beat
/// tolerated). After the first heartbeat the worst-case expiry drops to
/// this value.
pub const RECONCILE_HEARTBEAT_SECONDS: u64 = 30;
/// Interval between heartbeat renewals. Short enough that the first renewal
/// fires well before `RECONCILE_LOCK_SECONDS` expires.
pub const RECONCILE_HEARTBEAT_INTERVAL_SECONDS: u64 = 10;
/// Single-flight lock for JMAP push registration. Outlives the maximum JMAP
/// request timeout so a slow create cannot admit a duplicate.
pub const PUSH_REGISTER_LOCK_SECONDS: u64 = 360;
/// Verification rate limit per push subscription.
pub const PUSH_VERIFY_LIMIT_SECONDS: u64 = 30;
/// Telegram update de-duplication window.
pub const DEDUP_TG_SECONDS: u64 = DAY;
/// JMAP de-duplication window. Callback and reconcile share it, so a
/// re-baseline replay duplicates at most once instead of being dropped.
pub const DEDUP_JMAP_SECONDS: u64 = DAY;
/// In-flight delivery guard.
pub const DELIVERY_INFLIGHT_SECONDS: u64 = MINUTE;
/// Delivery idempotency window.
pub const DELIVERY_COMMITTED_SECONDS: u64 = WEEK;
/// How long a `disabled` push subscription stays marked before re-check.
pub const PUSH_DISABLED_SECONDS: u64 = DAY;
/// How long a `pending` push subscription waits for verification.
pub const PUSH_STATUS_PENDING_SECONDS: u64 = 900;
/// How long a `verified` push subscription stays trusted before re-check.
pub const PUSH_STATUS_VERIFIED_SECONDS: u64 = 300;
/// Push subscription code and its link to the subscription id.
pub const PUSH_SUBSCRIPTION_SECONDS: u64 = 300;
/// Callback-URL -> subscription-id registration window.
pub const PUSH_REGISTRATION_SECONDS: u64 = WEEK;
/// Orphan callback attribution window: a callback with an unknown
/// `subscription_id` waits here to be re-attribute after re-registration.
pub const PUSH_ORPHAN_SECONDS: u64 = WEEK;
/// `/admin` digest session. Must match the `expires_in` reported to the
/// admin client.
pub const ADMIN_SESSION_SECONDS: u64 = 1_800;
/// Retry counter retention. `retry_or_dlq` interpolates this into its Lua
/// script, so there is exactly one literal for it.
pub const RETRY_COUNTER_SECONDS: u64 = DAY;
/// `/ai` "临时 / 一次" grants and the explicit `/ai on` grants.
pub const CONSENT_TEMPORARY_SECONDS: u64 = HOUR;
/// `/ai` "今天".
pub const CONSENT_TODAY_SECONDS: u64 = DAY;
/// `/ai` "7天".
pub const CONSENT_WEEK_SECONDS: u64 = WEEK;
/// `/ai` "直到撤销 / 长期" upper bound.
pub const CONSENT_MAXIMUM_SECONDS: u64 = YEAR;
/// `/ai off` revocation. Falls through `set_ai_consent`'s `.max(1)` to a
/// 1-second key, which is meant to be already gone.
pub const CONSENT_REVOKED_SECONDS: u64 = 0;
}
#[derive(Debug, Error)]
pub enum StateError {
#[error("redis operation failed")]
Redis(#[from] redis::RedisError),
#[error("state lock poisoned")]
Poisoned,
#[error("configuration encryption failed")]
Encryption,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StreamMessage {
pub id: String,
pub payload: String,
}
/// Non-secret outbound tuning persisted in Redis for SPA/API management.
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)]
pub struct OutboundConfig {
pub jmap_timeout_ms: u64,
pub telegram_timeout_ms: u64,
pub llm_timeout_ms: u64,
pub max_retries: u8,
}
pub type RuntimeConfigProvider = Arc<RwLock<OutboundConfig>>;
pub type RuntimeBusinessConfigProvider = Arc<RwLock<Option<BusinessConfig>>>;
pub fn runtime_provider(config: OutboundConfig) -> RuntimeConfigProvider {
Arc::new(RwLock::new(config))
}
pub fn business_runtime_provider(config: Option<BusinessConfig>) -> RuntimeBusinessConfigProvider {
Arc::new(RwLock::new(config))
}
impl Default for OutboundConfig {
fn default() -> Self {
Self {
jmap_timeout_ms: 15_000,
telegram_timeout_ms: 10_000,
llm_timeout_ms: 30_000,
max_retries: 3,
}
}
}
/// Legacy single-event ceiling. It is the floor the derived ceiling must clear
/// and the fallback when the outbound config cannot be read.
const SINGLE_EVENT_CEILING_FLOOR_MS: u64 = 300_000;
/// Safety factor on the derived ceiling. Retries insert sleeps between
/// attempts, and the ceiling bounds a window that must not be exceeded while
/// the owning consumer is still alive.
const SINGLE_EVENT_CEILING_SAFETY: u64 = 2;
/// Upper bound on the XAUTOCLAIM idle window: a misconfigured timeout must not
/// push multi-instance recovery out by days. Six hours keeps a crashed batch
/// recoverable within the same working day.
const RECLAIM_IDLE_THRESHOLD_CAP_MS: u64 = 6 * 60 * 60 * 1000;
/// Worst-case wall clock for one event: every outbound call can burn its full
/// timeout once per attempt and `send_text` retries up to `max_retries` more
/// times, so the ceiling is
/// `(max_retries + 1) * (jmap + telegram + llm) * SAFETY`, floored at the
/// legacy constant so a zeroed-out config cannot shrink the window.
fn single_event_ceiling_ms(config: &OutboundConfig) -> u64 {
let attempts = config.max_retries as u64 + 1;
let per_attempt = config
.jmap_timeout_ms
.saturating_add(config.telegram_timeout_ms)
.saturating_add(config.llm_timeout_ms);
per_attempt
.saturating_mul(attempts)
.saturating_mul(SINGLE_EVENT_CEILING_SAFETY)
.max(SINGLE_EVENT_CEILING_FLOOR_MS)
}
/// Idle window handed to `XAUTOCLAIM`: the worst-case *batch* duration, since a
/// worker processes up to `count` events sequentially. Floored through
/// `single_event_ceiling_ms` and capped so a bad config cannot stretch
/// recovery into days. Only duplicate-on-multi-instance is at stake, never loss.
fn reclaim_idle_threshold_ms(config: &OutboundConfig, count: usize) -> u64 {
(count.max(1) as u64)
.saturating_mul(single_event_ceiling_ms(config))
.min(RECLAIM_IDLE_THRESHOLD_CAP_MS)
}
#[async_trait]
pub trait ReliableState: Send + Sync {
/// Atomic SET NX EX claim for Telegram update or `(account,email)` dedup key.
async fn claim_dedup(&self, key: &str, ttl_seconds: u64) -> Result<bool, StateError>;
/// Checks a durable idempotency commit without creating one.
async fn dedup_exists(&self, key: &str) -> Result<bool, StateError>;
/// Roll back a dedup claim when the following enqueue operation fails.
async fn release_dedup(&self, key: &str) -> Result<(), StateError>;
/// XADD enqueue; consumers perform XREADGROUP in a later worker phase.
async fn enqueue(&self, stream: &str, payload: &str) -> Result<String, StateError>;
/// Atomically claim `dedup_key` and XADD `payload` to `stream`.
///
/// The dedup key must only be written when the append succeeds: claiming
/// first and rolling back best-effort leaves a claimed-but-not-enqueued key
/// behind on a Redis hiccup, which silently drops the event for the whole
/// dedup TTL (REQ-RECONCILE-IDEMPOTENCY). Returns false when already claimed.
async fn claim_dedup_and_enqueue(
&self,
dedup_key: &str,
ttl_seconds: u64,
stream: &str,
payload: &str,
) -> Result<bool, StateError>;
/// XACK after successful processing.
async fn ack(&self, stream: &str, group: &str, message_id: &str) -> Result<(), StateError>;
/// XADD to a dead-letter stream after bounded retry policy decides to stop retrying.
async fn dead_letter(&self, stream: &str, payload: &str) -> Result<String, StateError>;
/// SET NX EX lock for reconcile single-flight.
async fn acquire_lock(
&self,
key: &str,
owner_token: &str,
ttl_seconds: u64,
) -> Result<bool, StateError>;
/// Release a lock after the bounded reconcile attempt has completed.
async fn release_lock(&self, key: &str, owner_token: &str) -> Result<(), StateError>;
/// Extend a lock only while the caller still owns it.
async fn renew_lock(
&self,
key: &str,
owner_token: &str,
ttl_seconds: u64,
) -> Result<bool, StateError>;
async fn read_batch(
&self,
stream: &str,
group: &str,
consumer: &str,
count: usize,
) -> Result<Vec<StreamMessage>, StateError>;
async fn retry_or_dlq(
&self,
stream: &str,
dlq: &str,
group: &str,
message: &StreamMessage,
max_attempts: u32,
) -> Result<bool, StateError>;
async fn set_ai_consent(&self, chat_id: i64, ttl_seconds: u64) -> Result<(), StateError>;
async fn ai_consent_until(&self, chat_id: i64) -> Result<Option<i64>, StateError>;
async fn clear_ai_consent(&self, chat_id: i64) -> Result<(), StateError>;
async fn get_outbound_config(&self) -> Result<OutboundConfig, StateError>;
async fn set_outbound_config(&self, config: &OutboundConfig) -> Result<(), StateError>;
/// Atomically read the encrypted-at-rest business configuration namespace.
/// Callers must redact secrets before returning it to HTTP clients.
async fn get_business_config(&self) -> Result<Option<serde_json::Value>, StateError>;
/// Replace the business configuration namespace; bootstrap callers must enforce
/// SET-NX semantics before invoking this operation (C-REDIS-ONLY-STATE).
async fn set_business_config(&self, config: &serde_json::Value) -> Result<(), StateError>;
/// Atomically initialize the business namespace; returns false when already set.
async fn initialize_business_config(
&self,
config: &serde_json::Value,
) -> Result<bool, StateError>;
/// Stores only the opaque session digest, never the bearer token itself.
async fn put_admin_session(&self, digest: &str, ttl_seconds: u64) -> Result<(), StateError>;
async fn admin_session_valid(&self, digest: &str) -> Result<bool, StateError>;
async fn revoke_admin_session(&self, digest: &str) -> Result<(), StateError>;
async fn business_config_revision(&self) -> Result<u64, StateError>;
/// Global durable enable gate; missing/read failures are interpreted by callers as disabled.
async fn is_enabled(&self) -> Result<bool, StateError>;
async fn set_enabled(&self, enabled: bool) -> Result<(), StateError>;
/// Durable JMAP Email/changes cursor. It is advanced only after all page
/// events have been enqueued successfully (REQ-RECONCILE-IDEMPOTENCY).
async fn get_reconcile_state(&self) -> Result<Option<String>, StateError>;
async fn set_reconcile_state(&self, state: &str) -> Result<(), StateError>;
/// Short-lived PushSubscription verification state; values are stored as
/// digests and never retained as plaintext credentials.
async fn remember_push_subscription(
&self,
subscription_id: &str,
verification_code: &str,
ttl_seconds: u64,
) -> Result<(), StateError>;
async fn push_subscription_verified(
&self,
subscription_id: &str,
verification_code: &str,
) -> Result<bool, StateError>;
async fn remember_push_subscription_id(&self, subscription_id: &str) -> Result<(), StateError>;
async fn set_push_subscription_status(
&self,
subscription_id: &str,
status: &str,
ttl_seconds: u64,
) -> Result<(), StateError>;
// NOTE: `push:subscription:{id}:status` is currently write-only
// ("pending"/"verified"/"disabled"). Authorization uses the verification
// digest instead; the status key stays as a lightweight ops/observability
// trail (inspectable via redis-cli), not a gate (建议-5a).
/// Drop the verification-code digest for `subscription_id`.
///
/// Callers must invoke this when disabling a subscription: otherwise a push
/// arriving within the residual TTL of the last verify call can still pass
/// `push_subscription_verified` and be enqueued (SAF-AUTH-JMAP-PUSH).
async fn forget_push_subscription(&self, subscription_id: &str) -> Result<(), StateError>;
async fn record_push_orphan(
&self,
subscription_id: &str,
request_id: &str,
) -> Result<(), StateError>;
async fn get_push_subscription_for_callback(
&self,
callback_url: &str,
) -> Result<Option<String>, StateError>;
async fn remember_push_subscription_for_callback(
&self,
callback_url: &str,
subscription_id: &str,
) -> Result<(), StateError>;
async fn remove_push_subscription_for_callback(
&self,
callback_url: &str,
) -> Result<(), StateError>;
/// 读取当前激活的 push 回调地址(current 指针),无注册时为 None。
///
/// per-URL 映射是分散的、无法枚举出"现在生效的是哪一个";这个指针让
/// /api/push/register 在更换地址时能找到旧订阅去注销,否则 Stalwart 会
/// 对同一邮件事件向新旧两个地址各推一次(重复通知)。
async fn get_push_registration_current(&self) -> Result<Option<String>, StateError>;
/// 记录当前激活的 push 回调地址。仅在 /api/push/register 真正建好订阅并
/// 写入 per-URL 映射之后调用;幂等命中(未创建订阅)不写。
async fn set_push_registration_current(&self, callback_url: &str) -> Result<(), StateError>;
/// 清除 current 指针。注销掉的是 current 地址时调用,否则指针会指向
/// 一个已 disabled 的订阅。
async fn clear_push_registration_current(&self) -> Result<(), StateError>;
}
/// Bound on a single Redis connect attempt, for both the initial connection
/// and every automatic reconnect.
///
/// `ConnectionManagerConfig` leaves this `None` by default, which leaves each
/// attempt at the OS connect timeout. Against a host that drops packets
/// instead of refusing them that is roughly two minutes per attempt. A
/// dead-but-reachable peer is far more likely to time out than to refuse, so
/// the bound has to be ours.
const REDIS_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
/// How many times a single connect/reconnect operation retries before giving
/// up. This deliberately overrides the crate default of 6.
///
/// Measured against an unreachable host with redis 0.27.6
/// (`ConnectionManager::new_with_config`):
/// - retries = 0 -> returns in ~0.5 ms with `IoError`
/// - retries = 1 -> returns in ~1.5 s with `IoError`
/// - retries = 6 -> **never returns** (still hung at 300 s)
///
/// So the crate's own default retry count is an infinite loop in this version,
/// not a bounded retry. Left at the default, a Redis host that is down when
/// the process boots makes startup hang forever: no failure, no log, no signal,
/// and a container that reports `Up` while serving nothing. The pre-manager
/// code path (`get_multiplexed_async_connection`) returned in about half a
/// millisecond, so the default silently regressed fail-fast boot.
///
/// One retry is still useful: it absorbs a transient refusal. Anything beyond
/// that is redundant here, because the manager re-enters this path on every
/// subsequent command once the connection drops, so recovery does not depend
/// on a single operation retrying many times.
const REDIS_CONNECT_RETRIES: usize = 1;
pub struct RedisState {
connection: redis::aio::ConnectionManager,
encryption_key: Option<[u8; 32]>,
}
impl RedisState {
/// Connects to externally managed REDIS_URL; URL/password are never logged.
pub async fn connect(redis_url: &str) -> Result<Self, StateError> {
Self::connect_with_encryption(redis_url, None).await
}
pub async fn connect_with_encryption(
redis_url: &str,
encryption_key: Option<[u8; 32]>,
) -> Result<Self, StateError> {
let client = redis::Client::open(redis_url)?;
// Both knobs are overridden deliberately; see the constant docs.
let manager_config = redis::aio::ConnectionManagerConfig::new()
.set_number_of_retries(REDIS_CONNECT_RETRIES)
.set_connection_timeout(REDIS_CONNECT_TIMEOUT);
Ok(Self {
connection: redis::aio::ConnectionManager::new_with_config(client, manager_config)
.await?,
encryption_key,
})
}
}
#[async_trait]
impl ReliableState for RedisState {
async fn claim_dedup(&self, key: &str, ttl_seconds: u64) -> Result<bool, StateError> {
set_nx_ex(self.connection.clone(), key, ttl_seconds).await
}
async fn release_dedup(&self, key: &str) -> Result<(), StateError> {
delete_key(self.connection.clone(), key).await
}
async fn dedup_exists(&self, key: &str) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let exists: i64 = redis::cmd("EXISTS")
.arg(key)
.query_async(&mut connection)
.await?;
Ok(exists != 0)
}
async fn enqueue(&self, stream: &str, payload: &str) -> Result<String, StateError> {
let mut connection = self.connection.clone();
Ok(redis::cmd("XADD")
.arg(stream)
.arg("*")
.arg("payload")
.arg(payload)
.query_async(&mut connection)
.await?)
}
async fn claim_dedup_and_enqueue(
&self,
dedup_key: &str,
ttl_seconds: u64,
stream: &str,
payload: &str,
) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
// KEYS[1] = dedup key, KEYS[2] = stream; ARGV[1] = ttl, ARGV[2] = payload.
// `SET NX` returning false means another worker owns the event; skip the
// append so the XADD is only attempted once per dedup window.
let claimed: i64 = redis::Script::new(
"if redis.call('set', KEYS[1], '1', 'NX', 'EX', ARGV[1]) then \
redis.call('xadd', KEYS[2], '*', 'payload', ARGV[2]) \
return 1 \
else \
return 0 \
end",
)
.key(dedup_key)
.key(stream)
.arg(ttl_seconds.max(1))
.arg(payload)
.invoke_async(&mut connection)
.await?;
Ok(claimed != 0)
}
async fn ack(&self, stream: &str, group: &str, message_id: &str) -> Result<(), StateError> {
let mut connection = self.connection.clone();
let _: i64 = redis::cmd("XACK")
.arg(stream)
.arg(group)
.arg(message_id)
.query_async(&mut connection)
.await?;
Ok(())
}
async fn dead_letter(&self, stream: &str, payload: &str) -> Result<String, StateError> {
self.enqueue(stream, payload).await
}
async fn acquire_lock(
&self,
key: &str,
owner_token: &str,
ttl_seconds: u64,
) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let result: Option<String> = redis::cmd("SET")
.arg(key)
.arg(owner_token)
.arg("NX")
.arg("EX")
.arg(ttl_seconds.max(1))
.query_async(&mut connection)
.await?;
Ok(result.is_some())
}
async fn release_lock(&self, key: &str, owner_token: &str) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::Script::new(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
)
.key(key)
.arg(owner_token)
.invoke_async::<i64>(&mut connection)
.await?;
Ok(())
}
async fn renew_lock(
&self,
key: &str,
owner_token: &str,
ttl_seconds: u64,
) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let renewed: i64 = redis::Script::new(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('expire', KEYS[1], ARGV[2]) else return 0 end",
)
.key(key)
.arg(owner_token)
.arg(ttl_seconds.max(1))
.invoke_async(&mut connection)
.await?;
Ok(renewed != 0)
}
async fn read_batch(
&self,
stream: &str,
group: &str,
consumer: &str,
count: usize,
) -> Result<Vec<StreamMessage>, StateError> {
let mut connection = self.connection.clone();
// The worker is deployable from an empty Redis: create its group lazily.
// BUSYGROUP is harmless when another instance created it first.
let _: Result<String, redis::RedisError> = redis::cmd("XGROUP")
.arg("CREATE")
.arg(stream)
.arg(group)
.arg("0-0")
.arg("MKSTREAM")
.query_async(&mut connection)
.await;
// Recover messages left in the consumer group's PEL after a crash or
// restart before reading new entries. The idle window has to clear the
// worst-case *batch* duration: a worker processes up to `count` events
// sequentially, each bounded by the single-event ceiling derived from
// the live outbound config. If the window is shorter than that, a
// second instance reclaims events that are still being processed and
// delivers them twice (建议-6). Single-instance deployments are
// unaffected; only duplicate-on-multi-instance is at stake, never loss.
let idle_threshold_ms = match self.get_outbound_config().await {
Ok(config) => reclaim_idle_threshold_ms(&config, count),
// A Redis error here must not break the delivery loop: fall back to
// the legacy constant instead of propagating the failure.
Err(_) => count.max(1) as u64 * SINGLE_EVENT_CEILING_FLOOR_MS,
};
let reclaimed: redis::streams::StreamAutoClaimReply = redis::cmd("XAUTOCLAIM")
.arg(stream)
.arg(group)
.arg(consumer)
.arg(idle_threshold_ms)
.arg("0-0")
.arg("COUNT")
.arg(count.max(1))
.query_async(&mut connection)
.await?;
let mut reclaimed_messages = Vec::new();
let mut malformed_ids = Vec::new();
for item in reclaimed.claimed {
let id = item.id.clone();
match stream_id_to_message(item) {
Some(message) => reclaimed_messages.push(message),
None => malformed_ids.push(id),
}
}
// A record without a `payload` field can never be processed. ACK it so
// it stops being re-claimed every cycle and squatting in the PEL
// (建议-7); a payload that is present but not valid JSON still routes
// through the normal Err -> DLQ path.
ack_malformed(&mut connection, stream, group, &malformed_ids).await;
if !reclaimed_messages.is_empty() {
return Ok(reclaimed_messages);
}
let reply: redis::streams::StreamReadReply = redis::cmd("XREADGROUP")
.arg("GROUP")
.arg(group)
.arg(consumer)
.arg("COUNT")
.arg(count.max(1))
.arg("STREAMS")
.arg(stream)
.arg(">")
.query_async(&mut connection)
.await?;
let mut messages = Vec::new();
let mut malformed_ids = Vec::new();
for key in reply.keys {
for item in key.ids {
let id = item.id.clone();
match stream_id_to_message(item) {
Some(message) => messages.push(message),
None => malformed_ids.push(id),
}
}
}
// Same handling as the XAUTOCLAIM path: drop unprocessable records from
// the PEL instead of letting them spin forever (建议-7).
ack_malformed(&mut connection, stream, group, &malformed_ids).await;
Ok(messages)
}
async fn retry_or_dlq(
&self,
stream: &str,
dlq: &str,
group: &str,
message: &StreamMessage,
max_attempts: u32,
) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let retry_key = format!("retry:{stream}:{}", message.id);
// Keep increment, DLQ append, and source ACK in one Redis script so a
// restart cannot leave an acknowledged event out of both queues. The
// counter TTL comes from `ttl::RETRY_COUNTER_SECONDS` rather than a
// second inline literal, so it cannot drift from the rest of the table.
let retry_ttl = ttl::RETRY_COUNTER_SECONDS;
let script = redis::Script::new(&format!(
r#"local n = redis.call('INCR', KEYS[1])
redis.call('EXPIRE', KEYS[1], {retry_ttl})
if n < tonumber(ARGV[1]) then return 0 end
redis.call('XADD', ARGV[2], '*', 'payload', ARGV[3])
redis.call('XACK', ARGV[4], ARGV[5], ARGV[6])
return 1"#
));
let moved: i64 = script
.key(retry_key)
.arg(max_attempts.max(1))
.arg(dlq)
.arg(&message.payload)
.arg(stream)
.arg(group)
.arg(&message.id)
.invoke_async(&mut connection)
.await?;
Ok(moved != 0)
}
async fn set_ai_consent(&self, chat_id: i64, ttl_seconds: u64) -> Result<(), StateError> {
let mut connection = self.connection.clone();
let key = format!("consent:ai:{chat_id}");
let expiry = unix_now().saturating_add(ttl_seconds as i64);
redis::cmd("SET")
.arg(key)
.arg(expiry)
.arg("EX")
.arg(ttl_seconds.max(1))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn ai_consent_until(&self, chat_id: i64) -> Result<Option<i64>, StateError> {
let mut connection = self.connection.clone();
let value: Option<i64> = redis::cmd("GET")
.arg(format!("consent:ai:{chat_id}"))
.query_async(&mut connection)
.await?;
Ok(value.filter(|expiry| *expiry > unix_now()))
}
async fn clear_ai_consent(&self, chat_id: i64) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("DEL")
.arg(format!("consent:ai:{chat_id}"))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn get_outbound_config(&self) -> Result<OutboundConfig, StateError> {
let mut connection = self.connection.clone();
let raw: Option<String> = redis::cmd("GET")
.arg("config:outbound")
.query_async(&mut connection)
.await?;
raw.map(|value| {
serde_json::from_str(&value).map_err(|_| {
redis::RedisError::from((redis::ErrorKind::TypeError, "invalid outbound config"))
})
})
.transpose()
.map(|config| config.unwrap_or_default())
.map_err(StateError::Redis)
}
async fn set_outbound_config(&self, config: &OutboundConfig) -> Result<(), StateError> {
let mut connection = self.connection.clone();
let value = serde_json::to_string(config).map_err(|_| {
redis::RedisError::from((redis::ErrorKind::TypeError, "serialize outbound config"))
})?;
redis::cmd("SET")
.arg("config:outbound")
.arg(value)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn get_business_config(&self) -> Result<Option<serde_json::Value>, StateError> {
let mut connection = self.connection.clone();
let raw: Option<String> = redis::cmd("GET")
.arg("config:business")
.query_async(&mut connection)
.await?;
raw.map(|value| decrypt_config(self.encryption_key.as_ref(), &value))
.transpose()
.map_err(|_| StateError::Encryption)
}
async fn set_business_config(&self, config: &serde_json::Value) -> Result<(), StateError> {
let mut connection = self.connection.clone();
let value = encrypt_config(self.encryption_key.as_ref(), config)?;
redis::cmd("SET")
.arg("config:business")
.arg(value)
.query_async::<()>(&mut connection)
.await?;
redis::cmd("INCR")
.arg("config:business:revision")
.query_async::<u64>(&mut connection)
.await?;
Ok(())
}
async fn initialize_business_config(
&self,
config: &serde_json::Value,
) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let value = encrypt_config(self.encryption_key.as_ref(), config)?;
let result: Option<String> = redis::cmd("SET")
.arg("config:business")
.arg(value)
.arg("NX")
.query_async(&mut connection)
.await?;
if result.is_some() {
redis::cmd("SET")
.arg("config:business:revision")
.arg(1_u64)
.arg("NX")
.query_async::<()>(&mut connection)
.await?;
}
Ok(result.is_some())
}
async fn put_admin_session(&self, digest: &str, ttl_seconds: u64) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg("config:admin_session")
.arg(digest)
.arg("EX")
.arg(ttl_seconds.max(1))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn admin_session_valid(&self, digest: &str) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let current: Option<String> = redis::cmd("GET")
.arg("config:admin_session")
.query_async(&mut connection)
.await?;
Ok(current.as_deref() == Some(digest))
}
async fn revoke_admin_session(&self, _digest: &str) -> Result<(), StateError> {
delete_key(self.connection.clone(), "config:admin_session").await
}
async fn business_config_revision(&self) -> Result<u64, StateError> {
let mut connection = self.connection.clone();
let value: Option<u64> = redis::cmd("GET")
.arg("config:business:revision")
.query_async(&mut connection)
.await?;
Ok(value.unwrap_or(0))
}
async fn is_enabled(&self) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let value: Option<bool> = redis::cmd("GET")
.arg("config:enabled")
.query_async(&mut connection)
.await?;
Ok(value.unwrap_or(false))
}
async fn set_enabled(&self, enabled: bool) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg("config:enabled")
.arg(if enabled { "1" } else { "0" })
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn get_reconcile_state(&self) -> Result<Option<String>, StateError> {
let mut connection = self.connection.clone();
Ok(redis::cmd("GET")
.arg("state:jmap:since")
.query_async(&mut connection)
.await?)
}
async fn set_reconcile_state(&self, state: &str) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg("state:jmap:since")
.arg(state)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn remember_push_subscription(
&self,
subscription_id: &str,
verification_code: &str,
ttl_seconds: u64,
) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg(format!("push:subscription:{subscription_id}"))
.arg(crate::config::session_digest(verification_code))
.arg("EX")
.arg(ttl_seconds.max(1))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn push_subscription_verified(
&self,
subscription_id: &str,
verification_code: &str,
) -> Result<bool, StateError> {
let mut connection = self.connection.clone();
let current: Option<String> = redis::cmd("GET")
.arg(format!("push:subscription:{subscription_id}"))
.query_async(&mut connection)
.await?;
Ok(current.as_deref() == Some(crate::config::session_digest(verification_code).as_str()))
}
async fn remember_push_subscription_id(&self, subscription_id: &str) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg("push:subscription:id")
.arg(subscription_id)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn set_push_subscription_status(
&self,
subscription_id: &str,
status: &str,
ttl_seconds: u64,
) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg(format!("push:subscription:{subscription_id}:status"))
.arg(status)
.arg("EX")
.arg(ttl_seconds.max(1))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn forget_push_subscription(&self, subscription_id: &str) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("DEL")
.arg(format!("push:subscription:{subscription_id}"))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn record_push_orphan(
&self,
subscription_id: &str,
request_id: &str,
) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg(format!("push:orphan:{subscription_id}"))
.arg(request_id)
.arg("EX")
.arg(ttl::PUSH_ORPHAN_SECONDS)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn get_push_subscription_for_callback(
&self,
callback_url: &str,
) -> Result<Option<String>, StateError> {
let mut connection = self.connection.clone();
Ok(redis::cmd("GET")
.arg(push_registration_key(callback_url))
.query_async(&mut connection)
.await?)
}
async fn remember_push_subscription_for_callback(
&self,
callback_url: &str,
subscription_id: &str,
) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("SET")
.arg(push_registration_key(callback_url))
.arg(subscription_id)
.arg("EX")
.arg(ttl::PUSH_REGISTRATION_SECONDS)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn remove_push_subscription_for_callback(
&self,
callback_url: &str,
) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("DEL")
.arg(push_registration_key(callback_url))
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn get_push_registration_current(&self) -> Result<Option<String>, StateError> {
let mut connection = self.connection.clone();
let reply: Option<String> = redis::cmd("GET")
.arg(PUSH_REGISTRATION_CURRENT_KEY)
.query_async(&mut connection)
.await?;
Ok(reply)
}
async fn set_push_registration_current(&self, callback_url: &str) -> Result<(), StateError> {
let mut connection = self.connection.clone();
// 与 per-URL 映射同 TTL:指针指向的注册过期后,指针本身也应一起消失。
redis::cmd("SET")
.arg(PUSH_REGISTRATION_CURRENT_KEY)
.arg(callback_url)
.arg("EX")
.arg(ttl::PUSH_REGISTRATION_SECONDS)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
async fn clear_push_registration_current(&self) -> Result<(), StateError> {
let mut connection = self.connection.clone();
redis::cmd("DEL")
.arg(PUSH_REGISTRATION_CURRENT_KEY)
.query_async::<()>(&mut connection)
.await?;
Ok(())
}
}
/// current 指针的键:当前激活的 push 回调地址原文。
///
/// 与 per-URL 映射共用前缀,但字面量 `current` 含非十六进制字符,永远不会
/// 与 `session_digest` 的十六进制摘要相撞。
const PUSH_REGISTRATION_CURRENT_KEY: &str = "push:registration:current";
fn push_registration_key(callback_url: &str) -> String {
format!(
"push:registration:{}",
crate::config::session_digest(callback_url)
)
}
fn stream_id_to_message(item: redis::streams::StreamId) -> Option<StreamMessage> {
let payload = item
.map
.get("payload")
.and_then(|value| redis::from_redis_value(value).ok());
payload.map(|payload| StreamMessage {
id: item.id,
payload,
})
}
/// XACK stream records that arrived without a parseable `payload` field. These
/// can never be processed, so acknowledging them keeps the consumer group's PEL
/// from filling up with unclaimable junk; the failure is logged for operators.
async fn ack_malformed(
connection: &mut redis::aio::ConnectionManager,
stream: &str,
group: &str,
ids: &[String],
) {
if ids.is_empty() {
return;
}
let ack = redis::cmd("XACK")
.arg(stream)
.arg(group)
.arg(ids)
.query_async::<i64>(connection)
.await;
if ack
.as_ref()
.map(|acked| *acked != ids.len() as i64)
.unwrap_or(true)
{
eprintln!(
"WARN malformed stream records ack incomplete (stream={stream}, group={group}, expected={}, result={ack:?})",
ids.len()
);
}
}
async fn set_nx_ex(
mut connection: redis::aio::ConnectionManager,
key: &str,
ttl_seconds: u64,
) -> Result<bool, StateError> {
let result: Option<String> = redis::cmd("SET")
.arg(key)
.arg("1")
.arg("NX")
.arg("EX")
// Guard against a ttl of 0, which Redis rejects with "invalid expire
// time" (same normalization as acquire_lock).
.arg(ttl_seconds.max(1))
.query_async(&mut connection)
.await?;
Ok(result.is_some())
}
#[derive(Clone, Default)]
pub struct MemoryState {
inner: Arc<Mutex<MemoryInner>>,
}
#[cfg(test)]
impl MemoryState {
pub fn enabled_for_tests() -> Self {
let state = Self::default();
state.inner.lock().expect("new state lock").enabled = true;
state
}
}
#[derive(Default)]
struct MemoryInner {
dedup: HashSet<String>,
locks: HashMap<String, String>,
push_registrations: HashMap<String, String>,
/// 当前激活的 push 回调地址(current 指针)
push_registration_current: Option<String>,
streams: HashMap<String, Vec<(String, String)>>,
retries: HashMap<(String, String), u32>,
consent: HashMap<i64, i64>,
next_id: u64,
outbound: OutboundConfig,
business_config: Option<serde_json::Value>,
admin_session: Option<(String, i64)>,
business_revision: u64,
enabled: bool,
reconcile_state: Option<String>,
push_subscriptions: HashMap<String, (String, i64)>,
push_subscription_id: Option<String>,
push_status: HashMap<String, (String, i64)>,
push_orphans: HashMap<String, String>,
}
#[async_trait]
impl ReliableState for MemoryState {
async fn claim_dedup(&self, key: &str, _ttl_seconds: u64) -> Result<bool, StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner.dedup.insert(key.to_owned()))
}
async fn release_dedup(&self, key: &str) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.dedup.remove(key);
Ok(())
}
async fn dedup_exists(&self, key: &str) -> Result<bool, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner.dedup.contains(key))
}
async fn enqueue(&self, stream: &str, payload: &str) -> Result<String, StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.next_id += 1;
let id = format!("{}-0", inner.next_id);
inner
.streams
.entry(stream.to_owned())
.or_default()
.push((id.clone(), payload.to_owned()));
Ok(id)
}
async fn claim_dedup_and_enqueue(
&self,
dedup_key: &str,
_ttl_seconds: u64,
stream: &str,
payload: &str,
) -> Result<bool, StateError> {
// Hold the lock across claim + append so MemoryState honors the same
// atomicity contract as the Lua-backed RedisState implementation.
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
if !inner.dedup.insert(dedup_key.to_owned()) {
return Ok(false);
}
inner.next_id += 1;
let id = format!("{}-0", inner.next_id);
inner
.streams
.entry(stream.to_owned())
.or_default()
.push((id, payload.to_owned()));
Ok(true)
}
async fn ack(&self, stream: &str, _group: &str, message_id: &str) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
if let Some(messages) = inner.streams.get_mut(stream) {
messages.retain(|(id, _)| id != message_id);
}
Ok(())
}
async fn dead_letter(&self, stream: &str, payload: &str) -> Result<String, StateError> {
self.enqueue(stream, payload).await
}
async fn acquire_lock(
&self,
key: &str,
owner_token: &str,
_ttl_seconds: u64,
) -> Result<bool, StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
if inner.locks.contains_key(key) {
return Ok(false);
}
inner.locks.insert(key.to_owned(), owner_token.to_owned());
Ok(true)
}
async fn release_lock(&self, key: &str, owner_token: &str) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
if inner
.locks
.get(key)
.is_some_and(|owner| owner == owner_token)
{
inner.locks.remove(key);
}
Ok(())
}
async fn renew_lock(
&self,
key: &str,
owner_token: &str,
_ttl_seconds: u64,
) -> Result<bool, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner
.locks
.get(key)
.is_some_and(|owner| owner == owner_token))
}
async fn read_batch(
&self,
stream: &str,
_group: &str,
_consumer: &str,
count: usize,
) -> Result<Vec<StreamMessage>, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner
.streams
.get(stream)
.into_iter()
.flatten()
.take(count.max(1))
.map(|(id, payload)| StreamMessage {
id: id.clone(),
payload: payload.clone(),
})
.collect())
}
async fn retry_or_dlq(
&self,
stream: &str,
dlq: &str,
_group: &str,
message: &StreamMessage,
max_attempts: u32,
) -> Result<bool, StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
let key = (stream.to_owned(), message.id.clone());
let attempts = {
let entry = inner.retries.entry(key).or_default();
*entry += 1;
*entry
};
if attempts >= max_attempts {
inner.next_id += 1;
let id = format!("{}-0", inner.next_id);
inner
.streams
.entry(dlq.to_owned())
.or_default()
.push((id, message.payload.clone()));
if let Some(messages) = inner.streams.get_mut(stream) {
messages.retain(|(source_id, _)| source_id != &message.id);
}
return Ok(true);
}
let _ = stream;
Ok(false)
}
async fn set_ai_consent(&self, chat_id: i64, ttl_seconds: u64) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner
.consent
.insert(chat_id, unix_now().saturating_add(ttl_seconds as i64));
Ok(())
}
async fn ai_consent_until(&self, chat_id: i64) -> Result<Option<i64>, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner
.consent
.get(&chat_id)
.copied()
.filter(|expiry| *expiry > unix_now()))
}
async fn clear_ai_consent(&self, chat_id: i64) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.consent.remove(&chat_id);
Ok(())
}
async fn get_outbound_config(&self) -> Result<OutboundConfig, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner.outbound.clone())
}
async fn set_outbound_config(&self, config: &OutboundConfig) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.outbound = config.clone();
Ok(())
}
async fn get_business_config(&self) -> Result<Option<serde_json::Value>, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner.business_config.clone())
}
async fn set_business_config(&self, config: &serde_json::Value) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.business_config = Some(config.clone());
inner.business_revision = inner.business_revision.saturating_add(1);
Ok(())
}
async fn initialize_business_config(
&self,
config: &serde_json::Value,
) -> Result<bool, StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
if inner.business_config.is_some() {
return Ok(false);
}
inner.business_config = Some(config.clone());
inner.business_revision = 1;
Ok(true)
}
async fn put_admin_session(&self, digest: &str, ttl_seconds: u64) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.admin_session = Some((
digest.to_owned(),
unix_now().saturating_add(ttl_seconds as i64),
));
Ok(())
}
async fn admin_session_valid(&self, digest: &str) -> Result<bool, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner
.admin_session
.as_ref()
.is_some_and(|(value, expiry)| value == digest && *expiry > unix_now()))
}
async fn revoke_admin_session(&self, digest: &str) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
if inner
.admin_session
.as_ref()
.is_some_and(|(value, _)| value == digest)
{
inner.admin_session = None;
}
Ok(())
}
async fn business_config_revision(&self) -> Result<u64, StateError> {
let inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
Ok(inner.business_revision)
}
async fn is_enabled(&self) -> Result<bool, StateError> {
Ok(self.inner.lock().map_err(|_| StateError::Poisoned)?.enabled)
}
async fn set_enabled(&self, enabled: bool) -> Result<(), StateError> {
self.inner.lock().map_err(|_| StateError::Poisoned)?.enabled = enabled;
Ok(())
}
async fn get_reconcile_state(&self) -> Result<Option<String>, StateError> {
Ok(self
.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.reconcile_state
.clone())
}
async fn set_reconcile_state(&self, state: &str) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.reconcile_state = Some(state.to_owned());
Ok(())
}
async fn remember_push_subscription(
&self,
subscription_id: &str,
verification_code: &str,
ttl_seconds: u64,
) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_subscriptions
.insert(
subscription_id.to_owned(),
(
crate::config::session_digest(verification_code),
unix_now().saturating_add(ttl_seconds as i64),
),
);
Ok(())
}
async fn push_subscription_verified(
&self,
subscription_id: &str,
verification_code: &str,
) -> Result<bool, StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
let Some((digest, expiry)) = inner.push_subscriptions.get(subscription_id) else {
return Ok(false);
};
if *expiry <= unix_now() {
inner.push_subscriptions.remove(subscription_id);
return Ok(false);
}
Ok(digest == &crate::config::session_digest(verification_code))
}
async fn remember_push_subscription_id(&self, subscription_id: &str) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_subscription_id = Some(subscription_id.to_owned());
Ok(())
}
async fn get_push_subscription_for_callback(
&self,
callback_url: &str,
) -> Result<Option<String>, StateError> {
Ok(self
.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_registrations
.get(&push_registration_key(callback_url))
.cloned())
}
async fn remember_push_subscription_for_callback(
&self,
callback_url: &str,
subscription_id: &str,
) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_registrations
.insert(
push_registration_key(callback_url),
subscription_id.to_owned(),
);
Ok(())
}
async fn remove_push_subscription_for_callback(
&self,
callback_url: &str,
) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_registrations
.remove(&push_registration_key(callback_url));
Ok(())
}
async fn get_push_registration_current(&self) -> Result<Option<String>, StateError> {
Ok(self
.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_registration_current
.clone())
}
async fn set_push_registration_current(&self, callback_url: &str) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_registration_current = Some(callback_url.to_owned());
Ok(())
}
async fn clear_push_registration_current(&self) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_registration_current = None;
Ok(())
}
async fn set_push_subscription_status(
&self,
subscription_id: &str,
status: &str,
ttl_seconds: u64,
) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_status
.insert(
subscription_id.to_owned(),
(
status.to_owned(),
unix_now().saturating_add(ttl_seconds as i64),
),
);
Ok(())
}
async fn forget_push_subscription(&self, subscription_id: &str) -> Result<(), StateError> {
let mut inner = self.inner.lock().map_err(|_| StateError::Poisoned)?;
inner.push_subscriptions.remove(subscription_id);
Ok(())
}
async fn record_push_orphan(
&self,
subscription_id: &str,
request_id: &str,
) -> Result<(), StateError> {
self.inner
.lock()
.map_err(|_| StateError::Poisoned)?
.push_orphans
.insert(subscription_id.to_owned(), request_id.to_owned());
Ok(())
}
}
fn unix_now() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |duration| duration.as_secs() as i64)
}
const CONFIG_AAD: &[u8] = b"message-weave/config:business/v1";
fn encrypt_config(
key: Option<&[u8; 32]>,
config: &serde_json::Value,
) -> Result<String, StateError> {
let key = key.ok_or(StateError::Encryption)?;
let unbound =
aead::UnboundKey::new(&aead::AES_256_GCM, key).map_err(|_| StateError::Encryption)?;
let sealing = aead::LessSafeKey::new(unbound);
let mut nonce = [0_u8; 12];
SystemRandom::new()
.fill(&mut nonce)
.map_err(|_| StateError::Encryption)?;
let mut body = serde_json::to_vec(config).map_err(|_| StateError::Encryption)?;
sealing
.seal_in_place_append_tag(
aead::Nonce::assume_unique_for_key(nonce),
aead::Aad::from(CONFIG_AAD),
&mut body,
)
.map_err(|_| StateError::Encryption)?;
Ok(format!("v1:{}:{}", hex_encode(&nonce), hex_encode(&body)))
}
fn decrypt_config(key: Option<&[u8; 32]>, value: &str) -> Result<serde_json::Value, StateError> {
let key = key.ok_or(StateError::Encryption)?;
let mut parts = value.split(':');
if parts.next() != Some("v1") {
return Err(StateError::Encryption);
}
let nonce = hex_decode(parts.next().ok_or(StateError::Encryption)?, 12)?;
let mut body = hex_decode(parts.next().ok_or(StateError::Encryption)?, usize::MAX)?;
if parts.next().is_some() {
return Err(StateError::Encryption);
}
let unbound =
aead::UnboundKey::new(&aead::AES_256_GCM, key).map_err(|_| StateError::Encryption)?;
let opening = aead::LessSafeKey::new(unbound);
let plain = opening
.open_in_place(
aead::Nonce::assume_unique_for_key(
nonce.try_into().map_err(|_| StateError::Encryption)?,
),
aead::Aad::from(CONFIG_AAD),
&mut body,
)
.map_err(|_| StateError::Encryption)?;
serde_json::from_slice(plain).map_err(|_| StateError::Encryption)
}
fn hex_encode(bytes: &[u8]) -> String {
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}
fn hex_decode(value: &str, max_len: usize) -> Result<Vec<u8>, StateError> {
if !value.len().is_multiple_of(2) || value.len() / 2 > max_len {
return Err(StateError::Encryption);
}
(0..value.len())
.step_by(2)
.map(|index| {
u8::from_str_radix(&value[index..index + 2], 16).map_err(|_| StateError::Encryption)
})
.collect()
}
async fn delete_key(
mut connection: redis::aio::ConnectionManager,
key: &str,
) -> Result<(), StateError> {
let _: i64 = redis::cmd("DEL")
.arg(key)
.query_async(&mut connection)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ttl_contract_is_pinned() {
// The dedup, lock, consent and replay windows decide whether the email
// -> Telegram chain loses or duplicates. The three primitives that issue
// EX/EXPIRE are covered against real Redis in real_redis_ttl_tests; this
// pins the literals the call sites now route through, so a wrong-but-
// valid number cannot slip in as a silent const change.
assert_eq!(ttl::MINUTE, 60);
assert_eq!(ttl::HOUR, 3_600);
assert_eq!(ttl::DAY, 86_400);
assert_eq!(ttl::WEEK, 604_800);
assert_eq!(ttl::YEAR, 31_536_000);
assert_eq!(ttl::RECONCILE_LOCK_SECONDS, 60);
assert_eq!(ttl::RECONCILE_HEARTBEAT_SECONDS, 30);
assert_eq!(ttl::RECONCILE_HEARTBEAT_INTERVAL_SECONDS, 10);
const {
assert!(
ttl::RECONCILE_HEARTBEAT_SECONDS < ttl::RECONCILE_LOCK_SECONDS,
"the heartbeat must renew before the reconcile lock expires"
);
assert!(
ttl::RECONCILE_HEARTBEAT_INTERVAL_SECONDS * 2 <= ttl::RECONCILE_HEARTBEAT_SECONDS,
"one missed heartbeat must not push renewal past the beat TTL"
);
assert!(
ttl::RECONCILE_HEARTBEAT_INTERVAL_SECONDS < ttl::RECONCILE_LOCK_SECONDS,
"the first renewal must fire before the initial hold expires"
);
}
assert_eq!(ttl::PUSH_REGISTER_LOCK_SECONDS, 360);
assert_eq!(ttl::PUSH_VERIFY_LIMIT_SECONDS, 30);
assert_eq!(ttl::PUSH_STATUS_PENDING_SECONDS, 900);
assert_eq!(ttl::PUSH_STATUS_VERIFIED_SECONDS, 300);
assert_eq!(ttl::PUSH_SUBSCRIPTION_SECONDS, 300);
assert_eq!(ttl::PUSH_REGISTRATION_SECONDS, 604_800);
assert_eq!(ttl::PUSH_ORPHAN_SECONDS, 604_800);
assert_eq!(ttl::ADMIN_SESSION_SECONDS, 1_800);
assert_eq!(ttl::DEDUP_TG_SECONDS, 86_400);
assert_eq!(ttl::DEDUP_JMAP_SECONDS, 86_400);
assert_eq!(
ttl::DEDUP_TG_SECONDS,
ttl::DEDUP_JMAP_SECONDS,
"callback and reconcile must deduplicate on the same window"
);
assert_eq!(ttl::DELIVERY_INFLIGHT_SECONDS, 60);
assert_eq!(ttl::DELIVERY_COMMITTED_SECONDS, 604_800);
assert_eq!(ttl::PUSH_DISABLED_SECONDS, 86_400);
// Retained in this table because retry_or_dlq interpolates it into its
// Lua script; the test above asserts the counter itself in
// real_redis_ttl_tests.
assert_eq!(ttl::RETRY_COUNTER_SECONDS, 86_400);
assert_eq!(ttl::CONSENT_TEMPORARY_SECONDS, 3_600);
assert_eq!(ttl::CONSENT_TODAY_SECONDS, 86_400);
assert_eq!(ttl::CONSENT_WEEK_SECONDS, 604_800);
assert_eq!(ttl::CONSENT_MAXIMUM_SECONDS, 31_536_000);
assert_eq!(ttl::CONSENT_REVOKED_SECONDS, 0);
const {
assert!(
ttl::CONSENT_TEMPORARY_SECONDS < ttl::CONSENT_TODAY_SECONDS
&& ttl::CONSENT_TODAY_SECONDS < ttl::CONSENT_WEEK_SECONDS
&& ttl::CONSENT_WEEK_SECONDS < ttl::CONSENT_MAXIMUM_SECONDS,
"the consent ladder must stay ordered, or a longer grant would
satisfy a shorter one"
);
}
}
#[test]
fn rediss_url_accepts_default_acl_and_encoded_password() {
let client = redis::Client::open("rediss://default:p%40ss%3Aword@example.invalid:6379/0");
assert!(client.is_ok());
let client = redis::Client::open("redis://default:p%40ss%3Aword@example.invalid:6379/0");
assert!(client.is_ok());
}
#[test]
fn reclaim_idle_threshold_is_derived_from_runtime_config() {
// (15s + 10s + 30s) * (3 + 1) attempts * 2 safety = 440s per event.
let default = OutboundConfig::default();
assert_eq!(single_event_ceiling_ms(&default), 440_000);
assert!(reclaim_idle_threshold_ms(&default, 1) >= SINGLE_EVENT_CEILING_FLOOR_MS);
assert!(reclaim_idle_threshold_ms(&default, 16) > reclaim_idle_threshold_ms(&default, 8));
let slow = OutboundConfig {
jmap_timeout_ms: 60_000,
telegram_timeout_ms: 60_000,
llm_timeout_ms: 60_000,
max_retries: 4,
};
assert!(reclaim_idle_threshold_ms(&slow, 1) > reclaim_idle_threshold_ms(&default, 1));
}
#[test]
fn reclaim_idle_threshold_is_floored_and_capped() {
let zeroed = OutboundConfig {
jmap_timeout_ms: 0,
telegram_timeout_ms: 0,
llm_timeout_ms: 0,
max_retries: 0,
};
assert_eq!(
single_event_ceiling_ms(&zeroed),
SINGLE_EVENT_CEILING_FLOOR_MS
);
let absurd = OutboundConfig {
jmap_timeout_ms: 3_600_000,
telegram_timeout_ms: 3_600_000,
llm_timeout_ms: 3_600_000,
max_retries: 5,
};
assert_eq!(
reclaim_idle_threshold_ms(&absurd, 64),
RECLAIM_IDLE_THRESHOLD_CAP_MS
);
}
#[test]
fn reclaim_idle_threshold_ignores_empty_batches() {
let default = OutboundConfig::default();
assert_eq!(
reclaim_idle_threshold_ms(&default, 0),
single_event_ceiling_ms(&default)
);
}
#[test]
fn encrypted_config_roundtrip_and_tamper_fail_closed() {
let key = [7_u8; 32];
let value = serde_json::json!({"secret": "value"});
let encoded = encrypt_config(Some(&key), &value).unwrap();
assert_eq!(decrypt_config(Some(&key), &encoded).unwrap(), value);
let mut tampered = encoded.into_bytes();
let index = tampered.len() - 1;
tampered[index] = if tampered[index] == b'0' { b'1' } else { b'0' };
let tampered = String::from_utf8(tampered).unwrap();
assert!(decrypt_config(Some(&key), &tampered).is_err());
assert!(decrypt_config(Some(&[8_u8; 32]), &tampered).is_err());
assert!(encrypt_config(None, &value).is_err());
}
#[tokio::test]
async fn memory_state_is_atomic_and_supports_stream_lifecycle() {
let state = MemoryState::default();
assert!(!state.is_enabled().await.unwrap());
state.set_enabled(true).await.unwrap();
assert!(state.is_enabled().await.unwrap());
state.set_enabled(false).await.unwrap();
assert!(!state.is_enabled().await.unwrap());
assert!(state.get_reconcile_state().await.unwrap().is_none());
state.set_reconcile_state("jmap-state-1").await.unwrap();
assert_eq!(
state.get_reconcile_state().await.unwrap().as_deref(),
Some("jmap-state-1")
);
assert!(state.claim_dedup("tg:42", 60).await.unwrap());
assert!(!state.claim_dedup("tg:42", 60).await.unwrap());
assert!(state.dedup_exists("tg:42").await.unwrap());
state.release_dedup("tg:42").await.unwrap();
assert!(!state.dedup_exists("tg:42").await.unwrap());
// A short in-flight lease can be cleared after a pre-send crash,
// while the post-send commit remains durable for ACK recovery.
assert!(state
.claim_dedup("delivery:inflight:events:1-0", 60)
.await
.unwrap());
state
.release_dedup("delivery:inflight:events:1-0")
.await
.unwrap();
assert!(!state
.dedup_exists("delivery:inflight:events:1-0")
.await
.unwrap());
assert!(state
.claim_dedup("delivery:committed:events:1-0", 604_800)
.await
.unwrap());
assert!(state
.dedup_exists("delivery:committed:events:1-0")
.await
.unwrap());
assert!(state
.acquire_lock("reconcile", "owner-a", 30)
.await
.unwrap());
assert!(!state
.acquire_lock("reconcile", "owner-b", 30)
.await
.unwrap());
state.release_lock("reconcile", "owner-b").await.unwrap();
assert!(!state
.acquire_lock("reconcile", "owner-b", 30)
.await
.unwrap());
assert!(!state.renew_lock("reconcile", "owner-b", 90).await.unwrap());
assert!(state.renew_lock("reconcile", "owner-a", 90).await.unwrap());
state.release_lock("reconcile", "owner-a").await.unwrap();
assert!(state
.acquire_lock("reconcile", "owner-b", 30)
.await
.unwrap());
state
.remember_push_subscription_for_callback(
"https://example.test/push/jmap",
"subscription-1",
)
.await
.unwrap();
assert_eq!(
state
.get_push_subscription_for_callback("https://example.test/push/jmap")
.await
.unwrap()
.as_deref(),
Some("subscription-1")
);
state
.set_push_subscription_status("subscription-1", "disabled", 60)
.await
.unwrap();
// `pending/verified/disabled` 状态键是只写的运维轨迹(建议-5a):
// 授权语义由验证码摘要承担,故这里不对其做读取断言。
state
.remove_push_subscription_for_callback("https://example.test/push/jmap")
.await
.unwrap();
assert!(state
.get_push_subscription_for_callback("https://example.test/push/jmap")
.await
.unwrap()
.is_none());
assert!(state.ai_consent_until(7).await.unwrap().is_none());
state.set_ai_consent(7, 3600).await.unwrap();
assert!(state.ai_consent_until(7).await.unwrap().is_some());
state.clear_ai_consent(7).await.unwrap();
assert!(state.ai_consent_until(7).await.unwrap().is_none());
let id = state.enqueue("events", "payload").await.unwrap();
let message = StreamMessage {
id: id.clone(),
payload: "payload".into(),
};
assert!(!state
.retry_or_dlq("events", "events-dlq", "workers", &message, 2)
.await
.unwrap());
assert!(state
.retry_or_dlq("events", "events-dlq", "workers", &message, 2)
.await
.unwrap());
assert!(state
.read_batch("events", "workers", "c", 10)
.await
.unwrap()
.is_empty());
assert_eq!(
state
.read_batch("events-dlq", "workers", "c", 10)
.await
.unwrap()
.len(),
1
);
state.ack("events-dlq", "workers", &id).await.unwrap();
}
}
/// Opt-in TTL assertions against a real Redis.
///
/// Everything else in this file verifies TTL *plumbing* with `MemoryState`,
/// which ignores TTL by design. The three write points below are the only
/// places an expiry actually reaches Redis, and before this module no test
/// asserted any value of them - a typo like `604_800 -> 60_480` would have
/// passed CI green.
///
/// Run with `REDIS_TEST_URL=... cargo test -- --ignored ttl`.
/// The URL is never printed.
#[cfg(test)]
mod real_redis_ttl_tests {
use super::*;
/// Assert `actual` ms is within `tolerance_ms` of `expected_seconds` s.
fn assert_within(actual_ms: i64, expected_seconds: u64, tolerance_ms: u64) {
assert!(
actual_ms > 0,
"expected a live TTL of ~{expected_seconds}s, got {actual_ms} ms - the key either has no expiry or expired already"
);
let expected_ms = expected_seconds as i64 * 1000;
assert!(
(actual_ms - expected_ms).abs() <= tolerance_ms as i64,
"expected PTTL near {expected_ms} ms, got {actual_ms} ms"
);
}
fn unique() -> String {
format!("{}:{}", std::process::id(), unix_now())
}
/// Unit-test binaries never run `main()`, so no rustls crypto provider is
/// installed and the TLS handshake to a `rediss://` URL panics before any
/// command is sent. Mirror the one-time setup in `main.rs`.
fn require_tls_provider() {
assert!(
crate::install_rustls_provider(),
"rustls needs a crypto provider before any Redis TLS client"
);
}
#[tokio::test]
#[ignore = "requires REDIS_TEST_URL pointing at a reachable Redis"]
async fn ttl_claim_dedup_sets_the_exactly_requested_expiry() {
let Some(url) = std::env::var_os("REDIS_TEST_URL") else {
eprintln!("skipped: REDIS_TEST_URL is not configured");
return;
};
require_tls_provider();
let state = RedisState::connect(url.to_str().expect("valid utf8 REDIS_TEST_URL"))
.await
.expect("REDIS_TEST_URL must be reachable");
let run = unique();
// Zero is floored to 1 second by `.max(1)`: Redis rejects a zero EX
// with "invalid expire time in SET command", so the unwrap below
// already proves the floor ran. PTTL rules out the other variant, where
// the EX is silently dropped and the key has no expiry at all (-1). A
// live readback can only be counting *down* from the 1s floor, and a
// whole second is usually gone by the time this reaches a remote TLS
// instance, so the floor is asserted as an upper bound rather than as
// an absolute value.
let zero = format!("ttlcheck:dedup:{run}:zero");
assert!(state.claim_dedup(&zero, 0).await.unwrap());
let mut conn = state.connection.clone();
let pttl: i64 = redis::cmd("PTTL")
.arg(&zero)
.query_async(&mut conn)
.await
.unwrap();
assert!(
pttl != -1,
"EX 0 was not normalized to EX 1: the dedup key was written with no expiry"
);
assert!(
pttl == -2 || pttl <= 1_000 + 2_000,
"expected the remaining TTL to be within the 1s floor, got {pttl} ms"
);
delete_key(state.connection.clone(), &zero).await.unwrap();
// A normal request is honoured to the second.
let one_week = format!("ttlcheck:dedup:{run}:week");
assert!(state.claim_dedup(&one_week, 604_800).await.unwrap());
let pttl: i64 = redis::cmd("PTTL")
.arg(&one_week)
.query_async(&mut conn)
.await
.unwrap();
assert_within(pttl, 604_800, 2_000);
delete_key(state.connection.clone(), &one_week)
.await
.unwrap();
}
#[tokio::test]
#[ignore = "requires REDIS_TEST_URL pointing at a reachable Redis"]
async fn ttl_consent_and_retry_landing_on_real_redis() {
let Some(url) = std::env::var_os("REDIS_TEST_URL") else {
eprintln!("skipped: REDIS_TEST_URL is not configured");
return;
};
require_tls_provider();
let state = RedisState::connect(url.to_str().expect("valid utf8 REDIS_TEST_URL"))
.await
.expect("REDIS_TEST_URL must be reachable");
let run = unique();
let mut conn = state.connection.clone();
// `set_ai_consent` writes an absolute expiry value plus a relative EX.
// Telegram ids stay under ~1e11 in magnitude, so anchor far outside that
// range: this test may target a shared database and must never shadow a
// real chat's consent flag.
let pid: i64 = std::process::id().into();
let chat_id: i64 = -(4_000_000_000_000_000_000) + pid;
state.set_ai_consent(chat_id, 3600).await.unwrap();
let consent_key = format!("consent:ai:{chat_id}");
let pttl: i64 = redis::cmd("PTTL")
.arg(&consent_key)
.query_async(&mut conn)
.await
.unwrap();
assert_within(pttl, 3600, 2_000);
delete_key(state.connection.clone(), &consent_key)
.await
.unwrap();
// `set_push_registration_current` reuses the registration TTL: the
// pointer is only useful while the registration it points at is still
// valid, so both must expire together. The key name is fixed and this
// may target a shared database, so a populated key means a real
// registration exists -- skip it rather than clobber it.
let previous = state.get_push_registration_current().await.unwrap();
if previous.is_some() {
eprintln!("skipped pointer TTL check: push:registration:current is already populated");
return;
}
state
.set_push_registration_current("ttlcheck://example.invalid/push")
.await
.unwrap();
assert_eq!(
state
.get_push_registration_current()
.await
.unwrap()
.as_deref(),
Some("ttlcheck://example.invalid/push")
);
let pttl: i64 = redis::cmd("PTTL")
.arg(PUSH_REGISTRATION_CURRENT_KEY)
.query_async(&mut conn)
.await
.unwrap();
assert_within(pttl, ttl::PUSH_REGISTRATION_SECONDS, 2_000);
assert!(state.clear_push_registration_current().await.is_ok());
assert_eq!(state.get_push_registration_current().await.unwrap(), None);
// The retry counter's 86400s lives in the Lua script, not at a call
// site, so it is asserted through `retry_or_dlq` rather than read back
// from a constant. max_attempts=99 keeps n below the cap, so the script
// returns before XADD/XACK and no stream or group needs to exist.
let stream_name = "ttlcheck:stream";
let message = StreamMessage {
id: format!("ttlcheck:{run}:msg"),
payload: "payload".into(),
};
// Mirror `retry_or_dlq`'s own key format exactly, or the PTTL would be
// read from a key the call never wrote.
let retry_key = format!("retry:{stream_name}:{}", message.id);
assert!(!state
.retry_or_dlq(stream_name, "ttlcheck:dlq", "ttlcheck:group", &message, 99)
.await
.unwrap());
let pttl: i64 = redis::cmd("PTTL")
.arg(&retry_key)
.query_async(&mut conn)
.await
.unwrap();
assert_within(pttl, 86_400, 2_000);
delete_key(state.connection.clone(), &retry_key)
.await
.unwrap();
}
}