落地系统异常:大模型/TMS/建群故障停住本单并告警,后台可重试失败步骤。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-09-21 14:01:49 +08:00
co-authored by Cursor
parent a5e73c01ae
commit 4381966c4e
26 changed files with 2098 additions and 72 deletions
+88 -34
View File
@@ -586,6 +586,14 @@ def apply_air_lock(
if not is_airline(flow, sender_id):
send_group_text(group_client_of(flow), chat_id, copy.air_airline_only_cabin())
return "not_airline"
from agent.policy.system_exception import reply_paused_ticket, ticket_is_paused
if ticket_is_paused(ticket):
reply_paused_ticket(
ticket=ticket,
reply=lambda text: send_group_text(group_client_of(flow), chat_id, text),
)
return "system_exception_paused"
no = ticket_no(ticket)
name = operator_name(flow, sender_id)
quote = ticket_view(ticket).get("quote") or {}
@@ -612,24 +620,43 @@ def apply_air_lock(
copy.air_lock_fail(work_order_no=no, operator=name, reason="主账未开放锁舱"),
)
return "lock_unavailable"
out = locker(
work_order_no=no,
option_no=picked,
sender_id=sender_id,
operator_name=name,
instruction_text=instruction,
) or {}
if not out.get("ok"):
send_group_text(
group_client_of(flow),
chat_id,
copy.air_lock_fail(
work_order_no=no,
operator=name,
reason=str(out.get("failReason") or out.get("error") or "锁舱失败"),
),
from agent.policy.system_exception import (
STEP_TMS_CABIN,
TYPE_TMS,
SystemExceptionEvent,
call_with_retries,
confirm_system_exception,
)
out = call_with_retries(
lambda: locker(
work_order_no=no,
option_no=picked,
sender_id=sender_id,
operator_name=name,
instruction_text=instruction,
)
return "lock_fail"
or {},
is_ok=lambda x: bool((x or {}).get("ok")),
)
if not out.get("ok"):
confirm_system_exception(
event=SystemExceptionEvent(
exception_type=TYPE_TMS,
step=STEP_TMS_CABIN,
reason="空运锁舱接口调用失败",
service="空运锁舱",
error_code=str(out.get("failReason") or out.get("error") or "LOCK_FAIL"),
work_order_no=no,
ticket_status=ticket_status(ticket),
conversation_kind="group",
conversation_target=chat_id,
extra={"option_no": picked, "instruction": instruction, "sender_id": sender_id},
),
ledger=flow.ledger,
reply=lambda text: send_group_text(group_client_of(flow), chat_id, text),
)
return "system_exception"
exec_no = str(out.get("lockId") or out.get("tmsRequestId") or "").strip()
quote_no = str(out.get("quoteId") or quote_id).strip()
flow.ledger.patch_collab_facts(
@@ -671,6 +698,14 @@ def apply_air_release(
if not is_airline(flow, sender_id):
send_group_text(group_client_of(flow), chat_id, copy.air_airline_only_cabin())
return "not_airline"
from agent.policy.system_exception import reply_paused_ticket, ticket_is_paused
if ticket_is_paused(ticket):
reply_paused_ticket(
ticket=ticket,
reply=lambda text: send_group_text(group_client_of(flow), chat_id, text),
)
return "system_exception_paused"
no = ticket_no(ticket)
if ticket_status(ticket) == "已成交":
send_group_text(group_client_of(flow), chat_id, copy.air_release_deal_blocked(no))
@@ -689,24 +724,43 @@ def apply_air_release(
copy.air_release_fail(work_order_no=no, operator=name, reason="主账未开放释放"),
)
return "release_unavailable"
out = releaser(
work_order_no=no,
lock_id=lock_id,
sender_id=sender_id,
operator_name=name,
instruction_text=instruction,
) or {}
if not out.get("ok"):
send_group_text(
group_client_of(flow),
chat_id,
copy.air_release_fail(
work_order_no=no,
operator=name,
reason=str(out.get("failReason") or out.get("error") or "释放失败"),
),
from agent.policy.system_exception import (
STEP_TMS_CABIN,
TYPE_TMS,
SystemExceptionEvent,
call_with_retries,
confirm_system_exception,
)
out = call_with_retries(
lambda: releaser(
work_order_no=no,
lock_id=lock_id,
sender_id=sender_id,
operator_name=name,
instruction_text=instruction,
)
return "release_fail"
or {},
is_ok=lambda x: bool((x or {}).get("ok")),
)
if not out.get("ok"):
confirm_system_exception(
event=SystemExceptionEvent(
exception_type=TYPE_TMS,
step=STEP_TMS_CABIN,
reason="空运释放舱位接口调用失败",
service="空运舱位",
error_code=str(out.get("failReason") or out.get("error") or "RELEASE_FAIL"),
work_order_no=no,
ticket_status=ticket_status(ticket),
conversation_kind="group",
conversation_target=chat_id,
extra={"lock_id": lock_id, "instruction": instruction, "sender_id": sender_id},
),
ledger=flow.ledger,
reply=lambda text: send_group_text(group_client_of(flow), chat_id, text),
)
return "system_exception"
exec_no = str(out.get("releaseId") or out.get("tmsRequestId") or lock_id).strip()
send_group_text(
group_client_of(flow),
+27 -1
View File
@@ -726,6 +726,17 @@ class AirTextInquiryFlow:
allow_b=allow_b,
chat_fn=chat_fn,
)
from agent.policy.system_exception import handle_extract_tech_fail
llm_phase = handle_extract_tech_fail(
snap=snap,
ledger=self._ledger,
reply=reply,
sender_id=sender_id,
sess=sess,
)
if llm_phase:
return llm_phase
mode = (snap.get("business_line") or "").upper()
facts = dict(snap.get("facts") or {})
if sess and sess.phase in {"clarify", "need_mode"}:
@@ -869,7 +880,12 @@ class AirTextInquiryFlow:
# 询价确认必须先到销售手机,再打 TMS。否则查得快时暂无报价卡会抢先发出。
self._wait_inquiry_sent(str(out_id), work_order_no=no)
tms = self._ledger.query_tms(work_order_no=no, facts=facts)
from agent.policy.system_exception import (
confirm_tms_quote_if_tech,
query_tms_resilient,
)
tms = query_tms_resilient(self._ledger, work_order_no=no, facts=facts)
sess = FlowSession(
sender_id=sender_id,
thread_id=sender_id,
@@ -883,6 +899,16 @@ class AirTextInquiryFlow:
)
if not tms.get("ok"):
# 没拿到 TMS 回包(超时/熔断/主账未出站)不能冒充「暂无报价」
if confirm_tms_quote_if_tech(
ledger=self._ledger,
reply=reply,
work_order_no=no,
tms=tms,
sender_id=sender_id,
ticket_status="询价中",
):
self.drop_current_bookmark(sender_id)
return "system_exception"
if tms.get("classification") == "TMS_QUERY_INCOMPLETE":
logger.warning("tms.query 未出站 wo=%s class=%s", no, tms.get("classification"))
reply("TMS 查询失败,请稍后重试。工单已创建:" + no)
+139
View File
@@ -2473,3 +2473,142 @@ def air_release_deal_blocked(work_order_no: str) -> str:
def air_release_need_lock(work_order_no: str) -> str:
no = (work_order_no or "").strip()
return f"工单{no}还没有有效锁舱,不能释放。"
# —— 系统异常(UC024):销售/群兜底与恢复;告警卡只给有权限的人 ——
TYPE_LLM = "大模型异常"
TYPE_TMS = "TMS异常"
TYPE_GROUP = "建群异常"
STEP_LLM_NO_TICKET = "llm_no_ticket"
STEP_LLM = "llm"
STEP_TMS_QUOTE = "tms_quote"
STEP_TMS_CABIN = "tms_cabin"
STEP_GROUP = "group"
SYS_EXC_LLM_FALLBACK = "大模型服务异常,IT排查中..."
SYS_EXC_TMS_FALLBACK = "TMS接口异常,IT运维排查中..."
SYS_EXC_RECOVERY_TMS = "TMS已恢复正常使用,感谢您的耐心等待。"
SYS_EXC_RECOVERY_LLM = "大模型已恢复正常使用,感谢您的耐心等待。"
SYS_EXC_RECOVERY_CABIN = "空运舱位接口已恢复正常使用,感谢您的耐心等待。"
SYS_EXC_RECOVERY_GROUP = "建群已恢复正常,感谢您的耐心等待。"
IMPACT_LLM_NO_TICKET = "尚未建单,不落后台工单"
IMPACT_LLM = "当前识别暂停,已有字段与工单状态保持不变"
IMPACT_TMS_QUOTE = "当前询价流程暂停,识别字段与原业务节点保持不变"
IMPACT_TMS_CABIN = "当前锁舱/释放暂停,工单状态保持不变"
IMPACT_GROUP = "当前建群/加人暂停,已建群不新建第二个群"
def system_exception_fallback(exception_type: str, *, names: list[str] | None = None) -> str:
"""判定系统异常后,给卡住的会话的兜底话术。"""
kind = (exception_type or "").strip()
if kind == TYPE_LLM:
return SYS_EXC_LLM_FALLBACK
if kind == TYPE_GROUP:
people = [str(x).strip() for x in (names or []) if str(x).strip()]
if not people:
return "无法添加同事,请联系管理员检查企业微信配置。"
joined = "、".join(people)
return f"无法添加 {joined} 同事,请联系管理员检查企业微信配置。"
return SYS_EXC_TMS_FALLBACK
def system_exception_recovery(step: str) -> str:
"""后台点重试成功后,先发这一句,再补没做完的事。"""
key = (step or "").strip()
if key == STEP_LLM or key == STEP_LLM_NO_TICKET:
return SYS_EXC_RECOVERY_LLM
if key == STEP_TMS_CABIN:
return SYS_EXC_RECOVERY_CABIN
if key == STEP_GROUP:
return SYS_EXC_RECOVERY_GROUP
return SYS_EXC_RECOVERY_TMS
def system_exception_impact(step: str) -> str:
"""告警卡「影响范围」,按卡住的步骤。"""
key = (step or "").strip()
if key == STEP_LLM_NO_TICKET:
return IMPACT_LLM_NO_TICKET
if key == STEP_LLM:
return IMPACT_LLM
if key == STEP_TMS_CABIN:
return IMPACT_TMS_CABIN
if key == STEP_GROUP:
return IMPACT_GROUP
return IMPACT_TMS_QUOTE
def system_exception_alert_payload(
*,
work_order_no: str,
exception_type: str,
service: str,
error_code: str,
step: str,
occurred_at: str,
test_prefix: bool = False,
) -> dict[str, Any]:
"""
系统异常通知卡:走询价消息服务,不走询价小助手。
五行对齐现网示例:异常类型、接口/服务、错误码、影响范围、发生时间。
"""
no = (work_order_no or "").strip() or "无"
title = "系统异常通知"
if test_prefix:
title = "[test] " + title
rows = [
{"keyname": "异常类型", "value": (exception_type or "").strip() or "-"},
{"keyname": "接口/服务", "value": (service or "").strip() or "-"},
{"keyname": "错误码", "value": (error_code or "").strip() or "-"},
{"keyname": "影响范围", "value": system_exception_impact(step)},
{"keyname": "发生时间", "value": (occurred_at or "").strip() or "-"},
]
from agent.config import get_settings
base = str(getattr(get_settings(), "public_base_url", "") or "").strip().rstrip("/")
action_url = base or "https://work.weixin.qq.com"
return {
"msgtype": "template_card",
"wecom_app": "notify",
"template_card": {
"card_type": "text_notice",
"source": {"desc": "询价机器人系统告警", "desc_color": 0},
"main_title": {
"title": title,
"desc": f"关联工单 {no}",
},
"horizontal_content_list": rows,
"card_action": {"type": 1, "url": action_url},
},
}
def system_exception_alert_text(
*,
work_order_no: str,
exception_type: str,
service: str,
error_code: str,
step: str,
occurred_at: str,
test_prefix: bool = False,
) -> str:
"""告警卡纯文本兜底(通道降级时仍能看懂)。"""
no = (work_order_no or "").strip() or "无"
head = "系统异常通知"
if test_prefix:
head = "[test] " + head
lines = [
"询价机器人系统告警",
head,
f"关联工单 {no}",
f"异常类型:{(exception_type or '').strip() or '-'}",
f"接口/服务:{(service or '').strip() or '-'}",
f"错误码:{(error_code or '').strip() or '-'}",
f"影响范围:{system_exception_impact(step)}",
f"发生时间:{(occurred_at or '').strip() or '-'}",
]
return "\n".join(lines)
+33 -2
View File
@@ -123,6 +123,17 @@ class LandTextInquiryFlow(SeaTextInquiryFlow):
allow_b=allow_b,
chat_fn=chat_fn,
)
from agent.policy.system_exception import handle_extract_tech_fail
llm_phase = handle_extract_tech_fail(
snap=snap,
ledger=self._ledger,
reply=reply,
sender_id=sender_id,
sess=sess,
)
if llm_phase:
return llm_phase
mode = (snap.get("business_line") or "").upper()
snap_facts = dict(snap.get("facts") or {})
facts = dict(snap_facts)
@@ -308,7 +319,11 @@ class LandTextInquiryFlow(SeaTextInquiryFlow):
) or ""
self._wait_inquiry_sent(str(out_id), work_order_no=no)
tms = self._ledger.query_tms(work_order_no=no, facts=facts, business_line="LAND")
from agent.policy.system_exception import query_tms_resilient
tms = query_tms_resilient(
self._ledger, work_order_no=no, facts=facts, business_line="LAND"
)
sess = FlowSession(
sender_id=sender_id,
thread_id=sender_id,
@@ -335,7 +350,11 @@ class LandTextInquiryFlow(SeaTextInquiryFlow):
reply(copy.LAND_CONFIRM_EXPIRED)
return "land_confirm_expired"
reply(copy.LAND_TMS_RETRYING.format(wo=no))
tms = self._ledger.query_tms(work_order_no=no, facts=facts, business_line="LAND")
from agent.policy.system_exception import query_tms_resilient
tms = query_tms_resilient(
self._ledger, work_order_no=no, facts=facts, business_line="LAND"
)
return self._apply_land_tms(sess=sess, tms=tms, reply=reply, retrying=True)
def _apply_land_tms(
@@ -350,6 +369,18 @@ class LandTextInquiryFlow(SeaTextInquiryFlow):
_ = retrying
no = (sess.work_order_no or "").strip()
if not tms.get("ok"):
from agent.policy.system_exception import confirm_tms_quote_if_tech
if confirm_tms_quote_if_tech(
ledger=self._ledger,
reply=reply,
work_order_no=no,
tms=tms,
sender_id=sess.sender_id,
ticket_status="询价中",
):
self.drop_current_bookmark(sess.sender_id)
return "system_exception"
reply("TMS 查询失败,请稍后重试。工单已创建:" + no)
sess.phase = "tms_tech"
self._save(sess)
+76 -5
View File
@@ -113,6 +113,17 @@ class SeaTextInquiryFlow(AirTextInquiryFlow):
allow_b=allow_b,
chat_fn=chat_fn,
)
from agent.policy.system_exception import handle_extract_tech_fail
llm_phase = handle_extract_tech_fail(
snap=snap,
ledger=self._ledger,
reply=reply,
sender_id=sender_id,
sess=sess,
)
if llm_phase:
return llm_phase
mode = (snap.get("business_line") or "").upper()
facts = dict(snap.get("facts") or {})
if sess and sess.phase in {"clarify", "need_mode"}:
@@ -263,7 +274,14 @@ class SeaTextInquiryFlow(AirTextInquiryFlow):
self._wait_inquiry_sent(str(out_id), work_order_no=no)
tms_facts = dict(assemble_sea_query(facts).get("tms_facts") or facts)
tms = self._ledger.query_tms(work_order_no=no, facts=tms_facts, business_line="SEA")
from agent.policy.system_exception import (
confirm_tms_quote_if_tech,
query_tms_resilient,
)
tms = query_tms_resilient(
self._ledger, work_order_no=no, facts=tms_facts, business_line="SEA"
)
sess = FlowSession(
sender_id=sender_id,
thread_id=sender_id,
@@ -276,6 +294,16 @@ class SeaTextInquiryFlow(AirTextInquiryFlow):
wait_version=1,
)
if not tms.get("ok"):
if confirm_tms_quote_if_tech(
ledger=self._ledger,
reply=reply,
work_order_no=no,
tms=tms,
sender_id=sender_id,
ticket_status="询价中",
):
self.drop_current_bookmark(sender_id)
return "system_exception"
if tms.get("classification") == "TMS_QUERY_INCOMPLETE":
logger.warning("tms.query 未出站 wo=%s class=%s", no, tms.get("classification"))
reply("TMS 查询失败,请稍后重试。工单已创建:" + no)
@@ -549,9 +577,29 @@ class SeaTextInquiryFlow(AirTextInquiryFlow):
userids.append(uid)
client = self.group_client()
created = client.create_group(name=no, userids=userids) or {}
from agent.policy.system_exception import call_with_retries, confirm_group_if_tech
created = call_with_retries(
lambda: client.create_group(name=no, userids=userids) or {},
is_ok=lambda x: bool((x or {}).get("ok") and (x or {}).get("chat_id")),
)
if not created.get("ok") or not created.get("chat_id"):
reply(copy.sea_group_create_fail(reason=str(created.get("error") or "")))
getter = getattr(self._ledger, "get_ticket", None)
st = ""
if callable(getter):
tic = getter(work_order_no=no)
st = str(getattr(tic, "status", "") or (tic.get("status") if isinstance(tic, dict) else "") or "")
confirm_group_if_tech(
ledger=self._ledger,
reply=reply,
work_order_no=no,
sender_id=sess.sender_id,
missing_names=product_names,
reason=str(created.get("error") or "企微建群失败"),
error_code=str(created.get("error") or "GROUP_CREATE"),
ticket_status=st,
extra={"userids": userids, "product_names": product_names},
)
return sess.phase or "wait_collab"
chat_id = str(created.get("chat_id") or "").strip()
@@ -707,9 +755,32 @@ class SeaTextInquiryFlow(AirTextInquiryFlow):
if uid not in userids:
userids.append(uid)
client = self.group_client()
created = client.create_group(name=display_no, userids=userids) or {}
from agent.policy.system_exception import call_with_retries, confirm_group_if_tech
created = call_with_retries(
lambda: client.create_group(name=display_no, userids=userids) or {},
is_ok=lambda x: bool((x or {}).get("ok") and (x or {}).get("chat_id")),
)
if not created.get("ok") or not created.get("chat_id"):
reply(copy.sea_group_create_fail(reason=str(created.get("error") or "")))
real = (sess.work_order_no or "").strip()
mark_no = real if real.upper().startswith("WO") else ""
st = ""
if mark_no:
getter = getattr(self._ledger, "get_ticket", None)
if callable(getter):
tic = getter(work_order_no=mark_no)
st = str(getattr(tic, "status", "") or (tic.get("status") if isinstance(tic, dict) else "") or "")
confirm_group_if_tech(
ledger=self._ledger,
reply=reply,
work_order_no=mark_no,
sender_id=sess.sender_id,
missing_names=product_names,
reason=str(created.get("error") or "企微建群失败"),
error_code=str(created.get("error") or "GROUP_CREATE"),
ticket_status=st,
extra={"userids": userids, "product_names": product_names, "display_no": display_no},
)
return sess.phase or "handoff_offer"
chat_id = str(created.get("chat_id") or "").strip()
brief = copy.handoff_group_brief(
@@ -0,0 +1,816 @@
"""
系统异常(UC024):技术故障判定、默默重试、工单两列、会话兜底、运维告警、后台重试。
本文件职责:只处理大模型 / TMS / 建群加人的超时、报错、空结果。
禁止:把无报价、字段不齐、没配人当成系统异常;禁止为记异常去建空工单;
禁止在回调线程等完整 LLM / 查 TMS。
"""
from __future__ import annotations
import logging
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Callable, Optional
from zoneinfo import ZoneInfo
from agent.policy import inquiry_copy as copy
logger = logging.getLogger(__name__)
_SHANGHAI = ZoneInfo("Asia/Shanghai")
TYPE_LLM = copy.TYPE_LLM
TYPE_TMS = copy.TYPE_TMS
TYPE_GROUP = copy.TYPE_GROUP
STEP_LLM_NO_TICKET = copy.STEP_LLM_NO_TICKET
STEP_LLM = copy.STEP_LLM
STEP_TMS_QUOTE = copy.STEP_TMS_QUOTE
STEP_TMS_CABIN = copy.STEP_TMS_CABIN
STEP_GROUP = copy.STEP_GROUP
MAX_AUTO_RETRIES = 3
EnqueueFn = Callable[..., tuple[bool, str]]
ReplyFn = Callable[[str], Any]
OkFn = Callable[[Any], bool]
@dataclass
class SystemExceptionEvent:
"""一次已确认的系统异常(自动重试 3 次仍失败之后)。"""
exception_type: str
step: str
reason: str
service: str
error_code: str
work_order_no: str = ""
ticket_status: str = ""
conversation_kind: str = "private"
conversation_target: str = ""
missing_names: list[str] = field(default_factory=list)
extra: dict[str, Any] = field(default_factory=dict)
def is_tms_tech_failure(tms: dict[str, Any] | None) -> bool:
"""
TMS 技术故障:接口失败/超时。
正常返回但没报价、字段不齐未出站,都不算。
"""
row = tms or {}
if row.get("ok"):
return False
cls = str(row.get("classification") or "").strip()
if cls in {"TMS_NO_QUOTE_1002", "TMS_QUERY_INCOMPLETE"}:
return False
return True
def should_mark_ticket(*, work_order_no: str, ticket_status: str) -> bool:
"""有正式工单且不是已关闭,才写两列。"""
if not (work_order_no or "").strip():
return False
return (ticket_status or "").strip() != "已关闭"
def ticket_exception_type(ticket: Any) -> str:
"""从主账工单(对象或字典)读当前系统异常类型。"""
if ticket is None:
return ""
if isinstance(ticket, dict):
return str(
ticket.get("system_exception")
or ticket.get("systemException")
or ""
).strip()
return str(getattr(ticket, "system_exception", "") or "").strip()
def ticket_is_paused(ticket: Any) -> bool:
"""工单两列有值即暂停这一票。"""
return bool(ticket_exception_type(ticket))
def extract_work_order_nos(text: str) -> list[str]:
"""从原话抽出正式工单号(WO + 日期流水)。不认虚拟号。"""
import re
found = re.findall(r"\bWO\d{8,}\b", text or "", flags=re.IGNORECASE)
out: list[str] = []
seen: set[str] = set()
for raw in found:
no = raw.upper()
if no in seen:
continue
seen.add(no)
out.append(no)
return out
def call_with_retries(
fn: Callable[[], Any],
*,
is_ok: OkFn,
attempts: int = MAX_AUTO_RETRIES,
) -> Any:
"""
默默重试。中途成功立刻返回;会话里不说「正在重试」。
调用方在 Worker 槽使用,禁止放进企微回调线程。
"""
last: Any = None
times = max(1, int(attempts))
for idx in range(times):
try:
last = fn()
except Exception as exc: # noqa: BLE001
logger.warning("系统异常自动重试捕获异常 n=%s/%s err=%s", idx + 1, times, exc)
last = {"ok": False, "error": str(exc), "classification": "TECH_EXCEPTION"}
continue
if is_ok(last):
return last
logger.info("系统异常自动重试未成功 n=%s/%s", idx + 1, times)
return last
def _now_text(now: datetime | None) -> str:
stamp = now or datetime.now(_SHANGHAI)
return stamp.strftime("%Y-%m-%d %H:%M:%S")
def _test_prefix() -> bool:
try:
from agent.config import get_settings
return (get_settings().ytd_env or "").strip().lower() == "test"
except Exception: # noqa: BLE001
return False
def notify_app_ready() -> bool:
"""询价消息服务已配才推告警。"""
from agent.policy.deal_outcome import notify_app_ready as deal_ready
return deal_ready()
def _enqueue_default(
*,
touser: str,
content: str,
dedupe_key: str,
payload: dict[str, Any],
) -> tuple[bool, str]:
from agent.channel.queue import get_message_store
return get_message_store().enqueue_outbound(
touser=touser,
content=content,
dedupe_key=dedupe_key,
payload=payload,
)
def _list_notify_users(ledger: Any) -> list[dict[str, Any]]:
lister = getattr(ledger, "list_exception_notify_users", None)
if not callable(lister):
return []
try:
rows = list(lister() or [])
except Exception:
logger.exception("列出系统异常通知人失败")
return []
out: list[dict[str, Any]] = []
seen: set[str] = set()
for row in rows:
if not isinstance(row, dict):
continue
uid = str(row.get("wecomId") or row.get("wecom_id") or "").strip()
if not uid or uid in seen:
continue
seen.add(uid)
out.append(row)
return out
def confirm_system_exception(
*,
event: SystemExceptionEvent,
ledger: Any = None,
reply: Optional[ReplyFn] = None,
enqueue: Optional[EnqueueFn] = None,
notify_ready: Optional[bool] = None,
now: Optional[datetime] = None,
) -> dict[str, Any]:
"""
3 次仍失败后:回话、有单则写两列、推有权限的人。
无正式工单:不建单、不 mark。已关闭:不 mark。没有可推的人:照样回话。
"""
no = (event.work_order_no or "").strip()
if reply is not None:
reply(copy.system_exception_fallback(event.exception_type, names=event.missing_names))
marked = False
if should_mark_ticket(work_order_no=no, ticket_status=event.ticket_status) and ledger is not None:
marker = getattr(ledger, "mark_system_exception", None)
if callable(marker):
payload = {
"step": event.step,
"conversation_kind": event.conversation_kind,
"conversation_target": event.conversation_target,
"missing_names": list(event.missing_names),
"service": event.service,
"error_code": event.error_code,
"extra": dict(event.extra or {}),
}
try:
result = marker(
work_order_no=no,
system_exception=event.exception_type,
system_exception_reason=(event.reason or "").strip() or event.exception_type,
step=event.step,
payload=payload,
) or {}
marked = bool(result.get("ok", True))
except Exception:
logger.exception("写入系统异常失败 wo=%s", no)
marked = False
ready = notify_app_ready() if notify_ready is None else bool(notify_ready)
notified = 0
users = _list_notify_users(ledger) if ledger is not None else []
if ready and users:
occurred = _now_text(now)
test_prefix = _test_prefix()
payload = copy.system_exception_alert_payload(
work_order_no=no,
exception_type=event.exception_type,
service=event.service,
error_code=event.error_code,
step=event.step,
occurred_at=occurred,
test_prefix=test_prefix,
)
text = copy.system_exception_alert_text(
work_order_no=no,
exception_type=event.exception_type,
service=event.service,
error_code=event.error_code,
step=event.step,
occurred_at=occurred,
test_prefix=test_prefix,
)
sender = enqueue or _enqueue_default
for row in users:
uid = str(row.get("wecomId") or row.get("wecom_id") or "").strip()
if not uid:
continue
try:
ok, info = sender(
touser=uid,
content=text,
dedupe_key=f"sys-exc:{no or 'none'}:{event.step}:{uid}:{occurred}",
payload=dict(payload),
)
if ok:
notified += 1
else:
logger.warning("系统异常告警入队失败 uid=%s info=%s", uid, info)
except Exception:
logger.exception("系统异常告警入队异常 uid=%s", uid)
return {
"ok": True,
"marked": marked,
"notified": notified,
"work_order_no": no,
"exception_type": event.exception_type,
"step": event.step,
}
def clear_system_exception(*, ledger: Any, work_order_no: str) -> dict[str, Any]:
"""重试成功后清两列。"""
no = (work_order_no or "").strip()
if not no:
return {"ok": False, "error": "blank"}
clearer = getattr(ledger, "clear_system_exception", None)
if not callable(clearer):
return {"ok": False, "error": "no_clear"}
return clearer(work_order_no=no) or {"ok": False}
def reply_paused_ticket(*, ticket: Any, reply: ReplyFn) -> None:
"""销售/群提到暂停工单号:只重复兜底,不重跑。"""
kind = ticket_exception_type(ticket) or TYPE_TMS
names: list[str] = []
if isinstance(ticket, dict):
payload = ticket.get("system_exception_payload") or ticket.get("systemExceptionPayload") or {}
if isinstance(payload, dict):
raw = payload.get("missing_names") or []
names = [str(x).strip() for x in raw if str(x).strip()]
else:
payload = getattr(ticket, "system_exception_payload", {}) or {}
if isinstance(payload, dict):
raw = payload.get("missing_names") or []
names = [str(x).strip() for x in raw if str(x).strip()]
reply(copy.system_exception_fallback(kind, names=names))
def query_tms_resilient(ledger: Any, **kwargs: Any) -> dict[str, Any]:
"""TMS 查价:最多 3 次,ok=true(含无价)即成功。"""
getter = getattr(ledger, "query_tms", None)
if not callable(getter):
return {"ok": False, "classification": "TMS_TECH_FAILURE", "has_price": False, "quote": {}}
out = call_with_retries(
lambda: getter(**kwargs),
is_ok=lambda x: bool((x or {}).get("ok")),
)
return out if isinstance(out, dict) else {"ok": False, "classification": "TMS_TECH_FAILURE"}
def confirm_tms_quote_if_tech(
*,
ledger: Any,
reply: ReplyFn,
work_order_no: str,
tms: dict[str, Any],
sender_id: str,
ticket_status: str = "询价中",
facts: dict[str, Any] | None = None,
) -> bool:
"""技术失败则标 TMS异常并回话告警。返回是否已按系统异常处理。"""
if not is_tms_tech_failure(tms):
return False
err = str(tms.get("classification") or tms.get("error") or "TMS_TECH_FAILURE").strip()
confirm_system_exception(
event=SystemExceptionEvent(
exception_type=TYPE_TMS,
step=STEP_TMS_QUOTE,
reason="TMS /quote/v2 接口调用失败",
service="TMS报价",
error_code=err or "TMS_TECH_FAILURE",
work_order_no=work_order_no,
ticket_status=ticket_status,
conversation_kind="private",
conversation_target=sender_id,
extra={"facts": dict(facts or tms.get("facts") or {})},
),
ledger=ledger,
reply=reply,
)
return True
def confirm_group_if_tech(
*,
ledger: Any,
reply: ReplyFn,
work_order_no: str,
sender_id: str,
missing_names: list[str],
reason: str,
error_code: str,
ticket_status: str = "",
extra: dict[str, Any] | None = None,
chat_id: str = "",
) -> None:
"""建群或加人技术失败(已重试 3 次)后确认异常。"""
confirm_system_exception(
event=SystemExceptionEvent(
exception_type=TYPE_GROUP,
step=STEP_GROUP,
reason=(reason or "").strip() or "企微建群/加人失败",
service="企微建群" if not chat_id else "企微加人",
error_code=(error_code or "GROUP_FAIL").strip(),
work_order_no=work_order_no,
ticket_status=ticket_status or "已报价",
conversation_kind="private",
conversation_target=sender_id,
missing_names=list(missing_names),
extra=dict(extra or {}),
),
ledger=ledger,
reply=reply,
)
def confirm_llm_if_tech(
*,
ledger: Any,
reply: ReplyFn,
sender_id: str,
reason: str,
work_order_no: str = "",
ticket_status: str = "",
conversation_kind: str = "private",
conversation_target: str = "",
extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""大模型超时/空/报错。无工单不建单。"""
no = (work_order_no or "").strip()
step = STEP_LLM if no else STEP_LLM_NO_TICKET
return confirm_system_exception(
event=SystemExceptionEvent(
exception_type=TYPE_LLM,
step=step,
reason=(reason or "").strip() or "模型返回空结果",
service="大模型识别",
error_code="EMPTY" if "空" in (reason or "") else "LLM_FAIL",
work_order_no=no,
ticket_status=ticket_status,
conversation_kind=conversation_kind,
conversation_target=conversation_target or sender_id,
extra=dict(extra or {}),
),
ledger=ledger,
reply=reply,
)
def handle_extract_tech_fail(
*,
snap: dict[str, Any],
ledger: Any,
reply: ReplyFn,
sender_id: str,
sess: Any = None,
) -> str:
"""抽字段模型超时/报错:标大模型异常并回话。返回阶段名,未失败返回空串。"""
if not isinstance(snap, dict) or not snap.get("tech_fail"):
return ""
wo = str(getattr(sess, "work_order_no", "") or "") if sess is not None else ""
st = str(getattr(sess, "status", "") or "") if sess is not None else ""
confirm_llm_if_tech(
ledger=ledger,
reply=reply,
sender_id=sender_id,
reason=str(snap.get("tech_error") or "模型返回空结果"),
work_order_no=wo,
ticket_status=st,
)
return "system_exception"
def _send_to_conversation(*, kind: str, target: str, text: str, extra: dict[str, Any] | None = None) -> str:
"""恢复/兜底发到卡住的会话:私聊走 outbox,群走 appchat。可带卡片 extra。"""
body = (text or "").strip()
dest = (target or "").strip()
if not body or not dest:
return ""
if (kind or "").strip() == "group":
try:
from agent.channel.outbox.sender import WeComAppClient
from agent.config import get_settings
s = get_settings()
client = WeComAppClient(
corp_id=s.wecom_corp_id,
secret=s.wecom_secret,
agent_id=int(s.wecom_agent_id or "0") or 0,
)
client.send_group(chat_id=dest, content=body)
return dest
except Exception:
logger.exception("系统异常群回话失败 chat=%s", dest)
return ""
try:
from agent.channel.queue import get_message_store
ok, info = get_message_store().enqueue_outbound(
touser=dest,
content=body,
dedupe_key=f"sys-exc-talk:{dest}:{hash(body) & 0xFFFF}",
payload=dict(extra or {}),
)
return str(info or "") if ok else ""
except Exception:
logger.exception("系统异常私聊回话失败 uid=%s", dest)
return ""
def run_admin_retry(work_order_no: str, *, ledger: Any = None) -> dict[str, Any]:
"""
后台点重试:只重跑失败步。成功先发恢复话术再继续;再失败再标再告警。
在 Worker HTTP 槽调用,禁止放进回调线程。
"""
no = (work_order_no or "").strip()
book = ledger
if book is None:
from agent.policy.air_text_flow import build_ledger
book = build_ledger()
getter = getattr(book, "get_ticket", None)
ticket = getter(work_order_no=no) if callable(getter) else None
if ticket is None:
getter = getattr(book, "get_for_agent", None)
ticket = getter(work_order_no=no, sender_id="") if callable(getter) else None
if not ticket_is_paused(ticket):
return {"ok": False, "error": "not_paused"}
if isinstance(ticket, dict):
step = str(ticket.get("system_exception_step") or "").strip()
payload = ticket.get("system_exception_payload") or {}
status = str(ticket.get("status") or "")
facts = dict(ticket.get("facts") or {})
line = str(ticket.get("business_line") or ticket.get("businessLine") or "")
else:
step = str(getattr(ticket, "system_exception_step", "") or "").strip()
payload = dict(getattr(ticket, "system_exception_payload", {}) or {})
status = str(getattr(ticket, "status", "") or "")
facts = dict(getattr(ticket, "facts", {}) or {})
line = str(getattr(ticket, "business_line", "") or "")
if not isinstance(payload, dict):
payload = {}
kind = str(payload.get("conversation_kind") or "private")
target = str(payload.get("conversation_target") or "")
extra = dict(payload.get("extra") or {})
if extra.get("facts") and not facts:
facts = {str(k): str(v) for k, v in dict(extra.get("facts") or {}).items()}
line = (line or str(extra.get("business_line") or "")).upper()
def reply(text: str, extra_msg: dict[str, Any] | None = None) -> str:
return _send_to_conversation(kind=kind, target=target, text=text, extra=extra_msg)
if step == STEP_TMS_QUOTE or not step:
tms = query_tms_resilient(book, work_order_no=no, facts=facts, business_line=line)
if is_tms_tech_failure(tms):
confirm_tms_quote_if_tech(
ledger=book,
reply=reply,
work_order_no=no,
tms=tms,
sender_id=target,
ticket_status=status,
facts=facts,
)
return {"ok": False, "error": "still_fail", "step": STEP_TMS_QUOTE}
reply(copy.system_exception_recovery(STEP_TMS_QUOTE))
clear_system_exception(ledger=book, work_order_no=no)
sender = target if kind != "group" else str(extra.get("sender_id") or "")
_resume_tms_quote(
ledger=book,
work_order_no=no,
facts=facts,
line=line,
sender_id=sender,
tms=tms,
reply=reply,
)
return {"ok": True, "step": STEP_TMS_QUOTE}
if step == STEP_GROUP:
extra = dict(payload.get("extra") or extra)
userids = [str(x).strip() for x in (extra.get("userids") or []) if str(x).strip()]
chat_id = str(extra.get("chat_id") or payload.get("chat_id") or "").strip()
try:
from agent.channel.outbox.sender import WeComAppClient
from agent.config import get_settings
s = get_settings()
client = WeComAppClient(
corp_id=s.wecom_corp_id,
secret=s.wecom_secret,
agent_id=int(s.wecom_agent_id or "0") or 0,
)
if chat_id and userids:
result = call_with_retries(
lambda: client.add_group_members(chat_id=chat_id, userids=userids) or {},
is_ok=lambda x: bool((x or {}).get("ok")),
)
elif userids:
name = no or str(extra.get("display_no") or "询价协同")
result = call_with_retries(
lambda: client.create_group(name=name, userids=userids) or {},
is_ok=lambda x: bool((x or {}).get("ok") and (x or {}).get("chat_id")),
)
else:
result = {"ok": False, "error": "no_users"}
except Exception as exc: # noqa: BLE001
result = {"ok": False, "error": str(exc)}
if not (result or {}).get("ok"):
confirm_group_if_tech(
ledger=book,
reply=reply,
work_order_no=no,
sender_id=target,
missing_names=[str(x) for x in (payload.get("missing_names") or extra.get("product_names") or [])],
reason=str((result or {}).get("error") or "企微建群/加人失败"),
error_code=str((result or {}).get("error") or "GROUP_FAIL"),
ticket_status=status,
extra=extra,
chat_id=chat_id,
)
return {"ok": False, "error": "still_fail", "step": STEP_GROUP}
new_chat = str((result or {}).get("chat_id") or chat_id).strip()
binder = getattr(book, "bind_collab_group", None)
if new_chat and callable(binder) and not chat_id:
binder(
work_order_no=no,
chat_id=new_chat,
member_ids=userids,
product_names=[str(x) for x in (extra.get("product_names") or [])],
product_ids=[str(x) for x in (extra.get("product_ids") or [])],
)
reply(copy.system_exception_recovery(STEP_GROUP))
clear_system_exception(ledger=book, work_order_no=no)
return {"ok": True, "step": STEP_GROUP}
if step in {STEP_LLM, STEP_LLM_NO_TICKET}:
reply(copy.system_exception_recovery(STEP_LLM))
if no:
clear_system_exception(ledger=book, work_order_no=no)
reply("请把刚才那句话再发一次。")
return {"ok": True, "step": step}
if step == STEP_TMS_CABIN:
return _resume_cabin_step(
ledger=book,
work_order_no=no,
extra=extra,
target=target,
status=status,
reply=reply,
)
return {"ok": False, "error": "unknown_step", "step": step}
def _flow_for_line(line: str) -> Any:
"""按业务线取私聊流程,用来重试后把报价卡/书签补回去。"""
key = (line or "").upper()
if key == "LAND":
from agent.policy.land_text_flow import get_land_text_flow
return get_land_text_flow()
if key == "AIR":
from agent.policy.air_text_flow import get_air_text_flow
return get_air_text_flow()
from agent.policy.sea_text_flow import get_sea_text_flow
return get_sea_text_flow()
def _resume_tms_quote(
*,
ledger: Any,
work_order_no: str,
facts: dict[str, Any],
line: str,
sender_id: str,
tms: dict[str, Any],
reply: ReplyFn,
) -> None:
"""重试查价成功:写入报价并按原流程出卡,销售能继续拉群。"""
from agent.policy.air_text_flow import FlowSession
no = (work_order_no or "").strip()
biz = (line or "SEA").upper()
if biz not in {"SEA", "LAND", "AIR"}:
biz = "SEA"
flow = _flow_for_line(biz)
sender = (sender_id or "").strip()
if not tms.get("has_price"):
miss = getattr(flow, "_emit_tms_miss", None)
if callable(miss):
miss(reply, no)
else:
reply(f"工单{no}暂无 TMS 报价。")
if sender:
sess = FlowSession(
sender_id=sender,
thread_id=sender,
phase="tms_miss",
business_line=biz,
facts=dict(facts or {}),
work_order_no=no,
status="询价中",
allowed=(copy.BTN_PULL_COLLAB,) if biz != "AIR" else (),
wait_version=1,
)
flow._save(sess)
return
quote = dict(tms.get("quote") or {})
up = ledger.upsert_quote(work_order_no=no, quote=quote, to_status="已报价")
if not up.get("ok"):
reply("主账写入报价失败,不能当作已报价。")
return
allowed = (copy.BTN_SKIP_COLLAB,) if biz == "AIR" else (copy.BTN_PULL_COLLAB, copy.BTN_SKIP_COLLAB)
if sender:
sess = FlowSession(
sender_id=sender,
thread_id=sender,
phase="wait_collab",
business_line=biz,
facts=dict(facts or {}),
work_order_no=no,
quote=quote,
status="已报价",
allowed=allowed,
wait_version=1,
)
copy.remember_tms_quote(sess, quote)
flow._save(sess)
if biz == "AIR":
reply(
copy.tms_hit_card(work_order_no=no, quote=quote, status="已报价"),
copy.tms_hit_wecom_payload(work_order_no=no, quote=quote, status="已报价"),
)
return
emit = getattr(flow, "_emit_sea_hit", None)
if callable(emit):
emit(
reply,
work_order_no=no,
quote=quote,
status="已报价",
business_line=biz,
)
return
reply(f"工单{no}已重新查到报价。")
def _resume_cabin_step(
*,
ledger: Any,
work_order_no: str,
extra: dict[str, Any],
target: str,
status: str,
reply: ReplyFn,
) -> dict[str, Any]:
"""重试空运锁舱或释放:payload 里有 lock_id 就是释放。"""
no = (work_order_no or "").strip()
lock_id = str(extra.get("lock_id") or "").strip()
option_no = str(extra.get("option_no") or "").strip() or "AIR-OPT-01"
instruction = str(extra.get("instruction") or "")
sender = str(extra.get("sender_id") or target or "").strip()
if lock_id:
releaser = getattr(ledger, "release_cabin", None)
out = call_with_retries(
lambda: releaser(
work_order_no=no,
lock_id=lock_id,
sender_id=sender,
instruction_text=instruction,
)
or {}
if callable(releaser)
else {"ok": False, "error": "no_release"},
is_ok=lambda x: bool((x or {}).get("ok")),
)
service = "空运舱位"
reason = "空运释放舱位接口调用失败"
else:
locker = getattr(ledger, "lock_cabin", None)
out = call_with_retries(
lambda: locker(
work_order_no=no,
option_no=option_no,
sender_id=sender,
instruction_text=instruction,
)
or {}
if callable(locker)
else {"ok": False, "error": "no_lock"},
is_ok=lambda x: bool((x or {}).get("ok")),
)
service = "空运锁舱"
reason = "空运锁舱接口调用失败"
if not (out or {}).get("ok"):
confirm_system_exception(
event=SystemExceptionEvent(
exception_type=TYPE_TMS,
step=STEP_TMS_CABIN,
reason=reason,
service=service,
error_code=str((out or {}).get("failReason") or (out or {}).get("error") or "CABIN_FAIL"),
work_order_no=no,
ticket_status=status,
conversation_kind="group",
conversation_target=target,
extra=dict(extra),
),
ledger=ledger,
reply=reply,
)
return {"ok": False, "error": "still_fail", "step": STEP_TMS_CABIN}
reply(copy.system_exception_recovery(STEP_TMS_CABIN))
clear_system_exception(ledger=ledger, work_order_no=no)
if not lock_id:
exec_no = str((out or {}).get("lockId") or (out or {}).get("tmsRequestId") or "").strip()
patcher = getattr(ledger, "patch_collab_facts", None)
if exec_no and callable(patcher):
patcher(
work_order_no=no,
facts={"__air_lock_id": exec_no, "__air_lock_option": option_no, "__pending_air_lock": ""},
sender_id=sender,
)
reply(copy.air_lock_ok(work_order_no=no, operator="同事", option_no=option_no, exec_no=exec_no, quote_no=""))
else:
reply(copy.air_release_ok(work_order_no=no, operator="同事", option_no="", exec_no=str((out or {}).get("releaseId") or lock_id), quote_no=""))
return {"ok": True, "step": STEP_TMS_CABIN}