252 lines
12 KiB
Python
252 lines
12 KiB
Python
 
|
"""T08 专项离线质检：统一适配 fixture、逐任务/用途准入、冻结 checkpoint 与上下文预算。
|
|||
|
|
|
|||
|
|
依据：docs/v2/11-judgment-providers.md §8、docs/v2/06-validation.md §4/§6；
|
|||
|
|
BR-17/18/19/20/37/38/39、E-10/13/14/15/25、AC-16/17/18/19/20/38/39/40/41。
|
|||
|
|
|
|||
|
|
本批**只交付离线 fixture 路径**。没有 Jev 凭据、没有 Laya 权重与授权，
|
|||
|
|
因此：
|
|||
|
|
- live 传输层只留注入点，未注入时明确 `LiveTransportUnavailable`；
|
|||
|
|
- fixture 通过只证明适配代码结构成立，**不证明**中文专项效果合格
|
|||
|
|
（06-validation.md §6："fixture 通过仅证明适配代码结构"）。
|
|||
|
|
|
|||
|
|
四条不可让渡的规则：
|
|||
|
|
1. **逐任务逐用途准入**：`task_validations` 里没有 (task, usage) 的 passed 记录，
|
|||
|
|
就不执行；服务能连通、别的任务通过、装好了权重，都不算准入（AC-38）。
|
|||
|
|
2. **权重不可隐式换**：运行冻结 `checkpoint_ref`；提供方实际权重不一致即拒绝，
|
|||
|
|
不按语言或负载偷偷路由到别的模型（AC-39）。
|
|||
|
|
3. **超预算不外发**：超出已验证上下文预算时返回 E-25 并保留原证据范围，
|
|||
|
|
绝不截断后半段照样发送（AC-40 / BR-39）。
|
|||
|
|
4. **歧义不硬选**：映射未知、中文否定、条件例外、来源矛盾都落 unknown/人工复核，
|
|||
|
|
不挑一个"看起来正面"的选项（AC-18）。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
from contracts.judgment import JudgmentProfile
|
|||
|
|
from contracts.prescreen import JudgmentConfig
|
|||
|
|
|
|||
|
|
FIXTURE_ROOT = Path(__file__).resolve().parent.parent / "fixtures" / "judgment"
|
|||
|
|
|
|||
|
|
# 离线预算估算：中文按 1 字≈1 token、ASCII 按 4 字符≈1 token 的保守上界。
|
|||
|
|
# 真实预算必须按选定模型的 tokenizer 实测填写（11 §8.1），这里的估算只用于
|
|||
|
|
# 离线 fixture 与"宁可拒绝"的方向，不作为生产阈值。
|
|||
|
|
_CJK_FLOOR = 0x2E80
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ContextBudgetExceeded(RuntimeError):
|
|||
|
|
"""超出已验证上下文预算（E-25）。携带原证据范围，供上层保留引用。"""
|
|||
|
|
|
|||
|
|
def __init__(self, message: str, evidence_ids=()):
|
|||
|
|
super().__init__(message)
|
|||
|
|
self.evidence_ids = list(evidence_ids)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class TaskNotValidated(ValueError):
|
|||
|
|
"""该任务/用途没有 passed 的验收登记（AC-38）。"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
class CheckpointMismatch(ValueError):
|
|||
|
|
"""实际权重修订与运行冻结值不一致（AC-39）。"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
class EgressNotApproved(PermissionError):
|
|||
|
|
"""未获批准的外部出口：不发送（AC-16/19）。"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
class LiveTransportUnavailable(RuntimeError):
|
|||
|
|
"""没有可用的 live 传输层（本批未接入硬件与授权）。"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def estimate_tokens(text: str) -> int:
|
|||
|
|
"""保守估算 token 数：CJK 每字 1，其它每 4 字符 1，向上取整。"""
|
|||
|
|
if not text:
|
|||
|
|
return 0
|
|||
|
|
cjk = sum(1 for char in text if ord(char) >= _CJK_FLOOR)
|
|||
|
|
other = len(text) - cjk
|
|||
|
|
return cjk + (other + 3) // 4
|
|||
|
|
|
|||
|
|
|
|||
|
|
def load_fixture(name: str) -> dict:
|
|||
|
|
return json.loads((FIXTURE_ROOT / name).read_text(encoding="utf-8"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------ 准入
|
|||
|
|
def validation_for(profile: JudgmentProfile, task: str, usage: str):
|
|||
|
|
"""取出该任务与用途的登记；没有登记返回 None，不做"默认通过"。"""
|
|||
|
|
for item in profile.task_validations:
|
|||
|
|
if item.task == task and item.allowed_usage == usage:
|
|||
|
|
return item
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def assert_task_allowed(profile: JudgmentProfile, task: str, usage: str) -> None:
|
|||
|
|
"""逐任务逐用途准入（AC-38 / AC-41）。
|
|||
|
|
|
|||
|
|
offline 与 runtime 分开登记：只有 offline 通过的配置不能进 runtime；
|
|||
|
|
runtime 还要求验收是独立评测——同模型自评或其生成标签不算证据。
|
|||
|
|
"""
|
|||
|
|
if profile.readiness != "ready":
|
|||
|
|
raise TaskNotValidated("该专项配置尚未就绪，不能执行任何任务")
|
|||
|
|
record = validation_for(profile, task, usage)
|
|||
|
|
if record is None or record.decision != "passed":
|
|||
|
|
raise TaskNotValidated(
|
|||
|
|
f"该配置未在用途 {usage} 通过任务 {task} 的验收；"
|
|||
|
|
"连通、安装或其它任务通过都不能替代该任务准入")
|
|||
|
|
if usage == "runtime" and not record.independent:
|
|||
|
|
raise TaskNotValidated("runtime 准入必须是独立评测结果，同模型自评不能作为效果证据")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def assert_checkpoint_frozen(config: JudgmentConfig, profile: JudgmentProfile) -> None:
|
|||
|
|
"""运行冻结值优先：配置登记被改动后，旧运行仍按原值执行（AC-20/39）。"""
|
|||
|
|
if profile.checkpoint_ref == "unresolved":
|
|||
|
|
raise CheckpointMismatch("权重修订不可复现（unresolved），不得用于任何执行")
|
|||
|
|
if config.checkpoint_ref and config.checkpoint_ref != profile.checkpoint_ref:
|
|||
|
|
raise CheckpointMismatch(
|
|||
|
|
"登记配置的权重修订与本次运行冻结值不一致；请新建运行而不是就地替换")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def assert_within_budget(question: dict, profile: JudgmentProfile) -> None:
|
|||
|
|
"""调用前算预算；超限即拒绝，不截断、不外发（AC-40）。
|
|||
|
|
|
|||
|
|
正文（含提示词）与选项分开计：选项预算不能被正文的余量顶掉，反之亦然。
|
|||
|
|
"""
|
|||
|
|
body = "".join(str(item) for item in question.get("evidence_text") or ())
|
|||
|
|
prompt = str(question.get("prompt") or "")
|
|||
|
|
options = " ".join(str(item) for item in question.get("options") or ())
|
|||
|
|
evidence_ids = question.get("evidence_ids") or ()
|
|||
|
|
|
|||
|
|
body_tokens = estimate_tokens(body) + estimate_tokens(prompt)
|
|||
|
|
if body_tokens > profile.input_token_limit:
|
|||
|
|
raise ContextBudgetExceeded(
|
|||
|
|
f"正文约 {body_tokens} token，超出该模型已验证输入预算 "
|
|||
|
|
f"{profile.input_token_limit}；请缩小问题或转人工复核",
|
|||
|
|
evidence_ids=evidence_ids)
|
|||
|
|
|
|||
|
|
option_tokens = estimate_tokens(options)
|
|||
|
|
if option_tokens > profile.option_token_limit:
|
|||
|
|
raise ContextBudgetExceeded(
|
|||
|
|
f"选项约 {option_tokens} token，超出该模型已验证选项预算 "
|
|||
|
|
f"{profile.option_token_limit}；请缩减候选项或转人工复核",
|
|||
|
|
evidence_ids=evidence_ids)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------ 映射
|
|||
|
|
def _unknown(reason: str) -> dict:
|
|||
|
|
return {"kind": "unknown", "reason_code": reason}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def map_local_answer(raw: dict, question: dict, profile: JudgmentProfile) -> dict:
|
|||
|
|
"""本地模型原始输出 → 统一契约形状。
|
|||
|
|
|
|||
|
|
歧义一律落 unknown/inconclusive：映射未知、中文否定、条件例外、来源矛盾
|
|||
|
|
都不许"挑一个正面结果"（AC-18）。`confidence` 只作为 self_reported 指标
|
|||
|
|
保留原语义，**不**换算成任何概率。
|
|||
|
|
"""
|
|||
|
|
mapping = (raw.get("mapping") or {})
|
|||
|
|
if raw.get("conflicts"):
|
|||
|
|
return _unknown("inconclusive")
|
|||
|
|
if raw.get("exception_applies"):
|
|||
|
|
return _unknown("inconclusive")
|
|||
|
|
label = raw.get("label")
|
|||
|
|
if not label:
|
|||
|
|
return _unknown("missing_evidence" if not raw.get("evidence_ids") else "inconclusive")
|
|||
|
|
option = mapping.get(label)
|
|||
|
|
if option is None:
|
|||
|
|
# 例如中文否定表述（"不符合"）未登记映射：不得当成正面选项使用
|
|||
|
|
return _unknown("inconclusive")
|
|||
|
|
metrics = []
|
|||
|
|
if raw.get("confidence") is not None:
|
|||
|
|
metrics.append({"name": "self_confidence", "value": float(raw["confidence"]),
|
|||
|
|
"semantics": "模型自报置信度，未经校准，与审批概率无关",
|
|||
|
|
"source": "self_reported"})
|
|||
|
|
return {"kind": question["type"], "value": option, "native_metrics": metrics,
|
|||
|
|
"actual_checkpoint_ref": raw.get("actual_checkpoint_ref")}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# jev 厂商字段名与本地完全不同：映射表按 mapping_version 固定，主审不读厂商字段
|
|||
|
|
# （AC-17）。新增厂商版本要新增一行映射并单独登记验收，不改这一层的行为。
|
|||
|
|
# 本行只登记 choice 题型的映射：其它题型没有登记就落 unknown，而不是借这一行
|
|||
|
|
# 顺带支持（06-validation.md §6："阈值和映射按提供方/模型/题型独立登记"）。
|
|||
|
|
_DEV_VENDOR_CHOICES = {
|
|||
|
|
"APPROVE": "supported",
|
|||
|
|
"REJECT": "unsupported",
|
|||
|
|
# COND_APPROVE（附条件）刻意**不**映射为通过：附条件不等于通过（BR-21）
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def map_jev_answer(raw: dict, question: dict, profile: JudgmentProfile) -> dict:
|
|||
|
|
"""Jev 厂商响应 → 统一契约形状（fixture 验证用，不发送任何请求）。"""
|
|||
|
|
if question["type"] != "choice":
|
|||
|
|
return _unknown("inconclusive")
|
|||
|
|
envelope = raw.get("result") or {}
|
|||
|
|
code = envelope.get("decision_code")
|
|||
|
|
value = _DEV_VENDOR_CHOICES.get(code)
|
|||
|
|
if value is None:
|
|||
|
|
return _unknown("inconclusive")
|
|||
|
|
metrics = []
|
|||
|
|
for name, item in (raw.get("metrics") or {}).items():
|
|||
|
|
# 厂商原生指标保留语义与来源；不跨提供方套阈值，也不补齐缺失概率（AC-18）
|
|||
|
|
metrics.append({"name": str(name), "value": float(item),
|
|||
|
|
"semantics": "厂商原生指标，定义与兴邦校准口径不一致",
|
|||
|
|
"source": "provider_native"})
|
|||
|
|
return {"kind": question["type"], "value": value, "native_metrics": metrics,
|
|||
|
|
"actual_checkpoint_ref": raw.get("checkpoint")}
|
|||
|
|
|
|||
|
|
|
|||
|
|
MAPPERS = {"local": map_local_answer, "jev": map_jev_answer}
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------------ 适配入口
|
|||
|
|
def build_provider_call(config: JudgmentConfig, profile: JudgmentProfile, task: str,
|
|||
|
|
usage: str, *, fixture=None, transport=None, call_log=None):
|
|||
|
|
"""构造 `judge()` 用的 provider_call：先过准入与冻结，再决定是否真的发送。
|
|||
|
|
|
|||
|
|
`fixture(question) -> raw` 是离线路径（不联网）；`transport(target, payload)`
|
|||
|
|
是 live 路径。两者不能互相冒充：call_log 会记录 mode，测试据此断言
|
|||
|
|
"local 运行期间没有任何 jev 请求"（AC-17）。
|
|||
|
|
"""
|
|||
|
|
assert_task_allowed(profile, task, usage)
|
|||
|
|
assert_checkpoint_frozen(config, profile)
|
|||
|
|
mapper = MAPPERS[profile.provider]
|
|||
|
|
|
|||
|
|
def call(config: JudgmentConfig, question: dict) -> dict:
|
|||
|
|
# 预算在发送之前判定：超限即结束，调用记录里不会出现任何发送
|
|||
|
|
assert_within_budget(question, profile)
|
|||
|
|
payload = json.dumps({
|
|||
|
|
"question_id": question["id"], "version": question["version"],
|
|||
|
|
"type": question["type"], "prompt": question.get("prompt", ""),
|
|||
|
|
"options": list(question.get("options") or ()),
|
|||
|
|
"evidence_ids": list(question.get("evidence_ids") or ()),
|
|||
|
|
"checkpoint_ref": config.checkpoint_ref,
|
|||
|
|
"prompt_version": config.prompt_version,
|
|||
|
|
"mapping_version": config.mapping_version,
|
|||
|
|
}, ensure_ascii=False)
|
|||
|
|
|
|||
|
|
if fixture is not None:
|
|||
|
|
raw = fixture(question)
|
|||
|
|
mode = "fixture"
|
|||
|
|
else:
|
|||
|
|
if config.provider == "jev" and not profile.egress_approved:
|
|||
|
|
raise EgressNotApproved("该外部出口未获批准，已拒绝发送")
|
|||
|
|
if transport is None:
|
|||
|
|
raise LiveTransportUnavailable(
|
|||
|
|
"本批未接入 live 传输层（缺硬件与授权），只能走离线 fixture")
|
|||
|
|
raw = transport(profile.provider, payload)
|
|||
|
|
mode = "live"
|
|||
|
|
|
|||
|
|
if call_log is not None:
|
|||
|
|
call_log.append({"target": config.provider, "mode": mode,
|
|||
|
|
"question_id": question["id"], "payload": payload})
|
|||
|
|
|
|||
|
|
adapted = mapper(raw, question, profile)
|
|||
|
|
actual = adapted.get("actual_checkpoint_ref")
|
|||
|
|
if actual and actual != config.checkpoint_ref:
|
|||
|
|
raise CheckpointMismatch(
|
|||
|
|
f"提供方实际权重 {actual} 与运行冻结值 {config.checkpoint_ref} 不一致")
|
|||
|
|
return adapted
|
|||
|
|
|
|||
|
|
return call
|