289 lines
15 KiB
Python
289 lines
15 KiB
Python
 
|
"""纠偏第四批 · 改进发布接到真实知识发布入口（BR-51 / AC-57）。
|
|||
|
|
|
|||
|
|
审计（audit.json）确认的偏差模式之一是「只改 `released` 状态或返回一个版本号」就当发布成功。
|
|||
|
|
本测试走**真实执行链**证明发布真的生效：
|
|||
|
|
|
|||
|
|
卡片草稿 → 隔离评测 → 独立审核发布 v1.0.0 → 真实运行（冻结 v1.0.0）
|
|||
|
|
→ 提出改进候选（指向 v1.1.0 草稿）→ 旧/新对照 → 提交 → 他人审核
|
|||
|
|
→ release（**不提交任何版本号**）→ 真实发布 v1.1.0，覆盖 release 指针
|
|||
|
|
→ 再跑一次：新运行用 v1.1.0；回放第一次运行仍是 v1.0.0
|
|||
|
|
|
|||
|
|
反例同样锁死：客户端自报 `released_ref` 被 422 拒绝；候选指向未经评测的草稿时
|
|||
|
|
发布被 409 拒绝且**不产生**新版本、不改指针；非知识类目标没有真实入口，诚实拒绝而非伪造。
|
|||
|
|
|
|||
|
|
不用注入记录；临时库；不触碰真实数据根。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import tempfile
|
|||
|
|
import unittest
|
|||
|
|
from pathlib import Path
|
|||
|
|
from unittest.mock import patch
|
|||
|
|
|
|||
|
|
from fastapi import FastAPI
|
|||
|
|
from fastapi.testclient import TestClient
|
|||
|
|
|
|||
|
|
from api import prescreen as prescreen_api
|
|||
|
|
from api import project_assistant as api
|
|||
|
|
from api import risk_agent_config as catalog
|
|||
|
|
from services import assistant_store as store
|
|||
|
|
from services import improvements, knowledge_cards, trajectory
|
|||
|
|
from services.assistant_worker import DurableWorker
|
|||
|
|
from services.project_assistant import ProjectAssistantService
|
|||
|
|
|
|||
|
|
PROJECT = "DEMO-B"
|
|||
|
|
BASE = "/api/xingbang/risk-agent/project-assistant"
|
|||
|
|
PRESCREEN = BASE + "/prescreen"
|
|||
|
|
MANAGER = {"X-Xingbang-Demo-Role": "manager"}
|
|||
|
|
REVIEWER = {"X-Xingbang-Demo-Role": "reviewer"}
|
|||
|
|
|
|||
|
|
CARD_ID = "skill.history.repayment"
|
|||
|
|
V1 = "1.0.0"
|
|||
|
|
V2 = "1.1.0"
|
|||
|
|
CARD_CODE = "history.repayment.dependency"
|
|||
|
|
|
|||
|
|
# 自定义 Skill 的发布门禁要求独立测试样本（component_evaluation 的既有规则）
|
|||
|
|
TEST_CASES = [
|
|||
|
|
{"name": "缺证样本", "values": {"financial.operating_cashflow": None},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {CARD_CODE: "unknown"}},
|
|||
|
|
{"name": "已提供样本", "values": {"financial.operating_cashflow": "1200"},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {CARD_CODE: "pass"}},
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
BASELINE = {"id": "assistant.current", "version": "1.0.0"}
|
|||
|
|
CANDIDATE = {"id": "prescreen.v2", "version": "0.2.0"}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def sample(sample_id, *, baseline_pass=False, candidate_pass=True,
|
|||
|
|
predicted="2026-09-29T01:00:00+00:00", revealed="2026-09-29T03:00:00+00:00"):
|
|||
|
|
return {"id": sample_id, "split": "dev", "observed": True,
|
|||
|
|
"baseline_pass": baseline_pass, "candidate_pass": candidate_pass,
|
|||
|
|
"predicted_at": predicted, "label_revealed_at": revealed,
|
|||
|
|
"source_in_dev": False}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def improvement_evaluation():
|
|||
|
|
"""一次独立、无泄漏、结论为 improved 的旧/新对照。"""
|
|||
|
|
return {"baseline_ref": BASELINE, "candidate_ref": CANDIDATE,
|
|||
|
|
"dataset_ref": "fixtures/cases/manifest.json", "split_ref": "split.v1",
|
|||
|
|
"independent": True,
|
|||
|
|
"samples": [sample("S-1", baseline_pass=False),
|
|||
|
|
sample("S-2", baseline_pass=True)]}
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ImprovementReleaseTests(unittest.TestCase):
|
|||
|
|
def setUp(self):
|
|||
|
|
self.temp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
|
|||
|
|
self.path = Path(self.temp.name)
|
|||
|
|
self.service = ProjectAssistantService(self.path / "assistant.sqlite3")
|
|||
|
|
self.patches = [
|
|||
|
|
patch.object(api, "service", lambda: self.service),
|
|||
|
|
patch.object(prescreen_api, "_service", lambda: self.service),
|
|||
|
|
patch.object(catalog, "DB_PATH", self.path / "catalog.sqlite3"),
|
|||
|
|
patch.dict(os.environ, {"XINGBANG_AUTH_USERS": "", "XINGBANG_REQUIRE_AUTH": "0",
|
|||
|
|
"XINGBANG_LOCAL_MODEL_URL": "", "XINGBANG_DEMO_ROLE_LOGIN": "1"}),
|
|||
|
|
]
|
|||
|
|
for item in self.patches:
|
|||
|
|
item.start()
|
|||
|
|
self.worker = DurableWorker(self.service)
|
|||
|
|
app = FastAPI()
|
|||
|
|
app.include_router(api.router)
|
|||
|
|
app.include_router(prescreen_api.router)
|
|||
|
|
self.client = TestClient(app)
|
|||
|
|
|
|||
|
|
def tearDown(self):
|
|||
|
|
self.worker.stop()
|
|||
|
|
self.client.close()
|
|||
|
|
for item in reversed(self.patches):
|
|||
|
|
item.stop()
|
|||
|
|
self.temp.cleanup()
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------ 工具
|
|||
|
|
def call(self, method, path, body=None, headers=MANAGER, root=PRESCREEN):
|
|||
|
|
return self.client.request(method, root + path, json=body, headers=headers)
|
|||
|
|
|
|||
|
|
def draft_card(self, version):
|
|||
|
|
"""人工新增一个该组件的新版本草稿（不走模型，故不受模型未配置影响）。"""
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
return knowledge_cards.propose(db, {
|
|||
|
|
"id": CARD_ID, "version": version, "kind": "skill",
|
|||
|
|
"name": "核查回款集中依赖", "engine": "required",
|
|||
|
|
"inputs": ["financial.operating_cashflow"],
|
|||
|
|
"code": CARD_CODE, "title": "核查回款集中依赖",
|
|||
|
|
"test_cases": TEST_CASES,
|
|||
|
|
}, "demo-manager", PROJECT, origin="history_derived", source_event_ids=["event-1"])
|
|||
|
|
|
|||
|
|
def publish_v1_and_run_once(self):
|
|||
|
|
"""发布 v1.0.0、装配方案并真实跑一次，返回（已发布方案 id, 第一次运行）。"""
|
|||
|
|
self.draft_card(V1)
|
|||
|
|
published = self.call("POST", f"/knowledge-candidates/{CARD_ID}/transitions",
|
|||
|
|
{"action": "evaluate"}, REVIEWER)
|
|||
|
|
self.assertEqual(published.status_code, 200, published.text)
|
|||
|
|
published = self.call("POST", f"/knowledge-candidates/{CARD_ID}/transitions",
|
|||
|
|
{"action": "publish"}, headers=REVIEWER)
|
|||
|
|
self.assertEqual(published.status_code, 200, published.text)
|
|||
|
|
self.assertEqual(published.json()["version"], V1)
|
|||
|
|
|
|||
|
|
plan = self.call("POST", f"/projects/{PROJECT}/review-plans", {
|
|||
|
|
"name": "含历史经验卡的预审方案",
|
|||
|
|
"skills": ["corporate.consistency", "corporate.cashflow", "corporate.assets",
|
|||
|
|
"corporate.coverage", CARD_ID],
|
|||
|
|
"rules": ["demo.coverage"], "template": "template.ai-pre-review",
|
|||
|
|
"budget": {"timeout_seconds": 180, "max_tool_calls": 18},
|
|||
|
|
"audience": ["manager"], "run_kind": "pre_review"})
|
|||
|
|
self.assertEqual(plan.status_code, 201, plan.text)
|
|||
|
|
plan_id = plan.json()["component_id"]
|
|||
|
|
self.assertEqual(self.call("POST", f"/review-plans/{plan_id}/transitions",
|
|||
|
|
{"action": "evaluate"}, REVIEWER).status_code, 200)
|
|||
|
|
self.assertEqual(self.call("POST", f"/review-plans/{plan_id}/transitions",
|
|||
|
|
{"action": "publish"}, REVIEWER).status_code, 200)
|
|||
|
|
return plan_id, self.run_with(plan_id)
|
|||
|
|
|
|||
|
|
def run_with(self, plan_id):
|
|||
|
|
queued = self.call("POST", f"/projects/{PROJECT}/runs",
|
|||
|
|
{"requirement": "核对回款与现金流", "plan_id": plan_id}, root=BASE)
|
|||
|
|
self.assertIn(queued.status_code, (200, 202), queued.text)
|
|||
|
|
run_id = queued.json()["id"]
|
|||
|
|
self.assertTrue(self.worker.process_next(), "队列里应当有一次待执行运行")
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
return store.get(db, "run", run_id)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def checks_of(run, code):
|
|||
|
|
return [check for step in run["steps"] for check in step.get("findings", [])
|
|||
|
|
if check.get("code") == code]
|
|||
|
|
|
|||
|
|
def released_version(self):
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
return store.get(db, "release", CARD_ID)["version_id"]
|
|||
|
|
|
|||
|
|
def propose_improvement(self, *, target="knowledge", revision_ref=None, actor=MANAGER,
|
|||
|
|
source_run_id=None):
|
|||
|
|
body = {"target": target,
|
|||
|
|
"target_ref": {"id": CARD_ID, "version": V1},
|
|||
|
|
"proposed_revision_ref": revision_ref or {"id": CARD_ID, "version": V2},
|
|||
|
|
"change_summary": "把回款集中依赖的核查口径扩到制造业设备直租",
|
|||
|
|
"source_run_ids": [source_run_id] if source_run_id else ["run-1"]}
|
|||
|
|
created = self.call("POST", f"/projects/{PROJECT}/improvements", body, actor)
|
|||
|
|
self.assertEqual(created.status_code, 201, created.text)
|
|||
|
|
return created.json()
|
|||
|
|
|
|||
|
|
def approve(self, candidate, actor=REVIEWER):
|
|||
|
|
"""对照评测 → 提交 → 由**另一位**身份通过审核。"""
|
|||
|
|
evaluated = self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "evaluate", "evaluation": improvement_evaluation()})
|
|||
|
|
self.assertEqual(evaluated.status_code, 200, evaluated.text)
|
|||
|
|
self.assertEqual(evaluated.json()["status"], "ready", evaluated.text)
|
|||
|
|
self.assertEqual(self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "submit"}).status_code, 200)
|
|||
|
|
approved = self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "approve"}, actor)
|
|||
|
|
self.assertEqual(approved.status_code, 200, approved.text)
|
|||
|
|
self.assertEqual(approved.json()["status"], "approved")
|
|||
|
|
return approved.json()
|
|||
|
|
|
|||
|
|
# ------------------------------------------------------------ AC-57
|
|||
|
|
def test_release_publishes_the_real_version_and_old_run_still_replays(self):
|
|||
|
|
plan_id, run_old = self.publish_v1_and_run_once()
|
|||
|
|
old_refs = {(item["id"], item["version"]) for item in run_old["component_versions"]}
|
|||
|
|
self.assertIn((CARD_ID, V1), old_refs, "第一次运行应冻结 v1.0.0")
|
|||
|
|
self.assertEqual(self.released_version(), f"{CARD_ID}@{V1}")
|
|||
|
|
|
|||
|
|
# 1) 新版本草稿 + 隔离评测（发布门禁，与候选的旧/新对照是两道不同的门）
|
|||
|
|
self.draft_card(V2)
|
|||
|
|
evaluated = self.call("POST", f"/knowledge-candidates/{CARD_ID}/transitions",
|
|||
|
|
{"action": "evaluate"}, REVIEWER)
|
|||
|
|
self.assertEqual(evaluated.status_code, 200, evaluated.text)
|
|||
|
|
|
|||
|
|
# 2) 改进候选：对照 → 提交 → 他人审核 → 发布（**请求体不含任何版本号**）
|
|||
|
|
candidate = self.propose_improvement(source_run_id=run_old["id"])
|
|||
|
|
self.approve(candidate)
|
|||
|
|
released = self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "release"}, REVIEWER)
|
|||
|
|
self.assertEqual(released.status_code, 200, released.text)
|
|||
|
|
body = released.json()
|
|||
|
|
self.assertEqual(body["status"], "released")
|
|||
|
|
# 版本引用由服务端从真实发布结果产生
|
|||
|
|
self.assertEqual(body["released_ref"], {"id": CARD_ID, "version": V2})
|
|||
|
|
|
|||
|
|
# 3) 发布真的生效：release 指针被覆盖，不可变版本已落库
|
|||
|
|
self.assertEqual(self.released_version(), f"{CARD_ID}@{V2}")
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
versions = {row["id"] for row in store.listing(db, "component_version")}
|
|||
|
|
self.assertIn(f"{CARD_ID}@{V2}", versions, "必须真的写入不可变组件版本")
|
|||
|
|
|
|||
|
|
# 4) 新运行使用新版本（同一个已发布方案，无需重装）
|
|||
|
|
run_new = self.run_with(plan_id)
|
|||
|
|
new_refs = {(item["id"], item["version"]) for item in run_new["component_versions"]}
|
|||
|
|
self.assertIn((CARD_ID, V2), new_refs, "新运行必须取到新版本")
|
|||
|
|
new_checks = self.checks_of(run_new, CARD_CODE)
|
|||
|
|
self.assertEqual(len(new_checks), 1, "新版本必须真的产生核查项")
|
|||
|
|
self.assertEqual(new_checks[0]["basis_ids"], [f"{CARD_ID}@{V2}"])
|
|||
|
|
|
|||
|
|
# 5) 旧运行仍可回放，且冻结的版本没有被新发布改写
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
replayed = trajectory.replay(db, PROJECT, run_old["id"])
|
|||
|
|
reread = store.get(db, "run", run_old["id"])
|
|||
|
|
step = next(item for item in replayed["steps"]
|
|||
|
|
if (item["component_ref"] or {}).get("id") == CARD_ID)
|
|||
|
|
self.assertEqual(step["component_ref"]["version"], V1,
|
|||
|
|
"旧运行回放必须仍是发布前的版本")
|
|||
|
|
self.assertEqual({(item["id"], item["version"]) for item in reread["component_versions"]},
|
|||
|
|
old_refs, "旧运行的冻结版本不得被新发布改写")
|
|||
|
|
self.assertTrue(replayed["read_only"])
|
|||
|
|
|
|||
|
|
def test_client_supplied_version_is_rejected_and_changes_nothing(self):
|
|||
|
|
"""自报 `released_ref` 不得让候选变成已发布，也不得凭空造出组件版本。"""
|
|||
|
|
candidate = self.propose_improvement()
|
|||
|
|
self.approve(candidate)
|
|||
|
|
|
|||
|
|
forged = self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "release",
|
|||
|
|
"released_ref": {"id": CARD_ID, "version": "9.9.9"}}, REVIEWER)
|
|||
|
|
self.assertEqual(forged.status_code, 422, forged.text)
|
|||
|
|
self.assertEqual(forged.json()["error"]["code"], "INVALID_REQUEST")
|
|||
|
|
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
stored = store.get(db, improvements.KIND, candidate["id"])
|
|||
|
|
versions = {row["id"] for row in store.listing(db, "component_version")}
|
|||
|
|
self.assertEqual(stored["status"], "approved", "拒绝后候选状态不得前进")
|
|||
|
|
self.assertIsNone(stored["released_ref"])
|
|||
|
|
self.assertNotIn(f"{CARD_ID}@9.9.9", versions)
|
|||
|
|
|
|||
|
|
def test_release_keeps_the_previous_version_when_the_draft_was_not_evaluated(self):
|
|||
|
|
"""未过隔离评测的草稿不得被发布；失败时保留旧版（发布不是"改状态"）。"""
|
|||
|
|
_, run_old = self.publish_v1_and_run_once()
|
|||
|
|
self.assertEqual(self.released_version(), f"{CARD_ID}@{V1}")
|
|||
|
|
self.draft_card(V2) # 只建草稿，**不做**隔离评测
|
|||
|
|
|
|||
|
|
candidate = self.propose_improvement(source_run_id=run_old["id"])
|
|||
|
|
self.approve(candidate)
|
|||
|
|
refused = self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "release"}, REVIEWER)
|
|||
|
|
self.assertEqual(refused.status_code, 409, refused.text)
|
|||
|
|
self.assertEqual(refused.json()["error"]["code"], "IMPROVEMENT_NOT_VALIDATED")
|
|||
|
|
|
|||
|
|
self.assertEqual(self.released_version(), f"{CARD_ID}@{V1}", "发布失败必须保留旧版")
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
versions = {row["id"] for row in store.listing(db, "component_version")}
|
|||
|
|
stored = store.get(db, improvements.KIND, candidate["id"])
|
|||
|
|
self.assertNotIn(f"{CARD_ID}@{V2}", versions, "未评测的版本不得落库")
|
|||
|
|
self.assertEqual(stored["status"], "approved", "失败时候选状态不得前进")
|
|||
|
|
|
|||
|
|
def test_unsupported_target_is_refused_instead_of_faked(self):
|
|||
|
|
"""题库/编排尚无真实发布入口：诚实拒绝，不用改状态冒充发布。"""
|
|||
|
|
candidate = self.propose_improvement(target="question_profile")
|
|||
|
|
self.approve(candidate)
|
|||
|
|
refused = self.call("POST", f"/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "release"}, REVIEWER)
|
|||
|
|
self.assertEqual(refused.status_code, 409, refused.text)
|
|||
|
|
self.assertIn("尚无真实发布入口", refused.json()["error"]["message"])
|
|||
|
|
with self.service.connect() as db:
|
|||
|
|
stored = store.get(db, improvements.KIND, candidate["id"])
|
|||
|
|
self.assertNotEqual(stored["status"], "released")
|
|||
|
|
self.assertIsNone(stored["released_ref"])
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
unittest.main()
|