200 lines
11 KiB
Python
200 lines
11 KiB
Python
 
|
"""纠偏第四批 · 真实服务器冒烟（不是单测的替代，是"走真实 HTTP 执行链"的补充证据）。
|
|||
|
|
|
|||
|
|
对**运行中的** uvicorn（127.0.0.1:8012）按界面实际会发的同一串请求走一遍：
|
|||
|
|
建卡片 → 隔离评测 → 发布 v1.0.0 → 结构化修订出新版本 → 隔离评测 → 录入会议结果
|
|||
|
|
→ 提改进候选 → 旧/新对照 → 提交 → 换身份（reviewer）审核 → 发布（**不带任何版本号**）
|
|||
|
|
→ 复查发布指针是否真的移到新版本。
|
|||
|
|
|
|||
|
|
只用一个本脚本新建的组件 id（服务端生成），不触碰既有组件的发布指针。
|
|||
|
|
|
|||
|
|
用法（先起后端，环境配置见 17 四批交付记录）：
|
|||
|
|
cd backend && XINGBANG_DEMO_ROLE_LOGIN=1 venv/Scripts/python.exe \
|
|||
|
|
-m uvicorn app:app --host 127.0.0.1 --port 8012
|
|||
|
|
backend/venv/Scripts/python.exe scripts/xingbang_batch4_smoke.py
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
import urllib.error
|
|||
|
|
import urllib.request
|
|||
|
|
|
|||
|
|
BASE = "http://127.0.0.1:8012/api/xingbang/risk-agent/project-assistant"
|
|||
|
|
PRESCREEN = BASE + "/prescreen"
|
|||
|
|
PROJECT = "DEMO-B"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def call(method, url, body=None, role="manager"):
|
|||
|
|
data = None if body is None else json.dumps(body).encode()
|
|||
|
|
request = urllib.request.Request(url, data=data, method=method)
|
|||
|
|
request.add_header("Content-Type", "application/json")
|
|||
|
|
request.add_header("X-Xingbang-Demo-Role", role)
|
|||
|
|
try:
|
|||
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|||
|
|
return response.status, json.loads(response.read() or b"{}")
|
|||
|
|
except urllib.error.HTTPError as exc:
|
|||
|
|
return exc.code, json.loads(exc.read() or b"{}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def show(label, status, body):
|
|||
|
|
print(f"[{status}] {label}: {json.dumps(body, ensure_ascii=False)[:220]}")
|
|||
|
|
return body
|
|||
|
|
|
|||
|
|
|
|||
|
|
steps = []
|
|||
|
|
|
|||
|
|
# 1) 新建一张卡片草稿（组件 id 由服务端生成，不碰既有组件）
|
|||
|
|
status, draft = call("POST", f"{PRESCREEN}/projects/{PROJECT}/knowledge-candidates", {
|
|||
|
|
"kind": "skill", "title": "冒烟：回款集中依赖核查",
|
|||
|
|
"content": {"engine": "required", "inputs": ["financial.operating_cashflow"],
|
|||
|
|
"code": "smoke.repayment.check",
|
|||
|
|
"test_cases": [
|
|||
|
|
{"name": "缺证样本", "values": {"financial.operating_cashflow": None},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {"smoke.repayment.check": "unknown"}},
|
|||
|
|
{"name": "已提供样本", "values": {"financial.operating_cashflow": "1200"},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {"smoke.repayment.check": "pass"}}]},
|
|||
|
|
}, "manager")
|
|||
|
|
show("新建草稿", status, draft)
|
|||
|
|
steps.append(("新建草稿", status))
|
|||
|
|
component_id = draft["id"].split("@")[0]
|
|||
|
|
|
|||
|
|
# 2) 隔离评测 + 独立发布 v1.0.0
|
|||
|
|
status, _ = call("POST", f"{PRESCREEN}/knowledge-candidates/{component_id}@1.0.0/transitions",
|
|||
|
|
{"action": "evaluate"}, "reviewer")
|
|||
|
|
steps.append(("草稿隔离评测 v1.0.0", status))
|
|||
|
|
status, published = call("POST", f"{PRESCREEN}/knowledge-candidates/{component_id}@1.0.0/transitions",
|
|||
|
|
{"action": "publish"}, "reviewer")
|
|||
|
|
show("发布 v1.0.0", status, published)
|
|||
|
|
steps.append(("发布 v1.0.0", status))
|
|||
|
|
|
|||
|
|
# 3) 结构化修订出新版本（不调模型），并做发布门禁要求的隔离评测
|
|||
|
|
status, revised = call("PATCH", f"{PRESCREEN}/knowledge-candidates/{component_id}", {
|
|||
|
|
"changes": {"test_cases": [
|
|||
|
|
{"name": "缺证样本", "values": {"financial.operating_cashflow": None},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {"smoke.repayment.check": "unknown"}},
|
|||
|
|
{"name": "已提供样本", "values": {"financial.operating_cashflow": "1200"},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {"smoke.repayment.check": "pass"}},
|
|||
|
|
{"name": "边界样本", "values": {"financial.operating_cashflow": "0"},
|
|||
|
|
"expected_outputs": {}, "expected_checks": {"smoke.repayment.check": "pass"}}]},
|
|||
|
|
"note": "冒烟：补充边界样本"}, "manager")
|
|||
|
|
show("修订出新版本", status, revised)
|
|||
|
|
steps.append(("修订出新版本", status))
|
|||
|
|
# 版本号由服务端升（_bump 升 patch 位），不在这里硬编码，避免"以为发布了另一个版本"
|
|||
|
|
new_draft_id = revised["id"]
|
|||
|
|
new_version = revised["version"]
|
|||
|
|
print(f" 修订后的草稿：{new_draft_id}")
|
|||
|
|
status, _ = call("POST", f"{PRESCREEN}/knowledge-candidates/{new_draft_id}/transitions",
|
|||
|
|
{"action": "evaluate"}, "reviewer")
|
|||
|
|
steps.append(("草稿隔离评测新版本", status))
|
|||
|
|
|
|||
|
|
# 4) AC-56：录入会议结果。先看有没有可对应的材料版本（运行输入快照）
|
|||
|
|
status, runs = call("GET", f"{PRESCREEN}/projects/{PROJECT}/runs")
|
|||
|
|
snapshots = [item["input_snapshot_id"] for item in runs.get("items", []) if item.get("input_snapshot_id")]
|
|||
|
|
outcome_body = {"raw_code": "附条件通过", "meeting_id": "SMOKE-4", "conditions": ["补充担保后生效"]}
|
|||
|
|
if snapshots:
|
|||
|
|
outcome_body["reviewed_snapshot_id"] = snapshots[0]
|
|||
|
|
status, outcome = call("POST", f"{PRESCREEN}/projects/{PROJECT}/review-outcomes", outcome_body, "manager")
|
|||
|
|
show("录入会议结果", status, outcome)
|
|||
|
|
steps.append(("录入会议结果", status))
|
|||
|
|
expected_status = "matched" if snapshots else "unmatched"
|
|||
|
|
|
|||
|
|
# 会议结果分组：未匹配必须单列且不计入分母
|
|||
|
|
status, outcome_list = call("GET", f"{PRESCREEN}/projects/{PROJECT}/review-outcomes")
|
|||
|
|
counts = outcome_list.get("counts_toward_denominator", {})
|
|||
|
|
print(f" 未匹配单列 {len(outcome_list.get('unmatched', []))} 条 · 计入分母 {counts.get('observed')} 条")
|
|||
|
|
steps.append(("会议结果分组", status))
|
|||
|
|
|
|||
|
|
# 5) 提改进候选（必须引用来源运行或会议结果）
|
|||
|
|
status, candidate = call("POST", f"{PRESCREEN}/projects/{PROJECT}/improvements", {
|
|||
|
|
"target": "knowledge",
|
|||
|
|
"target_ref": {"id": component_id, "version": "1.0.0"},
|
|||
|
|
"proposed_revision_ref": {"id": component_id, "version": new_version},
|
|||
|
|
"change_summary": "冒烟：把回款集中依赖的核查口径扩到含零值边界",
|
|||
|
|
"source_run_ids": [], "outcome_ids": [outcome["id"]]}, "manager")
|
|||
|
|
show("提出改进候选", status, candidate)
|
|||
|
|
steps.append(("提出改进候选", status))
|
|||
|
|
|
|||
|
|
status, evaluated = call("POST", f"{PRESCREEN}/improvements/{candidate['id']}/transitions", {
|
|||
|
|
"action": "evaluate",
|
|||
|
|
"evaluation": {"baseline_ref": {"id": "assistant.current", "version": "1.0.0"},
|
|||
|
|
"candidate_ref": {"id": "prescreen.v2", "version": "0.2.0"},
|
|||
|
|
"dataset_ref": "fixtures/cases/manifest.json", "split_ref": "split.v1",
|
|||
|
|
"independent": True,
|
|||
|
|
"samples": [
|
|||
|
|
{"id": "S-1", "split": "dev", "observed": True, "baseline_pass": False,
|
|||
|
|
"candidate_pass": True, "predicted_at": "2026-09-29T01:00:00+00:00",
|
|||
|
|
"label_revealed_at": "2026-09-29T03:00:00+00:00", "source_in_dev": False},
|
|||
|
|
{"id": "S-2", "split": "dev", "observed": True, "baseline_pass": True,
|
|||
|
|
"candidate_pass": True, "predicted_at": "2026-09-29T01:00:00+00:00",
|
|||
|
|
"label_revealed_at": "2026-09-29T03:00:00+00:00", "source_in_dev": False}]}},
|
|||
|
|
"manager")
|
|||
|
|
show("旧/新对照", status, evaluated)
|
|||
|
|
steps.append(("旧/新对照", status))
|
|||
|
|
|
|||
|
|
status, _ = call("POST", f"{PRESCREEN}/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "submit"}, "manager")
|
|||
|
|
steps.append(("提交审核", status))
|
|||
|
|
status, approved = call("POST", f"{PRESCREEN}/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "approve"}, "reviewer")
|
|||
|
|
show("独立审核通过", status, approved)
|
|||
|
|
steps.append(("独立审核通过", status))
|
|||
|
|
|
|||
|
|
# 6) 发布：请求体**只有 action**，没有任何版本号
|
|||
|
|
status, released = call("POST", f"{PRESCREEN}/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "release"}, "reviewer")
|
|||
|
|
show("发布（请求体不含 released_ref）", status, released)
|
|||
|
|
steps.append(("发布", status))
|
|||
|
|
released_ref = released.get("released_ref")
|
|||
|
|
|
|||
|
|
# 7) 反例：客户端自报版本号必须被拒
|
|||
|
|
status, forged = call("POST", f"{PRESCREEN}/improvements/{candidate['id']}/transitions",
|
|||
|
|
{"action": "release", "released_ref": {"id": component_id, "version": "9.9.9"}},
|
|||
|
|
"reviewer")
|
|||
|
|
show("反例：自报 released_ref", status, forged.get("error", forged))
|
|||
|
|
steps.append(("反例：自报版本号被拒", status))
|
|||
|
|
|
|||
|
|
# 8) 复查：发布指针是否真的移到了新版本
|
|||
|
|
status, listing = call("GET", f"{PRESCREEN}/projects/{PROJECT}/knowledge-candidates")
|
|||
|
|
mine = [card for card in listing.get("items", []) if card["id"] == component_id]
|
|||
|
|
show("发布后的卡片", status, mine)
|
|||
|
|
published_versions = sorted(card["version"] for card in mine if card["status"] == "published")
|
|||
|
|
|
|||
|
|
# 最硬的一条：发布**指针**是否真的指向新版本。列表会列出全部不可变历史版本
|
|||
|
|
# （旧版本保留是正确行为），只断言"列表里有新版本"证明不了发布生效。
|
|||
|
|
import sqlite3
|
|||
|
|
from pathlib import Path
|
|||
|
|
db = sqlite3.connect(
|
|||
|
|
Path(__file__).resolve().parents[1] / "backend" / "xingbang_data" / "project_assistant.sqlite3")
|
|||
|
|
pointer = json.loads(db.execute(
|
|||
|
|
"SELECT body FROM assistant_objects WHERE kind='release' AND id=?", (component_id,)).fetchone()[0])
|
|||
|
|
db.close()
|
|||
|
|
print(f" 发布指针：{pointer.get('version_id')}")
|
|||
|
|
|
|||
|
|
print("\n=== 冒烟结论 ===")
|
|||
|
|
for label, code in steps:
|
|||
|
|
print(f" {label}: HTTP {code}")
|
|||
|
|
print(f" 服务端返回的 released_ref: {released_ref}")
|
|||
|
|
print(f" 该组件的已发布版本: {published_versions}")
|
|||
|
|
print(f" 会议结果匹配状态: {outcome.get('status')}（预期 {expected_status}）")
|
|||
|
|
print(f" 自报版本号被拒: {forged.get('error', {}).get('code')}")
|
|||
|
|
# 反例那一步**预期**就是 4xx，不算失败
|
|||
|
|
positive = [(label, code) for label, code in steps if label != "反例：自报版本号被拒"]
|
|||
|
|
ok = (all(code < 300 for _, code in positive)
|
|||
|
|
and released_ref == {"id": component_id, "version": new_version}
|
|||
|
|
and new_version in published_versions # 不可变新版本已落库
|
|||
|
|
and pointer.get("version_id") == f"{component_id}@{new_version}" # 指针真的移了
|
|||
|
|
and forged.get("error", {}).get("code") == "INVALID_REQUEST"
|
|||
|
|
and outcome.get("status") == expected_status)
|
|||
|
|
checks = {
|
|||
|
|
"正向步骤全部 < 300": ([item for item in positive if item[1] >= 300], all(c < 300 for _, c in positive)),
|
|||
|
|
"released_ref 由服务端产生": (released_ref, released_ref == {"id": component_id, "version": new_version}),
|
|||
|
|
"新不可变版本已落库": (published_versions, new_version in published_versions),
|
|||
|
|
"发布指针已移动": (pointer.get("version_id"), pointer.get("version_id") == f"{component_id}@{new_version}"),
|
|||
|
|
"自报版本号被拒": (forged.get("error", {}).get("code"),
|
|||
|
|
forged.get("error", {}).get("code") == "INVALID_REQUEST"),
|
|||
|
|
"会议结果匹配状态": (outcome.get("status"), outcome.get("status") == expected_status),
|
|||
|
|
}
|
|||
|
|
for label, (value, passed) in checks.items():
|
|||
|
|
print(f" {'OK ' if passed else 'NG '} {label}: {value}")
|
|||
|
|
|
|||
|
|
print(f"\n结果：{'通过' if ok else '未通过'}")
|
|||
|
|
raise SystemExit(0 if ok else 1)
|