chore: mini-openclaw 初始版本
Next.js 前端 + FastAPI/LangChain 后端透明 agent平台 - 含 .gitignore, 敏感文件(sessions数据/.env/venv等)已排除 - 待后续添加 trajectory 自进化机制
This commit is contained in:
NathanAlex7
@@ -0,0 +1,188 @@
|
||||
"""POST /api/chat — SSE streaming chat with Agent."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import traceback
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from graph.agent import agent_manager
|
||||
from graph.session_manager import session_manager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
message: str
|
||||
session_id: str = "default"
|
||||
stream: bool = True
|
||||
|
||||
|
||||
async def _generate_title(session_id: str) -> str | None:
|
||||
"""Generate a title for a session using DeepSeek. Returns title or None."""
|
||||
try:
|
||||
messages = session_manager.load_session(session_id)
|
||||
first_user = ""
|
||||
first_assistant = ""
|
||||
for msg in messages:
|
||||
if msg["role"] == "user" and not first_user:
|
||||
first_user = msg["content"][:200]
|
||||
elif msg["role"] == "assistant" and not first_assistant:
|
||||
first_assistant = msg["content"][:200]
|
||||
if first_user and first_assistant:
|
||||
break
|
||||
|
||||
if not first_user:
|
||||
return None
|
||||
|
||||
from langchain_deepseek import ChatDeepSeek
|
||||
from langchain_core.messages import HumanMessage as HM
|
||||
|
||||
llm = ChatDeepSeek(
|
||||
model=os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
|
||||
api_key=os.getenv("DEEPSEEK_API_KEY"),
|
||||
api_base=os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com"),
|
||||
temperature=0.3,
|
||||
)
|
||||
|
||||
prompt = (
|
||||
f"根据以下对话内容，生成一个不超过10个字的中文标题，只输出标题文本，不要加引号或标点。\n\n"
|
||||
f"用户: {first_user}\n"
|
||||
f"助手: {first_assistant}"
|
||||
)
|
||||
|
||||
result = await llm.ainvoke([HM(content=prompt)])
|
||||
title = result.content.strip().strip('"\'""''')[:20]
|
||||
session_manager.update_title(session_id, title)
|
||||
return title
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
|
||||
async def event_generator(message: str, session_id: str) -> AsyncGenerator[dict, None]:
|
||||
"""Generate SSE events from agent stream.
|
||||
|
||||
Tracks multiple response segments — each time the agent finishes
|
||||
tool calls and starts generating new text, a new_response event is
|
||||
emitted and a new segment begins. Each segment is saved as a
|
||||
separate assistant message in the session history.
|
||||
"""
|
||||
try:
|
||||
# Use merged history for agent context (combines consecutive assistant msgs)
|
||||
history = session_manager.load_session_for_agent(session_id)
|
||||
is_first_message = len(history) == 0
|
||||
|
||||
# Per-segment tracking
|
||||
segments: list[dict] = [] # finalized segments
|
||||
current_segment: dict = {"content": "", "tool_calls": []}
|
||||
|
||||
async for event in agent_manager.astream(message, history):
|
||||
event_type = event.get("type", "unknown")
|
||||
|
||||
if event_type == "retrieval":
|
||||
yield {
|
||||
"event": "retrieval",
|
||||
"data": json.dumps(
|
||||
{"query": event["query"], "results": event["results"]},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
|
||||
elif event_type == "token":
|
||||
current_segment["content"] += event["content"]
|
||||
yield {
|
||||
"event": "token",
|
||||
"data": json.dumps({"content": event["content"]}, ensure_ascii=False),
|
||||
}
|
||||
|
||||
elif event_type == "new_response":
|
||||
# Finalize current segment and start a new one
|
||||
segments.append(current_segment)
|
||||
current_segment = {"content": "", "tool_calls": []}
|
||||
yield {
|
||||
"event": "new_response",
|
||||
"data": json.dumps({}, ensure_ascii=False),
|
||||
}
|
||||
|
||||
elif event_type == "tool_start":
|
||||
current_segment["tool_calls"].append({
|
||||
"tool": event["tool"],
|
||||
"input": event.get("input", ""),
|
||||
})
|
||||
yield {
|
||||
"event": "tool_start",
|
||||
"data": json.dumps(
|
||||
{"tool": event["tool"], "input": event["input"]},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
|
||||
elif event_type == "tool_end":
|
||||
# Match with current segment's tool_calls to add output
|
||||
for tc in reversed(current_segment["tool_calls"]):
|
||||
if tc["tool"] == event["tool"] and "output" not in tc:
|
||||
tc["output"] = event["output"]
|
||||
break
|
||||
yield {
|
||||
"event": "tool_end",
|
||||
"data": json.dumps(
|
||||
{"tool": event["tool"], "output": event["output"]},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
|
||||
elif event_type == "done":
|
||||
# Finalize the last segment
|
||||
segments.append(current_segment)
|
||||
|
||||
# Save user message once
|
||||
session_manager.save_message(session_id, "user", message)
|
||||
|
||||
# Save each segment as a separate assistant message
|
||||
for seg in segments:
|
||||
tc = seg["tool_calls"] if seg["tool_calls"] else None
|
||||
session_manager.save_message(
|
||||
session_id, "assistant", seg["content"], tool_calls=tc
|
||||
)
|
||||
|
||||
yield {
|
||||
"event": "done",
|
||||
"data": json.dumps(
|
||||
{"content": event["content"], "session_id": session_id},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
|
||||
# Auto-generate title on first conversation
|
||||
if is_first_message:
|
||||
title = await _generate_title(session_id)
|
||||
if title:
|
||||
yield {
|
||||
"event": "title",
|
||||
"data": json.dumps(
|
||||
{"session_id": session_id, "title": title},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
yield {
|
||||
"event": "error",
|
||||
"data": json.dumps({"error": str(e)}, ensure_ascii=False),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/chat")
|
||||
async def chat(request: ChatRequest):
|
||||
if request.stream:
|
||||
return EventSourceResponse(
|
||||
event_generator(request.message, request.session_id)
|
||||
)
|
||||
# Non-streaming fallback
|
||||
result = await agent_manager.ainvoke(request.message, request.session_id)
|
||||
return {"reply": result}
|
||||
@@ -0,0 +1,71 @@
|
||||
"""POST /api/sessions/{session_id}/compress — Compress conversation history."""
|
||||
|
||||
import os
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
from graph.session_manager import session_manager
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
async def _generate_summary(messages: list[dict[str, Any]]) -> str:
|
||||
"""Use DeepSeek to generate a compressed summary of messages."""
|
||||
from langchain_deepseek import ChatDeepSeek
|
||||
|
||||
llm = ChatDeepSeek(
|
||||
model=os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
|
||||
api_key=os.getenv("DEEPSEEK_API_KEY"),
|
||||
api_base=os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com"),
|
||||
temperature=0.3,
|
||||
)
|
||||
|
||||
# Format messages for summary
|
||||
formatted = []
|
||||
for msg in messages:
|
||||
role = msg.get("role", "unknown")
|
||||
content = msg.get("content", "")
|
||||
if content:
|
||||
formatted.append(f"{role}: {content[:500]}")
|
||||
|
||||
conversation_text = "\n".join(formatted)
|
||||
|
||||
prompt = (
|
||||
"请将以下对话历史压缩为简洁的中文摘要，保留关键信息、决策和结论。"
|
||||
"摘要不超过500字。只输出摘要内容，不要添加额外说明。\n\n"
|
||||
f"{conversation_text}"
|
||||
)
|
||||
|
||||
result = await llm.ainvoke([HumanMessage(content=prompt)])
|
||||
return result.content.strip()
|
||||
|
||||
|
||||
@router.post("/sessions/{session_id}/compress")
|
||||
async def compress_session(session_id: str) -> dict[str, Any]:
|
||||
"""Compress the first 50% of conversation history into a summary."""
|
||||
messages = session_manager.load_session(session_id)
|
||||
if len(messages) < 4:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Not enough messages to compress (need at least 4)",
|
||||
)
|
||||
|
||||
# Take the first 50% of messages
|
||||
num_to_remove = max(4, len(messages) // 2)
|
||||
|
||||
messages_to_compress = messages[:num_to_remove]
|
||||
|
||||
try:
|
||||
summary = await _generate_summary(messages_to_compress)
|
||||
session_manager.compress_history(session_id, summary, num_to_remove)
|
||||
remaining = len(messages) - num_to_remove
|
||||
return {
|
||||
"archived_count": num_to_remove,
|
||||
"remaining_count": remaining,
|
||||
}
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
raise HTTPException(status_code=500, detail=f"Compression failed: {str(e)}")
|
||||
@@ -0,0 +1,23 @@
|
||||
"""GET/PUT /api/config/rag-mode — RAG mode toggle API."""
|
||||
|
||||
from fastapi import APIRouter
|
||||
from pydantic import BaseModel
|
||||
|
||||
from config import get_rag_mode, set_rag_mode
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class RagModeRequest(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
@router.get("/config/rag-mode")
|
||||
async def get_rag_mode_endpoint():
|
||||
return {"rag_mode": get_rag_mode()}
|
||||
|
||||
|
||||
@router.put("/config/rag-mode")
|
||||
async def set_rag_mode_endpoint(request: RagModeRequest):
|
||||
set_rag_mode(request.enabled)
|
||||
return {"rag_mode": request.enabled}
|
||||
@@ -0,0 +1,102 @@
|
||||
"""GET/POST /api/files + GET /api/skills — File read/write for Monaco editor."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
# Whitelist of editable directories (relative to backend/)
|
||||
ALLOWED_PREFIXES = ["workspace/", "memory/", "skills/", "knowledge/"]
|
||||
|
||||
# Whitelist of specific root-level files that can be accessed
|
||||
ALLOWED_ROOT_FILES = {"SKILLS_SNAPSHOT.md"}
|
||||
|
||||
|
||||
def _validate_path(rel_path: str) -> Path:
|
||||
"""Validate and resolve file path within allowed directories."""
|
||||
normalized = rel_path.replace("\\", "/").lstrip("./")
|
||||
if not (
|
||||
any(normalized.startswith(prefix) for prefix in ALLOWED_PREFIXES)
|
||||
or normalized in ALLOWED_ROOT_FILES
|
||||
):
|
||||
raise HTTPException(status_code=403, detail=f"Access denied: {rel_path}")
|
||||
full_path = (BASE_DIR / normalized).resolve()
|
||||
if not str(full_path).startswith(str(BASE_DIR)):
|
||||
raise HTTPException(status_code=403, detail="Path traversal detected")
|
||||
return full_path
|
||||
|
||||
|
||||
@router.get("/files")
|
||||
async def read_file(path: str):
|
||||
file_path = _validate_path(path)
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"File not found: {path}")
|
||||
content = file_path.read_text(encoding="utf-8")
|
||||
return {"path": path, "content": content}
|
||||
|
||||
|
||||
class FileSaveRequest(BaseModel):
|
||||
path: str
|
||||
content: str
|
||||
|
||||
|
||||
@router.post("/files")
|
||||
async def save_file(request: FileSaveRequest):
|
||||
file_path = _validate_path(request.path)
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_text(request.content, encoding="utf-8")
|
||||
|
||||
# Trigger memory index rebuild when MEMORY.md is saved
|
||||
normalized = request.path.replace("\\", "/").lstrip("./")
|
||||
if normalized == "memory/MEMORY.md":
|
||||
try:
|
||||
from graph.memory_indexer import get_memory_indexer
|
||||
|
||||
indexer = get_memory_indexer(BASE_DIR)
|
||||
indexer.rebuild_index()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"path": request.path, "status": "saved"}
|
||||
|
||||
|
||||
@router.get("/skills")
|
||||
async def list_skills():
|
||||
"""Scan skills/ directory and return skill list with name, path, description."""
|
||||
skills_dir = BASE_DIR / "skills"
|
||||
if not skills_dir.exists():
|
||||
return {"skills": []}
|
||||
|
||||
skills: list[dict[str, str]] = []
|
||||
for skill_dir in sorted(skills_dir.iterdir()):
|
||||
if not skill_dir.is_dir():
|
||||
continue
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
if not skill_md.exists():
|
||||
continue
|
||||
|
||||
name = skill_dir.name
|
||||
description = ""
|
||||
rel_path = f"skills/{name}/SKILL.md"
|
||||
|
||||
# Parse YAML frontmatter
|
||||
try:
|
||||
text = skill_md.read_text(encoding="utf-8")
|
||||
if text.startswith("---"):
|
||||
parts = text.split("---", 2)
|
||||
if len(parts) >= 3:
|
||||
meta = yaml.safe_load(parts[1])
|
||||
if isinstance(meta, dict):
|
||||
name = meta.get("name", name)
|
||||
description = meta.get("description", "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
skills.append({"name": name, "path": rel_path, "description": description})
|
||||
|
||||
return {"skills": skills}
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Session CRUD API — list / create / rename / delete / raw messages / generate title."""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from graph.session_manager import session_manager
|
||||
from graph.prompt_builder import build_system_prompt
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
# ── Request models ──────────────────────────────────────────
|
||||
|
||||
class RenameRequest(BaseModel):
|
||||
title: str
|
||||
|
||||
|
||||
# ── Endpoints ───────────────────────────────────────────────
|
||||
|
||||
@router.get("/sessions")
|
||||
async def list_sessions():
|
||||
"""List all sessions with title and metadata."""
|
||||
sessions = session_manager.list_sessions()
|
||||
return {"sessions": sessions}
|
||||
|
||||
|
||||
@router.post("/sessions")
|
||||
async def create_session():
|
||||
"""Create a new empty session."""
|
||||
session_id = f"session-{uuid.uuid4().hex[:12]}"
|
||||
meta = session_manager.create_session(session_id)
|
||||
return meta
|
||||
|
||||
|
||||
@router.put("/sessions/{session_id}")
|
||||
async def rename_session(session_id: str, req: RenameRequest):
|
||||
"""Rename an existing session."""
|
||||
try:
|
||||
session_manager.rename_session(session_id, req.title)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
return {"id": session_id, "title": req.title}
|
||||
|
||||
|
||||
@router.delete("/sessions/{session_id}")
|
||||
async def delete_session(session_id: str):
|
||||
"""Delete a session."""
|
||||
session_manager.delete_session(session_id)
|
||||
return {"status": "deleted", "id": session_id}
|
||||
|
||||
|
||||
@router.get("/sessions/{session_id}/messages")
|
||||
async def get_raw_messages(session_id: str):
|
||||
"""Get complete raw messages including system prompt."""
|
||||
data = session_manager.get_raw_messages(session_id)
|
||||
system_prompt = build_system_prompt(BASE_DIR)
|
||||
# Prepend system prompt as the first message
|
||||
all_messages = [{"role": "system", "content": system_prompt}] + data.get("messages", [])
|
||||
return {"session_id": session_id, "title": data.get("title", ""), "messages": all_messages}
|
||||
|
||||
|
||||
@router.get("/sessions/{session_id}/history")
|
||||
async def get_session_history(session_id: str):
|
||||
"""Get conversation history for display (no system prompt, includes tool_calls)."""
|
||||
messages = session_manager.load_session(session_id)
|
||||
return {"session_id": session_id, "messages": messages}
|
||||
|
||||
|
||||
@router.post("/sessions/{session_id}/generate-title")
|
||||
async def generate_title(session_id: str):
|
||||
"""Use DeepSeek to generate a short title from the first conversation turn."""
|
||||
messages = session_manager.load_session(session_id)
|
||||
if not messages:
|
||||
raise HTTPException(status_code=400, detail="No messages to generate title from")
|
||||
|
||||
# Get the first user message and first assistant reply
|
||||
first_user = ""
|
||||
first_assistant = ""
|
||||
for msg in messages:
|
||||
if msg["role"] == "user" and not first_user:
|
||||
first_user = msg["content"][:200]
|
||||
elif msg["role"] == "assistant" and not first_assistant:
|
||||
first_assistant = msg["content"][:200]
|
||||
if first_user and first_assistant:
|
||||
break
|
||||
|
||||
if not first_user:
|
||||
raise HTTPException(status_code=400, detail="No user message found")
|
||||
|
||||
try:
|
||||
from langchain_deepseek import ChatDeepSeek
|
||||
from langchain_core.messages import HumanMessage as HM
|
||||
|
||||
llm = ChatDeepSeek(
|
||||
model=os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
|
||||
api_key=os.getenv("DEEPSEEK_API_KEY"),
|
||||
api_base=os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com"),
|
||||
temperature=0.3,
|
||||
)
|
||||
|
||||
prompt = (
|
||||
f"根据以下对话内容，生成一个不超过10个字的中文标题，只输出标题文本，不要加引号或标点。\n\n"
|
||||
f"用户: {first_user}\n"
|
||||
f"助手: {first_assistant}"
|
||||
)
|
||||
|
||||
result = await llm.ainvoke([HM(content=prompt)])
|
||||
title = result.content.strip().strip('"\'""''')[:20]
|
||||
|
||||
session_manager.update_title(session_id, title)
|
||||
return {"session_id": session_id, "title": title}
|
||||
|
||||
except Exception as e:
|
||||
# Fallback: use first few chars of user message
|
||||
fallback_title = first_user[:10].strip()
|
||||
session_manager.update_title(session_id, fallback_title)
|
||||
return {"session_id": session_id, "title": fallback_title}
|
||||
@@ -0,0 +1,68 @@
|
||||
"""GET/POST /api/tokens — Token counting for sessions and files."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import tiktoken
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from graph.session_manager import session_manager
|
||||
from graph.prompt_builder import build_system_prompt
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
# Cache the encoder instance
|
||||
_encoder = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
|
||||
def _count_tokens(text: str) -> int:
|
||||
"""Count tokens using cl100k_base encoding."""
|
||||
return len(_encoder.encode(text))
|
||||
|
||||
|
||||
@router.get("/tokens/session/{session_id}")
|
||||
async def get_session_token_count(session_id: str) -> dict[str, Any]:
|
||||
"""Count tokens in a session: system prompt + all messages."""
|
||||
system_prompt = build_system_prompt(BASE_DIR)
|
||||
system_tokens = _count_tokens(system_prompt)
|
||||
|
||||
messages = session_manager.load_session(session_id)
|
||||
message_tokens = 0
|
||||
for msg in messages:
|
||||
message_tokens += _count_tokens(msg.get("content", ""))
|
||||
|
||||
return {
|
||||
"system_tokens": system_tokens,
|
||||
"message_tokens": message_tokens,
|
||||
"total_tokens": system_tokens + message_tokens,
|
||||
}
|
||||
|
||||
|
||||
class FileTokenRequest(BaseModel):
|
||||
paths: list[str]
|
||||
|
||||
|
||||
@router.post("/tokens/files")
|
||||
async def get_file_token_counts(request: FileTokenRequest) -> dict[str, Any]:
|
||||
"""Count tokens for a list of files."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for rel_path in request.paths:
|
||||
normalized = rel_path.replace("\\", "/").lstrip("./")
|
||||
full_path = (BASE_DIR / normalized).resolve()
|
||||
if not str(full_path).startswith(str(BASE_DIR)):
|
||||
results.append({"path": rel_path, "tokens": 0})
|
||||
continue
|
||||
if not full_path.exists():
|
||||
results.append({"path": rel_path, "tokens": 0})
|
||||
continue
|
||||
try:
|
||||
content = full_path.read_text(encoding="utf-8")
|
||||
tokens = _count_tokens(content)
|
||||
results.append({"path": rel_path, "tokens": tokens})
|
||||
except Exception:
|
||||
results.append({"path": rel_path, "tokens": 0})
|
||||
|
||||
return {"files": results}
|
||||
Reference in New Issue
Block a user