160 lines
8.5 KiB
TypeScript
160 lines
8.5 KiB
TypeScript
import {
|
|
FileAttachmentStore,
|
|
HarnessSession,
|
|
InMemoryApprovalBroker,
|
|
InMemoryEventStore,
|
|
InMemoryModelConnectionStore,
|
|
InMemorySessionRegistry,
|
|
SafeWorkspace,
|
|
SkillEventInference,
|
|
SkillIndex,
|
|
executeMockAdsMetrics,
|
|
FakeAgentSession,
|
|
} from "@agent-studio/harness";
|
|
import { access, mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join, resolve } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { Readable } from "node:stream";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
import { buildApp } from "../src/app.js";
|
|
|
|
const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
|
const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
|
|
const createdRoots: string[] = [];
|
|
|
|
afterEach(async () => {
|
|
await Promise.all(createdRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
|
|
});
|
|
|
|
describe("full fake Agent Studio flow", () => {
|
|
it.each(["approve", "deny", "timeout"] as const)("handles the %s report decision and cleans up", async (decision) => {
|
|
const root = await mkdtemp(join(tmpdir(), "agent-studio-full-flow-"));
|
|
createdRoots.push(root);
|
|
const sessionsRoot = join(root, "sessions");
|
|
await mkdir(sessionsRoot);
|
|
const events = new InMemoryEventStore();
|
|
const approvals = new InMemoryApprovalBroker({ eventStore: events, timeoutMs: decision === "timeout" ? 30 : 5_000 });
|
|
const models = new InMemoryModelConnectionStore({
|
|
validateUrl: (url) => Promise.resolve({ normalizedUrl: url, addresses: ["93.184.216.34"], redirectPolicy: "same-origin" }),
|
|
});
|
|
const attachments = new FileAttachmentStore({ sessionsRoot, eventStore: events });
|
|
const skillPath = join(projectRoot, ".agents/skills/ads-analysis/SKILL.md");
|
|
const fixturePath = join(projectRoot, "fixtures/ads-account.json");
|
|
const skillIndex = new SkillIndex([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]);
|
|
let fake: FakeAgentSession | undefined;
|
|
let reportPath = "";
|
|
|
|
const registry = new InMemorySessionRegistry<HarnessSession>({
|
|
sessionsRoot,
|
|
createSession: ({ sessionId, workspaceRoot }) => {
|
|
events.append(sessionId, { type: "session.started", payload: { cwd: "." } });
|
|
reportPath = join(workspaceRoot, "report.md");
|
|
return new HarnessSession({
|
|
sessionId,
|
|
workspaceRoot,
|
|
eventStore: events,
|
|
modelStore: models,
|
|
attachmentStore: attachments,
|
|
approvalBroker: approvals,
|
|
createSkillInference: (runId) => new SkillEventInference({ sessionId, runId, eventStore: events, skillIndex }),
|
|
createAgentPort: async ({ getActiveRunId }) => {
|
|
const workspace = await SafeWorkspace.create(workspaceRoot);
|
|
fake = new FakeAgentSession(async (port, prompt) => {
|
|
const runId = getActiveRunId();
|
|
if (runId === undefined) throw new Error("Run was not active");
|
|
expect(prompt.images).toHaveLength(1);
|
|
expect(prompt.images[0]).toMatchObject({ type: "image", mimeType: "image/png" });
|
|
|
|
port.emit({ type: "tool_execution_start", toolCallId: "tool_skill", toolName: "read", args: { path: skillPath } });
|
|
await port.flush();
|
|
port.emit({ type: "tool_execution_end", toolCallId: "tool_skill", toolName: "read", result: "skill metadata", isError: false });
|
|
port.emit({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta: "已读取分析方法。" } });
|
|
|
|
const metrics = await executeMockAdsMetrics(fixturePath, { accountId: "demo-account", days: 7 });
|
|
port.emit({ type: "tool_execution_start", toolCallId: "tool_metrics", toolName: "mock_ads_metrics", args: { accountId: "demo-account", days: 7 } });
|
|
port.emit({ type: "tool_execution_end", toolCallId: "tool_metrics", toolName: "mock_ads_metrics", result: metrics, isError: false });
|
|
|
|
const report = { title: "账户总结", body: `账户总结\nROAS ${String(metrics.roas)}` };
|
|
port.emit({ type: "tool_execution_start", toolCallId: "tool_report", toolName: "save_report_draft", args: report });
|
|
const resolution = await approvals.request({
|
|
approvalId: `ap_${runId}`,
|
|
sessionId,
|
|
runId,
|
|
toolCallId: "tool_report",
|
|
toolName: "save_report_draft",
|
|
arguments: report,
|
|
riskLevel: "medium",
|
|
});
|
|
if (resolution.status === "approved") {
|
|
const result = await workspace.writeReportDraft(report);
|
|
port.emit({ type: "tool_execution_end", toolCallId: "tool_report", toolName: "save_report_draft", result, isError: false });
|
|
} else {
|
|
port.emit({ type: "tool_execution_end", toolCallId: "tool_report", toolName: "save_report_draft", result: `Approval ${resolution.status}`, isError: true });
|
|
}
|
|
port.emit({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta: resolution.status === "approved" ? "报告已保存。" : "报告未写入。" } });
|
|
port.emit({ type: "message_end", message: { role: "assistant" } });
|
|
});
|
|
return fake;
|
|
},
|
|
});
|
|
},
|
|
});
|
|
const app = buildApp({
|
|
sessionRegistry: registry,
|
|
skillCatalog: { list: () => [{ name: "ads-analysis", description: "分析广告指标", source: "project", filePath: ".agents/skills/ads-analysis/SKILL.md", diagnostics: [] }] },
|
|
modelStore: models,
|
|
attachmentStore: attachments,
|
|
approvalBroker: approvals,
|
|
eventStore: events,
|
|
});
|
|
|
|
const created = await app.inject({ method: "POST", url: "/api/sessions" });
|
|
const { sessionId } = created.json<{ sessionId: string }>();
|
|
const configured = await app.inject({
|
|
method: "PUT",
|
|
url: `/api/sessions/${sessionId}/model-config`,
|
|
payload: { providerId: "training", api: "openai-responses", apiKey: "sk-test-secret", modelId: "training-model", supportsImages: true },
|
|
});
|
|
expect(configured.body).not.toContain("sk-test-secret");
|
|
const image = await attachments.add(sessionId, { name: "creative.png", mediaType: "image/png", stream: Readable.from(png) });
|
|
const sent = await app.inject({
|
|
method: "POST",
|
|
url: `/api/sessions/${sessionId}/messages`,
|
|
payload: { text: "分析 demo-account 并保存报告", attachmentIds: [image.attachmentId] },
|
|
});
|
|
expect(sent.statusCode).toBe(202);
|
|
|
|
await vi.waitFor(() => {
|
|
expect(events.listAfter(sessionId).events.some((event) => event.type === "approval.required")).toBe(true);
|
|
});
|
|
const approval = events.listAfter(sessionId).events.find((event) => event.type === "approval.required");
|
|
if (approval?.type !== "approval.required") throw new Error("Approval event missing");
|
|
if (decision !== "timeout") {
|
|
const response = await app.inject({ method: "POST", url: `/api/approvals/${approval.payload.approvalId}/${decision}` });
|
|
expect(response.statusCode).toBe(200);
|
|
}
|
|
await vi.waitFor(() => {
|
|
expect(events.listAfter(sessionId).events.at(-1)?.type).toBe("run.completed");
|
|
});
|
|
|
|
const allEvents = events.listAfter(sessionId).events;
|
|
expect(allEvents.map((event) => event.type)).toEqual(expect.arrayContaining([
|
|
"session.started", "attachment.uploaded", "user.message", "run.started", "skill.requested", "skill.loaded",
|
|
"tool.requested", "tool.completed", "approval.required", "approval.resolved", "assistant.delta", "assistant.completed", "run.completed",
|
|
]));
|
|
expect(allEvents.find((event) => event.type === "approval.resolved")).toMatchObject({ payload: { status: decision === "approve" ? "approved" : decision === "deny" ? "denied" : "expired" } });
|
|
expect(JSON.stringify(allEvents)).not.toContain("base64");
|
|
expect(JSON.stringify(allEvents)).not.toContain("sk-test-secret");
|
|
expect(fake?.prompts[0]?.images[0]?.data).toBeTruthy();
|
|
if (decision === "approve") expect(await readFile(reportPath, "utf8")).toContain("账户总结");
|
|
else await expect(access(reportPath)).rejects.toMatchObject({ code: "ENOENT" });
|
|
|
|
const deleted = await app.inject({ method: "DELETE", url: `/api/sessions/${sessionId}` });
|
|
expect(deleted.statusCode).toBe(200);
|
|
await expect(access(join(sessionsRoot, sessionId))).rejects.toMatchObject({ code: "ENOENT" });
|
|
expect(fake?.disposed).toBe(true);
|
|
await app.close();
|
|
});
|
|
});
|