test: cover complete fake agent flow

2026-09-08 10:35:11 +08:00
parent c4a88e4aae
commit 89ce7973c3
8 changed files with 406 additions and 2 deletions
@@ -0,0 +1,159 @@
+import {
+  FileAttachmentStore,
+  HarnessSession,
+  InMemoryApprovalBroker,
+  InMemoryEventStore,
+  InMemoryModelConnectionStore,
+  InMemorySessionRegistry,
+  SafeWorkspace,
+  SkillEventInference,
+  SkillIndex,
+  executeMockAdsMetrics,
+  FakeAgentSession,
+} from "@agent-studio/harness";
+import { access, mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Readable } from "node:stream";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { buildApp } from "../src/app.js";
+
+const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
+const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
+const createdRoots: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(createdRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
+});
+
+describe("full fake Agent Studio flow", () => {
+  it.each(["approve", "deny", "timeout"] as const)("handles the %s report decision and cleans up", async (decision) => {
+    const root = await mkdtemp(join(tmpdir(), "agent-studio-full-flow-"));
+    createdRoots.push(root);
+    const sessionsRoot = join(root, "sessions");
+    await mkdir(sessionsRoot);
+    const events = new InMemoryEventStore();
+    const approvals = new InMemoryApprovalBroker({ eventStore: events, timeoutMs: decision === "timeout" ? 30 : 5_000 });
+    const models = new InMemoryModelConnectionStore({
+      validateUrl: (url) => Promise.resolve({ normalizedUrl: url, addresses: ["93.184.216.34"], redirectPolicy: "same-origin" }),
+    });
+    const attachments = new FileAttachmentStore({ sessionsRoot, eventStore: events });
+    const skillPath = join(projectRoot, ".agents/skills/ads-analysis/SKILL.md");
+    const fixturePath = join(projectRoot, "fixtures/ads-account.json");
+    const skillIndex = new SkillIndex([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]);
+    let fake: FakeAgentSession | undefined;
+    let reportPath = "";
+
+    const registry = new InMemorySessionRegistry<HarnessSession>({
+      sessionsRoot,
+      createSession: ({ sessionId, workspaceRoot }) => {
+        events.append(sessionId, { type: "session.started", payload: { cwd: "." } });
+        reportPath = join(workspaceRoot, "report.md");
+        return new HarnessSession({
+          sessionId,
+          workspaceRoot,
+          eventStore: events,
+          modelStore: models,
+          attachmentStore: attachments,
+          approvalBroker: approvals,
+          createSkillInference: (runId) => new SkillEventInference({ sessionId, runId, eventStore: events, skillIndex }),
+          createAgentPort: async ({ getActiveRunId }) => {
+            const workspace = await SafeWorkspace.create(workspaceRoot);
+            fake = new FakeAgentSession(async (port, prompt) => {
+              const runId = getActiveRunId();
+              if (runId === undefined) throw new Error("Run was not active");
+              expect(prompt.images).toHaveLength(1);
+              expect(prompt.images[0]).toMatchObject({ type: "image", mimeType: "image/png" });
+
+              port.emit({ type: "tool_execution_start", toolCallId: "tool_skill", toolName: "read", args: { path: skillPath } });
+              await port.flush();
+              port.emit({ type: "tool_execution_end", toolCallId: "tool_skill", toolName: "read", result: "skill metadata", isError: false });
+              port.emit({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta: "已读取分析方法。" } });
+
+              const metrics = await executeMockAdsMetrics(fixturePath, { accountId: "demo-account", days: 7 });
+              port.emit({ type: "tool_execution_start", toolCallId: "tool_metrics", toolName: "mock_ads_metrics", args: { accountId: "demo-account", days: 7 } });
+              port.emit({ type: "tool_execution_end", toolCallId: "tool_metrics", toolName: "mock_ads_metrics", result: metrics, isError: false });
+
+              const report = { title: "账户总结", body: `账户总结\nROAS ${String(metrics.roas)}` };
+              port.emit({ type: "tool_execution_start", toolCallId: "tool_report", toolName: "save_report_draft", args: report });
+              const resolution = await approvals.request({
+                approvalId: `ap_${runId}`,
+                sessionId,
+                runId,
+                toolCallId: "tool_report",
+                toolName: "save_report_draft",
+                arguments: report,
+                riskLevel: "medium",
+              });
+              if (resolution.status === "approved") {
+                const result = await workspace.writeReportDraft(report);
+                port.emit({ type: "tool_execution_end", toolCallId: "tool_report", toolName: "save_report_draft", result, isError: false });
+              } else {
+                port.emit({ type: "tool_execution_end", toolCallId: "tool_report", toolName: "save_report_draft", result: `Approval ${resolution.status}`, isError: true });
+              }
+              port.emit({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta: resolution.status === "approved" ? "报告已保存。" : "报告未写入。" } });
+              port.emit({ type: "message_end", message: { role: "assistant" } });
+            });
+            return fake;
+          },
+        });
+      },
+    });
+    const app = buildApp({
+      sessionRegistry: registry,
+      skillCatalog: { list: () => [{ name: "ads-analysis", description: "分析广告指标", source: "project", filePath: ".agents/skills/ads-analysis/SKILL.md", diagnostics: [] }] },
+      modelStore: models,
+      attachmentStore: attachments,
+      approvalBroker: approvals,
+      eventStore: events,
+    });
+
+    const created = await app.inject({ method: "POST", url: "/api/sessions" });
+    const { sessionId } = created.json<{ sessionId: string }>();
+    const configured = await app.inject({
+      method: "PUT",
+      url: `/api/sessions/${sessionId}/model-config`,
+      payload: { providerId: "training", api: "openai-responses", apiKey: "sk-test-secret", modelId: "training-model", supportsImages: true },
+    });
+    expect(configured.body).not.toContain("sk-test-secret");
+    const image = await attachments.add(sessionId, { name: "creative.png", mediaType: "image/png", stream: Readable.from(png) });
+    const sent = await app.inject({
+      method: "POST",
+      url: `/api/sessions/${sessionId}/messages`,
+      payload: { text: "分析 demo-account 并保存报告", attachmentIds: [image.attachmentId] },
+    });
+    expect(sent.statusCode).toBe(202);
+
+    await vi.waitFor(() => {
+      expect(events.listAfter(sessionId).events.some((event) => event.type === "approval.required")).toBe(true);
+    });
+    const approval = events.listAfter(sessionId).events.find((event) => event.type === "approval.required");
+    if (approval?.type !== "approval.required") throw new Error("Approval event missing");
+    if (decision !== "timeout") {
+      const response = await app.inject({ method: "POST", url: `/api/approvals/${approval.payload.approvalId}/${decision}` });
+      expect(response.statusCode).toBe(200);
+    }
+    await vi.waitFor(() => {
+      expect(events.listAfter(sessionId).events.at(-1)?.type).toBe("run.completed");
+    });
+
+    const allEvents = events.listAfter(sessionId).events;
+    expect(allEvents.map((event) => event.type)).toEqual(expect.arrayContaining([
+      "session.started", "attachment.uploaded", "user.message", "run.started", "skill.requested", "skill.loaded",
+      "tool.requested", "tool.completed", "approval.required", "approval.resolved", "assistant.delta", "assistant.completed", "run.completed",
+    ]));
+    expect(allEvents.find((event) => event.type === "approval.resolved")).toMatchObject({ payload: { status: decision === "approve" ? "approved" : decision === "deny" ? "denied" : "expired" } });
+    expect(JSON.stringify(allEvents)).not.toContain("base64");
+    expect(JSON.stringify(allEvents)).not.toContain("sk-test-secret");
+    expect(fake?.prompts[0]?.images[0]?.data).toBeTruthy();
+    if (decision === "approve") expect(await readFile(reportPath, "utf8")).toContain("账户总结");
+    else await expect(access(reportPath)).rejects.toMatchObject({ code: "ENOENT" });
+
+    const deleted = await app.inject({ method: "DELETE", url: `/api/sessions/${sessionId}` });
+    expect(deleted.statusCode).toBe(200);
+    await expect(access(join(sessionsRoot, sessionId))).rejects.toMatchObject({ code: "ENOENT" });
+    expect(fake?.disposed).toBe(true);
+    await app.close();
+  });
+});