wujianhui/sw-ads-agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(swads): make daily report generation resilient

Browse Source
This commit is contained in:
Jeffrey Wu
2026-09-09 17:59:05 +08:00
parent f4e3f41c2c
commit 8361b10c66
13 changed files with 278 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/harness/tests/session.test.ts
+13 -4
View File
@@ -38,12 +38,12 @@ class FakeFactory implements AgentSessionFactory {
async create(input: AgentSessionFactoryInput): Promise<AgentSessionPort> { this.input = input; return this.port; }
}
async function setup(supportsImages = true) {
async function setup(supportsImages = true, runTimeoutMs = 5_000) {
const root = await mkdtemp(path.join(tmpdir(), "studio-session-")); const sessionsRoot = path.join(root, "sessions"); await mkdir(sessionsRoot);
await mkdir(path.join(root, ".agents/skills"), { recursive: true }); await mkdir(path.join(root, ".pi/skills"), { recursive: true });
const catalog = new SkillCatalog(root, path.join(root, "agent-dir")); await catalog.reload();
const events = new InMemoryEventStore(); const attachments = new FileAttachmentStore(sessionsRoot); const models = new InMemoryModelConnectionStore({}); const factory = new FakeFactory(supportsImages);
const registry = new InMemorySessionRegistry(sessionsRoot, events, attachments, models, catalog, factory, 500, 5_000);
const registry = new InMemorySessionRegistry(sessionsRoot, events, attachments, models, catalog, factory, 500, runTimeoutMs);
const session = await registry.create(); await registry.configureModel(session.sessionId, { providerId: "fake", api: "openai-responses", apiKey: "test-secret", modelId: "fake-model", supportsImages });
return { root, sessionsRoot, events, attachments, models, catalog, factory, registry, session };
}
@@ -71,6 +71,15 @@ describe("session lifecycle", () => {
it("cancels an active run and disposes on deletion", async () => {
const context = await setup(); await context.registry.send(context.session.sessionId, { text: "hello", attachmentIds: [] }); await context.registry.cancel(context.session.sessionId, "user"); expect(context.factory.port.aborted).toBe(true); await vi.waitFor(() => expect(context.session.busy).toBe(false)); await context.registry.delete(context.session.sessionId); expect(context.factory.port.disposed).toBe(true); await expect(stat(context.session.cwd)).rejects.toThrow();
});
it("treats the run timeout as an inactivity timeout", async () => {
vi.useFakeTimers();
try {
const context = await setup(true, 100); await context.registry.send(context.session.sessionId, { text: "hello", attachmentIds: [] });
await vi.advanceTimersByTimeAsync(80); context.factory.port.emit({ type: "message_update", message: { role: "assistant" }, assistantMessageEvent: { type: "text_delta", delta: "still working" } });
await vi.advanceTimersByTimeAsync(80); expect(context.session.busy).toBe(true);
await vi.advanceTimersByTimeAsync(21); await Promise.resolve(); expect(context.factory.port.aborted).toBe(true); expect(context.events.listAfter(context.session.sessionId).events.at(-1)?.type).toBe("run.cancelled");
} finally { vi.useRealTimers(); }
});
it("maps fake Pi deltas and tool events centrally", async () => {
const context = await setup(); const { runId } = await context.registry.send(context.session.sessionId, { text: "hello", attachmentIds: [] }); context.factory.port.emit({ type: "message_update", message: { role: "assistant" }, assistantMessageEvent: { type: "text_delta", delta: "流式" } }); context.factory.input?.toolRequested("tool_1", "mock_ads_metrics", { accountId: "demo-account", days: 7 }); context.factory.port.emit({ type: "tool_execution_start", toolCallId: "tool_1", toolName: "mock_ads_metrics", args: {} }); context.factory.port.emit({ type: "tool_execution_end", toolCallId: "tool_1", toolName: "mock_ads_metrics", result: { ok: true }, isError: false }); context.factory.port.complete(); await vi.waitFor(() => expect(context.session.busy).toBe(false)); const runEvents = context.events.listAfter(context.session.sessionId).events.filter((event) => event.runId === runId); expect(runEvents.map((event) => event.type)).toContain("assistant.delta"); expect(runEvents.filter((event) => event.type === "tool.requested")).toHaveLength(1); expect(runEvents.map((event) => event.type)).toContain("tool.completed");
});
@@ -95,9 +104,9 @@ describe("session lifecycle", () => {
const context = await setup(false); await context.registry.send(context.session.sessionId, { text: "report", attachmentIds: [] });
const accountKey = "acct_0123456789abcdef01234567", reportDate = "2026-09-06", reportId = "12345678-1234-4234-8234-123456789abc"; const url = `/api/reports/${accountKey}/${reportDate}/${reportId}`;
const artifact = { accountKey, reportDate, reportId, previewUrl: `${url}/png`, downloads: { png: `${url}/png`, html: `${url}/html`, json: `${url}/json` } };
context.factory.port.emit({ type: "tool_execution_end", toolCallId: "render", toolName: "render_swads_daily_report", isError: false, result: { content: [{ type: "text", text: "/private/SHOULD_NOT_LEAK full report body" }], details: artifact } });
context.factory.port.emit({ type: "tool_execution_end", toolCallId: "render", toolName: "render_swads_daily_report", isError: false, result: { content: [{ type: "text", text: "/private/SHOULD_NOT_LEAK full report body" }], details: { artifact, markdown: "## 可见日报摘要" } } });
context.factory.port.emit({ type: "tool_execution_end", toolCallId: "whoami", toolName: "swads_cli", isError: true, result: { content: [{ type: "text", text: "SWADS_AUTH_FAILED" }] } });
const events = context.events.listAfter(context.session.sessionId).events; const generated = events.find((e) => e.type === "report.generated"); expect(generated?.payload).toMatchObject({ assistantMessageId: context.session.currentRun!.assistantMessageId, artifact }); expect(JSON.stringify(events)).not.toContain("SHOULD_NOT_LEAK"); expect(events.find((e) => e.type === "tool.failed")?.payload).toMatchObject({ error: "SWADS_AUTH_FAILED" }); context.factory.port.complete();
const events = context.events.listAfter(context.session.sessionId).events; const generated = events.find((e) => e.type === "report.generated"); expect(generated?.payload).toMatchObject({ assistantMessageId: context.session.currentRun!.assistantMessageId, artifact, markdown: "## 可见日报摘要" }); expect(JSON.stringify(events)).not.toContain("SHOULD_NOT_LEAK"); expect(events.find((e) => e.type === "tool.failed")?.payload).toMatchObject({ error: "SWADS_AUTH_FAILED" }); context.factory.port.complete();
});
});
packages/harness/tests/swads.test.ts
+33 -2
View File
@@ -27,6 +27,12 @@ describe("swads CLI boundary", () => {
await f.tools.cli({ command: "metrics.catalog", arguments: { platform: "tiktok" } }, signal()); expect(f.connection.list).toHaveBeenCalledTimes(1); expect(f.connection.call).toHaveBeenCalledWith("metrics_catalog", { platform: "tiktok" }, expect.any(AbortSignal));
expect(JSON.stringify(f.tools.definitions())).not.toContain(token);
});
it("compacts large identity results before they enter model context", async () => {
const f = fixture(); const account = { platform_account_binding_id: "binding", tenant_id: "tenant", platform: "tiktok", external_account_id: "account", account_name: "Store", access_profile: "reader", permissions: Array(100).fill("private-permission"), is_current: false, reporting_timezone: "Asia/Kuala_Lumpur", reporting_utc_offset_minutes: 480, account_currency: "MYR" };
vi.mocked(f.connection.call).mockResolvedValueOnce({ structuredContent: { access: { user_id: "user", current_account: { ...account, is_current: true }, accounts: Array.from({ length: 57 }, (_, index) => ({ ...account, external_account_id: `account-${index}` })) }, orgs: { current_org_id: "org", organizations: [{ org_id: "org", name: "Org", slug: "org", role: "member" }] } } });
const result = await f.tools.cli({ command: "whoami", arguments: {} }, signal()); const serialized = JSON.stringify(result);
expect(serialized).not.toContain("private-permission"); expect(Buffer.byteLength(serialized)).toBeLessThan(50 * 1024); expect((result as { access: { accounts: unknown[] } }).access.accounts).toHaveLength(57);
});
it("refuses missing tools, non-read-only annotations and remote input drift", async () => {
const f = fixture(capabilities().filter((tool) => tool.name !== "metrics_semantic_query")); await expect(f.tools.cli({ command: "capabilities", arguments: {} }, signal())).rejects.toMatchObject({ code: "SWADS_TOOL_MISSING" }); await expect(f.tools.render(sampleReport(), signal())).rejects.toMatchObject({ code: "SWADS_PREFLIGHT_REQUIRED" }); expect(f.create).not.toHaveBeenCalled();
const list = capabilities(); list.find((tool) => tool.name === "runtime_findings")!.annotations = { readOnlyHint: false, destructiveHint: true }; const unsafe = fixture(list); await expect(unsafe.tools.cli({ command: "runtime.findings", arguments: { platform: "tiktok", external_account_id: "demo" } }, signal())).rejects.toMatchObject({ code: "SWADS_NOT_READ_ONLY" }); expect(unsafe.connection.call).not.toHaveBeenCalled();
@@ -35,8 +41,33 @@ describe("swads CLI boundary", () => {
it("requires successful real preflight and rows in this run to render", async () => {
const f = fixture(); await expect(f.tools.render(sampleReport(), signal())).rejects.toThrow(); await preflight(f.tools); const report = sampleReport(); report.notes.push(token); await f.tools.render(report, signal()); expect(JSON.stringify(vi.mocked(f.create).mock.calls)).not.toContain(token); f.nextRun(); await expect(f.tools.render(report, signal())).rejects.toThrow();
});
it("invalidates render evidence after query failure and rejects empty data", async () => {
const f = fixture(); await preflight(f.tools); vi.mocked(f.connection.call).mockResolvedValue({ structuredContent: { columns: [{ name: "spend" }], rows: [] } }); await expect(f.tools.cli({ command: "metrics.query", arguments: { platform: "tiktok", query: {} } }, signal())).rejects.toMatchObject({ code: "SWADS_NO_DATA" }); await expect(f.tools.render(sampleReport(), signal())).rejects.toThrow();
it("renders a deterministic fallback from four collected query kinds", async () => {
const f = fixture(); const result = (data: unknown) => ({ structuredContent: data });
await f.tools.cli({ command: "capabilities", arguments: {} }, signal());
vi.mocked(f.connection.call).mockResolvedValueOnce(result({ access: { current_account: { account_name: "Store", external_account_id: "account", account_currency: "MYR", reporting_timezone: "Asia/Kuala_Lumpur" } }, orgs: {} }));
await f.tools.cli({ command: "whoami", arguments: {} }, signal());
vi.mocked(f.connection.call).mockResolvedValueOnce(result({ metrics: [{ name: "spend" }] }));
await f.tools.cli({ command: "metrics.catalog", arguments: { platform: "tiktok" } }, signal());
const query = async (query_kind: string, columns: string[], rows: unknown[][]) => { vi.mocked(f.connection.call).mockResolvedValueOnce(result({ columns: columns.map((name) => ({ name })), rows, row_count: rows.length, account_context: { external_account_id: "account", account_currency: "MYR", reporting_timezone: "Asia/Kuala_Lumpur" } })); await f.tools.cli({ command: "metrics.query", arguments: { platform: "tiktok", query: { query_kind, date_from: "2026-09-06", date_to: "2026-09-06" } } }, signal()); };
await query("summary", ["spend", "gross_revenue", "orders", "ctr"], [[10, 20, 2, 0.01]]);
await query("timeseries", ["date", "spend", "gross_revenue"], [["2026-09-05", 10, 15], ["2026-09-06", 10, 20]]);
await query("campaign_breakdown", ["external_campaign_id", "external_campaign_name", "spend", "gross_revenue", "orders"], [["campaign", "GMV Max", 10, 20, 2]]);
await query("creative_breakdown", ["external_creative_id", "external_creative_name", "creative_creator_name", "spend", "gross_revenue", "orders", "product_clicks", "product_ctr"], [["creative", "", "", 10, 20, 2, 3, 0.02]]);
const fallback = await f.tools.renderCollected(signal());
expect(fallback.markdown).toContain("SW Ads 日报"); expect(f.create).toHaveBeenCalledOnce(); expect(vi.mocked(f.create).mock.calls[0]?.[0]).toMatchObject({ meta: { report_date: "2026-09-06" }, summary: { spend: 10, revenue: 20, roas: 2 }, creative: { source_groups: [{ label: "未知创作者" }], winners: [{ id: "creative", name: "creative", creator: "未知创作者" }] } });
});
it("accepts an empty entity breakdown when the observed account queries have data", async () => {
const f = fixture(); await preflight(f.tools);
vi.mocked(f.connection.call).mockResolvedValueOnce({ structuredContent: { columns: [], rows: [], row_count: 0 } });
await expect(f.tools.cli({ command: "metrics.query", arguments: { platform: "tiktok", query: { query_kind: "campaign_breakdown" } } }, signal())).resolves.toMatchObject({ rows: [], row_count: 0 });
await expect(f.tools.render(sampleReport(), signal())).resolves.toEqual({ ok: true });
});
it("preserves preflight for queued queries but invalidates render evidence after empty account data", async () => {
const f = fixture(); await preflight(f.tools);
vi.mocked(f.connection.call).mockResolvedValueOnce({ structuredContent: { columns: [], rows: [] } });
await expect(f.tools.cli({ command: "metrics.query", arguments: { platform: "tiktok", query: { query_kind: "summary" } } }, signal())).rejects.toMatchObject({ code: "SWADS_NO_DATA" });
await expect(f.tools.cli({ command: "metrics.query", arguments: { platform: "tiktok", query: { metrics: ["spend"] } } }, signal())).resolves.toMatchObject({ rows: [[100]] });
await expect(f.tools.render(sampleReport(), signal())).rejects.toMatchObject({ code: "SWADS_PREFLIGHT_REQUIRED" });
});
it("classifies upstream permission envelopes and response limits without their contents", async () => {
for (const [status, code] of [[401, "SWADS_AUTH_FAILED"], [403, "SWADS_FORBIDDEN"]] as const) { const f = fixture(); vi.mocked(f.connection.call).mockResolvedValue({ structuredContent: { error_code: "upstream", upstream_status: status, detail: token } }); await expect(f.tools.cli({ command: "whoami", arguments: {} }, signal())).rejects.toMatchObject({ code, message: code }); }