diff --git a/fixtures/ads-account.json b/fixtures/ads-account.json new file mode 100644 index 0000000..3e46352 --- /dev/null +++ b/fixtures/ads-account.json @@ -0,0 +1,37 @@ +{ + "accounts": [ + { + "accountId": "demo-account", + "baseDays": 7, + "campaigns": [ + { + "campaignId": "cmp-scale", + "name": "Scale Winners", + "spend": 4200, + "impressions": 140000, + "clicks": 3500, + "conversions": 210, + "revenue": 12600 + }, + { + "campaignId": "cmp-low-ctr", + "name": "Creative Fatigue", + "spend": 2800, + "impressions": 160000, + "clicks": 1600, + "conversions": 35, + "revenue": 4200 + }, + { + "campaignId": "cmp-high-cpa", + "name": "Broad Prospecting", + "spend": 3000, + "impressions": 100000, + "clicks": 3000, + "conversions": 50, + "revenue": 5000 + } + ] + } + ] +} diff --git a/packages/harness/src/errors/harness-error.ts b/packages/harness/src/errors/harness-error.ts new file mode 100644 index 0000000..0c438f9 --- /dev/null +++ b/packages/harness/src/errors/harness-error.ts @@ -0,0 +1,10 @@ +export class HarnessError extends Error { + public constructor( + public readonly code: string, + message: string, + public readonly details: Record = {}, + ) { + super(message); + this.name = "HarnessError"; + } +} diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 93bfccf..0723f64 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -1,2 +1,7 @@ export * from "./events/event-store.js"; export * from "./events/in-memory-event-store.js"; +export * from "./errors/harness-error.js"; +export * from "./tools/mock-ads-metrics.js"; +export * from "./tools/save-report-draft.js"; +export * from "./tools/tool-policy.js"; +export * from "./workspace/safe-workspace.js"; diff --git a/packages/harness/src/tools/mock-ads-metrics.ts b/packages/harness/src/tools/mock-ads-metrics.ts new file mode 100644 index 0000000..eb2583a --- /dev/null +++ b/packages/harness/src/tools/mock-ads-metrics.ts @@ -0,0 +1,142 @@ +import { readFile } from "node:fs/promises"; +import { defineTool } from "@earendil-works/pi-coding-agent"; +import { type Static, Type } from "typebox"; +import { Check } from "typebox/value"; +import { HarnessError } from "../errors/harness-error.js"; + +export const mockAdsMetricsInputSchema = Type.Object({ + accountId: Type.String({ minLength: 1, maxLength: 100 }), + days: Type.Integer({ minimum: 1, maximum: 30 }), +}, { additionalProperties: false }); + +const campaignSchema = Type.Object({ + campaignId: Type.String(), + name: Type.String(), + spend: Type.Number({ minimum: 0 }), + impressions: Type.Integer({ minimum: 0 }), + clicks: Type.Integer({ minimum: 0 }), + conversions: Type.Integer({ minimum: 0 }), + revenue: Type.Number({ minimum: 0 }), +}, { additionalProperties: false }); + +const fixtureSchema = Type.Object({ + accounts: Type.Array(Type.Object({ + accountId: Type.String(), + baseDays: Type.Integer({ minimum: 1 }), + campaigns: Type.Array(campaignSchema), + }, { additionalProperties: false })), +}, { additionalProperties: false }); + +export type MockAdsMetricsInput = Static; +type CampaignFixture = Static; + +export type CampaignMetrics = CampaignFixture & { + ctr: number; + cpa: number; + roas: number; +}; + +export type AdsMetrics = { + accountId: string; + days: number; + spend: number; + impressions: number; + clicks: number; + ctr: number; + conversions: number; + cpa: number; + revenue: number; + roas: number; + campaigns: CampaignMetrics[]; +}; + +export async function executeMockAdsMetrics( + fixturePath: string, + input: MockAdsMetricsInput, +): Promise { + if (!Check(mockAdsMetricsInputSchema, input)) { + throw new HarnessError("INVALID_TOOL_ARGUMENTS", "Invalid mock_ads_metrics arguments"); + } + + const parsed: unknown = JSON.parse(await readFile(fixturePath, "utf8")); + if (!Check(fixtureSchema, parsed)) { + throw new HarnessError("INVALID_ADS_FIXTURE", "Ads fixture has an invalid structure"); + } + const fixture = parsed; + const account = fixture.accounts.find((candidate) => candidate.accountId === input.accountId); + if (account === undefined) { + throw new HarnessError("ADS_ACCOUNT_NOT_FOUND", "Mock advertising account not found", { + accountId: input.accountId, + }); + } + + const scale = input.days / account.baseDays; + const campaigns = account.campaigns.map((campaign) => scaleCampaign(campaign, scale)); + const spend = sum(campaigns, "spend"); + const impressions = sum(campaigns, "impressions"); + const clicks = sum(campaigns, "clicks"); + const conversions = sum(campaigns, "conversions"); + const revenue = sum(campaigns, "revenue"); + + return { + accountId: input.accountId, + days: input.days, + spend: round2(spend), + impressions: Math.round(impressions), + clicks: Math.round(clicks), + ctr: ratio(clicks * 100, impressions), + conversions: Math.round(conversions), + cpa: ratio(spend, conversions), + revenue: round2(revenue), + roas: ratio(revenue, spend), + campaigns, + }; +} + +export function createMockAdsMetricsTool(fixturePath: string) { + return defineTool({ + name: "mock_ads_metrics", + label: "Mock Ads Metrics", + description: "Read deterministic training metrics for a mock advertising account. This does not access a real ad platform.", + promptSnippet: "Read mock advertising account metrics", + parameters: mockAdsMetricsInputSchema, + async execute(_toolCallId, params) { + const details = await executeMockAdsMetrics(fixturePath, params); + return { + content: [{ type: "text", text: JSON.stringify(details) }], + details, + }; + }, + }); +} + +function scaleCampaign(campaign: CampaignFixture, scale: number): CampaignMetrics { + const spend = round2(campaign.spend * scale); + const impressions = Math.round(campaign.impressions * scale); + const clicks = Math.round(campaign.clicks * scale); + const conversions = Math.round(campaign.conversions * scale); + const revenue = round2(campaign.revenue * scale); + return { + ...campaign, + spend, + impressions, + clicks, + conversions, + revenue, + ctr: ratio(clicks * 100, impressions), + cpa: ratio(spend, conversions), + roas: ratio(revenue, spend), + }; +} + +function sum(items: CampaignMetrics[], key: "spend" | "impressions" | "clicks" | "conversions" | "revenue"): number { + return items.reduce((total, item) => total + item[key], 0); +} + +function ratio(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : round2(numerator / denominator); +} + +function round2(value: number): number { + return Math.round((value + Number.EPSILON) * 100) / 100; +} diff --git a/packages/harness/src/tools/save-report-draft.ts b/packages/harness/src/tools/save-report-draft.ts new file mode 100644 index 0000000..d5dff3c --- /dev/null +++ b/packages/harness/src/tools/save-report-draft.ts @@ -0,0 +1,26 @@ +import { defineTool } from "@earendil-works/pi-coding-agent"; +import { Type } from "typebox"; +import type { SafeWorkspace } from "../workspace/safe-workspace.js"; + +export const saveReportDraftInputSchema = Type.Object({ + title: Type.Optional(Type.String({ minLength: 1, maxLength: 200 })), + body: Type.String({ minLength: 1, maxLength: 200_000 }), +}, { additionalProperties: false }); + +export function createSaveReportDraftTool(workspace: SafeWorkspace) { + return defineTool({ + name: "save_report_draft", + label: "Save Report Draft", + description: "Save a report draft to the fixed report.md file in the current training Session. Requires approval.", + promptSnippet: "Save a report draft after analysis", + parameters: saveReportDraftInputSchema, + executionMode: "sequential", + async execute(_toolCallId, params) { + const details = await workspace.writeReportDraft(params); + return { + content: [{ type: "text", text: `Saved report draft to ${details.relativePath}` }], + details, + }; + }, + }); +} diff --git a/packages/harness/src/tools/tool-policy.ts b/packages/harness/src/tools/tool-policy.ts new file mode 100644 index 0000000..093acb8 --- /dev/null +++ b/packages/harness/src/tools/tool-policy.ts @@ -0,0 +1,9 @@ +export type ToolPolicyDecision = "allow" | "ask" | "deny"; + +const allowedTools = new Set(["read", "grep", "find", "ls", "mock_ads_metrics"]); + +export function decideToolPolicy(toolName: string): ToolPolicyDecision { + if (allowedTools.has(toolName)) return "allow"; + if (toolName === "save_report_draft") return "ask"; + return "deny"; +} diff --git a/packages/harness/src/workspace/safe-workspace.ts b/packages/harness/src/workspace/safe-workspace.ts new file mode 100644 index 0000000..fb0b246 --- /dev/null +++ b/packages/harness/src/workspace/safe-workspace.ts @@ -0,0 +1,142 @@ +import { randomUUID } from "node:crypto"; +import { lstat, open, realpath, rename, rm } from "node:fs/promises"; +import { isAbsolute, join, parse, relative, resolve, sep } from "node:path"; +import { HarnessError } from "../errors/harness-error.js"; + +const protectedSegments = new Set([ + ".env", + ".git", + ".ssh", + "id_rsa", + "id_ed25519", +]); + +export type ReportDraftInput = { + title?: string; + body: string; +}; + +export class SafeWorkspace { + private constructor(public readonly root: string) {} + + public static async create(root: string): Promise { + const canonicalRoot = await realpath(resolve(root)); + if (canonicalRoot === parse(canonicalRoot).root) { + throw new HarnessError("UNSAFE_WORKSPACE_ROOT", "Filesystem root cannot be used as a workspace"); + } + const metadata = await lstat(canonicalRoot); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new HarnessError("UNSAFE_WORKSPACE_ROOT", "Workspace root must be a regular directory"); + } + return new SafeWorkspace(canonicalRoot); + } + + public async resolveReadable(inputPath: string): Promise { + const candidate = isAbsolute(inputPath) ? resolve(inputPath) : resolve(this.root, inputPath); + this.#assertContained(candidate); + this.#assertUnprotected(candidate); + await this.#assertNoSymlinkSegments(candidate); + const canonical = await realpath(candidate); + this.#assertContained(canonical); + const metadata = await lstat(canonical); + if (!metadata.isFile()) { + throw new HarnessError("NOT_A_REGULAR_FILE", "Only regular files may be read"); + } + return canonical; + } + + public async writeReportDraft(input: ReportDraftInput): Promise<{ relativePath: "report.md"; byteSize: number }> { + if (input.body.trim().length === 0) { + throw new HarnessError("REPORT_BODY_REQUIRED", "Report body cannot be empty"); + } + + const target = join(this.root, "report.md"); + const temporary = join(this.root, `.report-${randomUUID()}.tmp`); + this.#assertContained(target); + await this.#assertNoSymlinkSegments(this.root); + + const title = input.title?.trim(); + const content = title === undefined || title.length === 0 + ? `${input.body.trim()}\n` + : `# ${title}\n\n${input.body.trim()}\n`; + const bytes = Buffer.from(content, "utf8"); + let handle: Awaited> | undefined; + + try { + handle = await open(temporary, "wx", 0o600); + await handle.writeFile(bytes); + await handle.sync(); + await handle.close(); + handle = undefined; + + try { + const existing = await lstat(target); + if (existing.isSymbolicLink()) { + throw new HarnessError("SYMLINK_NOT_ALLOWED", "Report target cannot be a symbolic link"); + } + } catch (error) { + if (error instanceof HarnessError) throw error; + if (!isMissingFileError(error)) throw error; + } + + await rename(temporary, target); + return { relativePath: "report.md", byteSize: bytes.byteLength }; + } finally { + await handle?.close().catch(() => undefined); + await rm(temporary, { force: true }).catch(() => undefined); + } + } + + public async remove(): Promise { + const metadata = await lstat(this.root).catch((error: unknown) => { + if (isMissingFileError(error)) return undefined; + throw error; + }); + if (metadata === undefined) return; + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new HarnessError("UNSAFE_WORKSPACE_ROOT", "Refusing to remove a non-directory workspace root"); + } + await rm(this.root, { recursive: true, force: false }); + } + + #assertContained(candidate: string): void { + const pathFromRoot = relative(this.root, candidate); + if (pathFromRoot === "") return; + if (pathFromRoot === ".." || pathFromRoot.startsWith(`..${sep}`) || isAbsolute(pathFromRoot)) { + throw new HarnessError("PATH_OUTSIDE_WORKSPACE", "Path is outside the Session workspace"); + } + } + + #assertUnprotected(candidate: string): void { + const pathFromRoot = relative(this.root, candidate); + const segments = pathFromRoot.split(sep); + if (segments.some((segment) => protectedSegments.has(segment.toLowerCase()))) { + throw new HarnessError("PROTECTED_PATH", "Protected paths cannot be accessed"); + } + } + + async #assertNoSymlinkSegments(candidate: string): Promise { + this.#assertContained(candidate); + const pathFromRoot = relative(this.root, candidate); + if (pathFromRoot === "") { + const rootMetadata = await lstat(this.root); + if (rootMetadata.isSymbolicLink()) { + throw new HarnessError("SYMLINK_NOT_ALLOWED", "Symbolic links are not allowed"); + } + return; + } + + let current = this.root; + for (const segment of pathFromRoot.split(sep)) { + current = join(current, segment); + const metadata = await lstat(current); + if (metadata.isSymbolicLink()) { + throw new HarnessError("SYMLINK_NOT_ALLOWED", "Symbolic links are not allowed"); + } + } + } +} + +function isMissingFileError(error: unknown): boolean { + return error instanceof Error && "code" in error && error.code === "ENOENT"; +} diff --git a/packages/harness/tests/safe-workspace.test.ts b/packages/harness/tests/safe-workspace.test.ts new file mode 100644 index 0000000..0a423ee --- /dev/null +++ b/packages/harness/tests/safe-workspace.test.ts @@ -0,0 +1,61 @@ +import { mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import * as harness from "../src/index.js"; + +type Workspace = { + resolveReadable(path: string): Promise; + writeReportDraft(input: { title?: string; body: string }): Promise<{ relativePath: "report.md"; byteSize: number }>; + remove(): Promise; +}; + +type WorkspaceConstructor = { + create(root: string): Promise; +}; + +let root = ""; +let outside = ""; +let workspace: Workspace; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "agent-studio-workspace-")); + outside = await mkdtemp(join(tmpdir(), "agent-studio-outside-")); + await writeFile(join(root, "inside.txt"), "inside", "utf8"); + await writeFile(join(outside, "secret.txt"), "secret", "utf8"); + const Constructor = Reflect.get(harness, "SafeWorkspace") as WorkspaceConstructor | undefined; + expect(Constructor, "SafeWorkspace must be exported").toBeDefined(); + workspace = await Constructor!.create(root); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); +}); + +describe("SafeWorkspace", () => { + it("resolves an existing regular file inside the Session root", async () => { + await expect(workspace.resolveReadable("inside.txt")).resolves.toBe(await realpath(join(root, "inside.txt"))); + }); + + it.each(["../outside/secret.txt", "/tmp/outside.txt"])("rejects path escape %s", async (path) => { + await expect(workspace.resolveReadable(path)).rejects.toMatchObject({ code: "PATH_OUTSIDE_WORKSPACE" }); + }); + + it("rejects symlinks before following them", async () => { + await symlink(outside, join(root, "link")); + await expect(workspace.resolveReadable("link/secret.txt")).rejects.toMatchObject({ code: "SYMLINK_NOT_ALLOWED" }); + }); + + it("rejects protected path segments", async () => { + await mkdir(join(root, ".git")); + await writeFile(join(root, ".git", "config"), "credential", "utf8"); + await expect(workspace.resolveReadable(".git/config")).rejects.toMatchObject({ code: "PROTECTED_PATH" }); + }); + + it("writes only the fixed report.md path", async () => { + const result = await workspace.writeReportDraft({ title: "广告账户报告", body: "账户总结" }); + expect(result.relativePath).toBe("report.md"); + await expect(readFile(join(root, "report.md"), "utf8")).resolves.toBe("# 广告账户报告\n\n账户总结\n"); + }); +}); diff --git a/packages/harness/tests/tools.test.ts b/packages/harness/tests/tools.test.ts new file mode 100644 index 0000000..9f558a4 --- /dev/null +++ b/packages/harness/tests/tools.test.ts @@ -0,0 +1,55 @@ +import { resolve } from "node:path"; +import { Check } from "typebox/value"; +import { describe, expect, it } from "vitest"; +import { + decideToolPolicy, + executeMockAdsMetrics, + mockAdsMetricsInputSchema, +} from "../src/index.js"; + +const fixturePath = resolve(import.meta.dirname, "../../../fixtures/ads-account.json"); + +describe("mock_ads_metrics", () => { + it("uses an integer 1-30 day TypeBox schema", () => { + expect(Check(mockAdsMetricsInputSchema, { accountId: "demo-account", days: 7 })).toBe(true); + expect(Check(mockAdsMetricsInputSchema, { accountId: "demo-account", days: 0 })).toBe(false); + expect(Check(mockAdsMetricsInputSchema, { accountId: "demo-account", days: 7.5 })).toBe(false); + expect(Check(mockAdsMetricsInputSchema, { accountId: "demo-account", days: 31 })).toBe(false); + }); + + it("returns hand-checked zero-safe metrics and campaign details", async () => { + const result = await executeMockAdsMetrics(fixturePath, { accountId: "demo-account", days: 7 }); + expect(result).toMatchObject({ + spend: 10000, + impressions: 400000, + clicks: 8100, + ctr: 2.03, + conversions: 295, + cpa: 33.9, + revenue: 21800, + roas: 2.18, + days: 7, + }); + expect(result.campaigns).toContainEqual(expect.objectContaining({ + campaignId: "cmp-low-ctr", + ctr: 1, + cpa: 80, + })); + }); + + it("rejects unknown accounts with a stable error code", async () => { + await expect(executeMockAdsMetrics(fixturePath, { accountId: "missing", days: 7 })) + .rejects.toMatchObject({ code: "ADS_ACCOUNT_NOT_FOUND" }); + }); +}); + +describe("tool policy", () => { + it("allows only the explicit safe set, asks for report save, and denies unknown tools", () => { + expect(["read", "grep", "find", "ls", "mock_ads_metrics"].map(decideToolPolicy)).toEqual([ + "allow", "allow", "allow", "allow", "allow", + ]); + expect(decideToolPolicy("save_report_draft")).toBe("ask"); + expect(decideToolPolicy("bash")).toBe("deny"); + expect(decideToolPolicy("unknown_tool")).toBe("deny"); + }); +});