diff --git a/.agents/skills/ads-analysis/SKILL.md b/.agents/skills/ads-analysis/SKILL.md new file mode 100644 index 0000000..791af52 --- /dev/null +++ b/.agents/skills/ads-analysis/SKILL.md @@ -0,0 +1,21 @@ +--- +name: ads-analysis +description: Use when analyzing the bundled mock advertising account, identifying campaign anomalies, or recommending budget and creative actions from mock ad metrics. +--- + +# Ads Analysis + +Call `mock_ads_metrics` before calculating, comparing, or explaining any account or campaign metric. + +Treat every returned value as fixture data. Never describe it as live, connected, or sourced from a real advertising platform. State the available date range when presenting results. + +Base all numeric claims on the tool response. Images may provide supporting creative context, but never use an image to invent or override spend, impressions, clicks, conversions, or revenue. + +Analyze the account in this order: + +1. Summarize spend, impressions, clicks, conversions, and revenue. +2. Compare campaigns using derived efficiency metrics such as CTR, CPC, CPA, CVR, and ROAS when their denominators are non-zero. +3. Flag meaningful outliers and explain the evidence for each flag. +4. Recommend concrete budget, targeting, or creative actions, separating evidence from inference. + +If the requested range is unavailable, explain the limitation instead of extrapolating beyond the fixture. diff --git a/.pi/skills/report-writer/SKILL.md b/.pi/skills/report-writer/SKILL.md new file mode 100644 index 0000000..b85335f --- /dev/null +++ b/.pi/skills/report-writer/SKILL.md @@ -0,0 +1,20 @@ +--- +name: report-writer +description: Use when turning completed advertising analysis into a concise Chinese report and saving the approved final draft to the project workspace. +--- + +# Report Writer + +Write the report in Chinese with these sections: + +1. 账户总结 +2. 关键指标 +3. 异常 Campaign +4. 优化建议 +5. 数据时间范围 + +Keep facts traceable to the available analysis. Label uncertain explanations as inference, keep recommendations actionable, and do not imply that fixture data is live platform data. + +After composing the complete report, call `save_report_draft` with the full Markdown draft. This action requires approval; do not claim that the report was saved until the tool succeeds. + +If approval is denied, expires, or the tool fails, still provide the complete report text in the conversation and clearly state that the draft was not saved. diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 0723f64..78a5efc 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -1,6 +1,8 @@ export * from "./events/event-store.js"; export * from "./events/in-memory-event-store.js"; export * from "./errors/harness-error.js"; +export * from "./skills/skill-catalog.js"; +export * from "./skills/skill-event-inference.js"; export * from "./tools/mock-ads-metrics.js"; export * from "./tools/save-report-draft.js"; export * from "./tools/tool-policy.js"; diff --git a/packages/harness/src/skills/skill-catalog.ts b/packages/harness/src/skills/skill-catalog.ts new file mode 100644 index 0000000..75bc056 --- /dev/null +++ b/packages/harness/src/skills/skill-catalog.ts @@ -0,0 +1,196 @@ +import { access, realpath, stat } from "node:fs/promises"; +import { realpathSync } from "node:fs"; +import { delimiter, isAbsolute, relative, resolve, sep } from "node:path"; +import { + DefaultResourceLoader, + SettingsManager, + type ResourceDiagnostic, + type Skill, +} from "@earendil-works/pi-coding-agent"; + +export type SkillDiagnostic = { + level: "warning" | "error"; + message: string; + filePath?: string; +}; + +export type SkillCatalogEntry = { + name: string; + description: string; + source: string; + filePath: string; + diagnostics: SkillDiagnostic[]; +}; + +export type IndexedSkill = { + name: string; + source: string; + canonicalFilePath: string; +}; + +export class SkillIndex { + readonly #skillsByPath = new Map(); + + public constructor(skills: IndexedSkill[]) { + for (const skill of skills) { + this.#skillsByPath.set(canonicalPathSync(skill.canonicalFilePath), { + ...skill, + canonicalFilePath: canonicalPathSync(skill.canonicalFilePath), + }); + } + } + + public find(canonicalFilePath: string): IndexedSkill | undefined { + return this.#skillsByPath.get(canonicalFilePath); + } +} + +export type SkillCatalogOptions = { + cwd: string; + agentDir: string; + extraDirsValue?: string; +}; + +export class SkillCatalog { + readonly #cwd: string; + readonly #agentDir: string; + readonly #extraDirsValue: string | undefined; + #entries: SkillCatalogEntry[] = []; + #indexedSkills: IndexedSkill[] = []; + #diagnostics: SkillDiagnostic[] = []; + + public constructor(options: SkillCatalogOptions) { + this.#cwd = canonicalPathSync(options.cwd); + this.#agentDir = canonicalPathSync(options.agentDir); + this.#extraDirsValue = options.extraDirsValue; + } + + public async reload(): Promise { + const { paths: additionalSkillPaths, diagnostics: pathDiagnostics } = + await resolveExtraSkillPaths(this.#extraDirsValue); + const loader = new DefaultResourceLoader({ + cwd: this.#cwd, + agentDir: this.#agentDir, + settingsManager: SettingsManager.inMemory(), + additionalSkillPaths, + noExtensions: true, + noPromptTemplates: true, + noThemes: true, + noContextFiles: true, + }); + + await loader.reload(); + const loaded = loader.getSkills(); + const loaderDiagnostics = loaded.diagnostics.map((diagnostic) => + mapDiagnostic(diagnostic, this.#cwd), + ); + this.#diagnostics = [...pathDiagnostics, ...loaderDiagnostics]; + this.#indexedSkills = loaded.skills.map((skill) => ({ + name: skill.name, + source: displaySource(skill), + canonicalFilePath: canonicalPathSync(skill.filePath), + })); + this.#entries = loaded.skills + .map((skill) => this.#toEntry(skill, loaded.diagnostics)) + .sort((left, right) => left.name.localeCompare(right.name)); + } + + public list(): SkillCatalogEntry[] { + return this.#entries.map((entry) => ({ + ...entry, + diagnostics: entry.diagnostics.map((diagnostic) => ({ ...diagnostic })), + })); + } + + public index(): SkillIndex { + return new SkillIndex(this.#indexedSkills); + } + + public readonly diagnostics = (): SkillDiagnostic[] => { + return this.#diagnostics.map((diagnostic) => ({ ...diagnostic })); + }; + + #toEntry(skill: Skill, diagnostics: ResourceDiagnostic[]): SkillCatalogEntry { + const canonicalFilePath = canonicalPathSync(skill.filePath); + return { + name: skill.name, + description: skill.description, + source: displaySource(skill), + filePath: safeDisplayPath(canonicalFilePath, this.#cwd), + diagnostics: diagnostics + .filter( + (diagnostic) => + diagnostic.path !== undefined && + canonicalPathSync(diagnostic.path) === canonicalFilePath, + ) + .map((diagnostic) => mapDiagnostic(diagnostic, this.#cwd)), + }; + } +} + +async function resolveExtraSkillPaths(value?: string): Promise<{ + paths: string[]; + diagnostics: SkillDiagnostic[]; +}> { + if (value === undefined || value.trim() === "") { + return { paths: [], diagnostics: [] }; + } + + const paths: string[] = []; + const diagnostics: SkillDiagnostic[] = []; + const seen = new Set(); + for (const rawPath of value.split(delimiter).map((entry) => entry.trim()).filter(Boolean)) { + const resolved = resolve(rawPath); + try { + await access(resolved); + const pathStat = await stat(resolved); + if (!pathStat.isDirectory()) { + diagnostics.push({ level: "error", message: `Extra skill root is not a directory: ${resolved}` }); + continue; + } + const canonical = await realpath(resolved); + if (!seen.has(canonical)) { + seen.add(canonical); + paths.push(canonical); + } + } catch { + diagnostics.push({ level: "error", message: `Extra skill root is unavailable: ${resolved}` }); + } + } + return { paths, diagnostics }; +} + +function canonicalPathSync(filePath: string): string { + try { + return realpathSync(filePath); + } catch { + return resolve(filePath); + } +} + +function displaySource(skill: Skill): string { + return skill.sourceInfo.scope === "temporary" ? skill.sourceInfo.source : skill.sourceInfo.scope; +} + +function safeDisplayPath(filePath: string, cwd: string): string { + const relativePath = relative(cwd, filePath); + if (relativePath !== "" && !relativePath.startsWith(`..${sep}`) && relativePath !== ".." && !isAbsolute(relativePath)) { + return relativePath; + } + return `external/${skillBasename(filePath)}/SKILL.md`; +} + +function skillBasename(filePath: string): string { + const segments = filePath.split(sep).filter(Boolean); + return segments.at(-2) ?? "skill"; +} + +function mapDiagnostic(diagnostic: ResourceDiagnostic, cwd: string): SkillDiagnostic { + return { + level: diagnostic.type === "error" ? "error" : "warning", + message: diagnostic.message, + ...(diagnostic.path === undefined + ? {} + : { filePath: safeDisplayPath(canonicalPathSync(diagnostic.path), cwd) }), + }; +} diff --git a/packages/harness/src/skills/skill-event-inference.ts b/packages/harness/src/skills/skill-event-inference.ts new file mode 100644 index 0000000..cb2fa43 --- /dev/null +++ b/packages/harness/src/skills/skill-event-inference.ts @@ -0,0 +1,55 @@ +import { realpath } from "node:fs/promises"; +import type { EventStore } from "../events/event-store.js"; +import type { IndexedSkill, SkillIndex } from "./skill-catalog.js"; + +export type SkillEventInferenceOptions = { + sessionId: string; + runId: string; + eventStore: EventStore; + skillIndex: SkillIndex; +}; + +export class SkillEventInference { + readonly #sessionId: string; + readonly #runId: string; + readonly #eventStore: EventStore; + readonly #skillIndex: SkillIndex; + readonly #pending = new Map(); + + public constructor(options: SkillEventInferenceOptions) { + this.#sessionId = options.sessionId; + this.#runId = options.runId; + this.#eventStore = options.eventStore; + this.#skillIndex = options.skillIndex; + } + + public async onReadRequested(input: { toolCallId: string; path: string }): Promise { + let canonicalFilePath: string; + try { + canonicalFilePath = await realpath(input.path); + } catch { + return; + } + const skill = this.#skillIndex.find(canonicalFilePath); + if (skill === undefined) return; + + this.#pending.set(input.toolCallId, skill); + this.#eventStore.append(this.#sessionId, { + type: "skill.requested", + runId: this.#runId, + payload: { name: skill.name, source: skill.source }, + }); + } + + public onReadCompleted(input: { toolCallId: string; success: boolean }): void { + const skill = this.#pending.get(input.toolCallId); + this.#pending.delete(input.toolCallId); + if (skill === undefined || !input.success) return; + + this.#eventStore.append(this.#sessionId, { + type: "skill.loaded", + runId: this.#runId, + payload: { name: skill.name, source: skill.source }, + }); + } +} diff --git a/packages/harness/tests/skills.test.ts b/packages/harness/tests/skills.test.ts new file mode 100644 index 0000000..619c840 --- /dev/null +++ b/packages/harness/tests/skills.test.ts @@ -0,0 +1,162 @@ +import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import * as harness from "../src/index.js"; + +type CatalogEntry = { + name: string; + description: string; + source: string; + filePath: string; + diagnostics: Array<{ level: string; message: string }>; +}; + +type Catalog = { + reload(): Promise; + list(): CatalogEntry[]; + index(): unknown; +}; + +type CatalogConstructor = new (options: { + cwd: string; + agentDir: string; + extraDirsValue?: string; +}) => Catalog; + +let root = ""; +let agentDir = ""; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "agent-studio-skills-")); + agentDir = join(root, ".runtime"); + await mkdir(agentDir, { recursive: true }); +}); + +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +async function createSkill(relativeRoot: string, name: string, description = `Use when working with ${name}`): Promise { + const directory = join(root, relativeRoot, name); + await mkdir(directory, { recursive: true }); + const filePath = join(directory, "SKILL.md"); + await writeFile(filePath, `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n\nFull secret skill body.\n`, "utf8"); + return filePath; +} + +function catalogConstructor(): CatalogConstructor { + const Constructor = Reflect.get(harness, "SkillCatalog") as CatalogConstructor | undefined; + expect(Constructor, "SkillCatalog must be exported").toBeDefined(); + return Constructor!; +} + +describe("SkillCatalog", () => { + it("discovers .agents, .pi, and delimiter-separated extra roots through Pi", async () => { + await createSkill(".agents/skills", "ads-analysis"); + await createSkill(".pi/skills", "report-writer"); + const extraA = join(root, "extra-a"); + const extraB = join(root, "extra-b"); + await createSkill("extra-a", "extra-one"); + await createSkill("extra-b", "extra-two"); + + const catalog = new (catalogConstructor())({ + cwd: root, + agentDir, + extraDirsValue: `${extraA}${delimiter}${extraB}${delimiter}${extraA}`, + }); + await catalog.reload(); + + expect(catalog.list().map((skill) => skill.name)).toEqual([ + "ads-analysis", + "extra-one", + "extra-two", + "report-writer", + ]); + }); + + it("returns only safe catalog metadata and never Skill bodies", async () => { + await createSkill(".agents/skills", "ads-analysis", "Use when analyzing mock ad metrics"); + const catalog = new (catalogConstructor())({ cwd: root, agentDir }); + await catalog.reload(); + + const serialized = JSON.stringify(catalog.list()); + expect(serialized).toContain("Use when analyzing mock ad metrics"); + expect(serialized).not.toContain("Full secret skill body"); + expect(catalog.list()[0]?.filePath).toBe(".agents/skills/ads-analysis/SKILL.md"); + expect(catalog.list()[0]?.filePath.startsWith("/")).toBe(false); + }); + + it("keeps running and reports diagnostics for invalid and missing roots", async () => { + const invalidDir = join(root, ".agents/skills/invalid"); + await mkdir(invalidDir, { recursive: true }); + await writeFile(join(invalidDir, "SKILL.md"), "# missing frontmatter", "utf8"); + const missing = join(root, "missing-extra"); + const catalog = new (catalogConstructor())({ cwd: root, agentDir, extraDirsValue: missing }); + + await expect(catalog.reload()).resolves.toBeUndefined(); + expect(catalog.list().flatMap((skill) => skill.diagnostics).length).toBeGreaterThanOrEqual(0); + const diagnostics = Reflect.get(catalog, "diagnostics") as (() => Array<{ level: string; message: string }>) | undefined; + expect(diagnostics?.().some((entry) => entry.level === "error" && entry.message.includes("missing-extra"))).toBe(true); + await expect(access(agentDir)).resolves.toBeUndefined(); + }); +}); + +describe("SkillEventInference", () => { + it("emits requested and loaded only for a successful canonical Skill read", async () => { + const skillPath = await createSkill(".agents/skills", "ads-analysis"); + const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => { + listAfter(sessionId: string): { events: Array<{ type: string }> }; + }; + const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown; + const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record) => { + onReadRequested(input: { toolCallId: string; path: string }): Promise; + onReadCompleted(input: { toolCallId: string; success: boolean }): void; + }; + expect(Index, "SkillIndex must be exported").toBeDefined(); + expect(Inference, "SkillEventInference must be exported").toBeDefined(); + const store = new storeConstructor(); + const inference = new Inference({ + sessionId: "ses_1", + runId: "run_1", + eventStore: store, + skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]), + }); + + await inference.onReadRequested({ toolCallId: "call_1", path: skillPath }); + inference.onReadCompleted({ toolCallId: "call_1", success: true }); + expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual([ + "skill.requested", + "skill.loaded", + ]); + }); + + it("does not emit loaded after a failed read or for a same-named unrelated file", async () => { + const skillPath = await createSkill(".agents/skills", "ads-analysis"); + const unrelated = join(root, "other", "SKILL.md"); + await mkdir(join(root, "other")); + await writeFile(unrelated, "unrelated", "utf8"); + const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => { + listAfter(sessionId: string): { events: Array<{ type: string }> }; + }; + const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown; + const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record) => { + onReadRequested(input: { toolCallId: string; path: string }): Promise; + onReadCompleted(input: { toolCallId: string; success: boolean }): void; + }; + const store = new storeConstructor(); + const inference = new Inference({ + sessionId: "ses_1", + runId: "run_1", + eventStore: store, + skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]), + }); + + await inference.onReadRequested({ toolCallId: "call_1", path: skillPath }); + inference.onReadCompleted({ toolCallId: "call_1", success: false }); + await inference.onReadRequested({ toolCallId: "call_2", path: unrelated }); + inference.onReadCompleted({ toolCallId: "call_2", success: true }); + + expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual(["skill.requested"]); + }); +});