feat(harness): discover skills progressively

.agents/skills/ads-analysis/SKILL.md

@@ -0,0 +1,21 @@
+---
+name: ads-analysis
+description: Use when analyzing the bundled mock advertising account, identifying campaign anomalies, or recommending budget and creative actions from mock ad metrics.
+---
+
+# Ads Analysis
+
+Call `mock_ads_metrics` before calculating, comparing, or explaining any account or campaign metric.
+
+Treat every returned value as fixture data. Never describe it as live, connected, or sourced from a real advertising platform. State the available date range when presenting results.
+
+Base all numeric claims on the tool response. Images may provide supporting creative context, but never use an image to invent or override spend, impressions, clicks, conversions, or revenue.
+
+Analyze the account in this order:
+
+1. Summarize spend, impressions, clicks, conversions, and revenue.
+2. Compare campaigns using derived efficiency metrics such as CTR, CPC, CPA, CVR, and ROAS when their denominators are non-zero.
+3. Flag meaningful outliers and explain the evidence for each flag.
+4. Recommend concrete budget, targeting, or creative actions, separating evidence from inference.
+
+If the requested range is unavailable, explain the limitation instead of extrapolating beyond the fixture.

.pi/skills/report-writer/SKILL.md

@@ -0,0 +1,20 @@
+---
+name: report-writer
+description: Use when turning completed advertising analysis into a concise Chinese report and saving the approved final draft to the project workspace.
+---
+
+# Report Writer
+
+Write the report in Chinese with these sections:
+
+1. 账户总结
+2. 关键指标
+3. 异常 Campaign
+4. 优化建议
+5. 数据时间范围
+
+Keep facts traceable to the available analysis. Label uncertain explanations as inference, keep recommendations actionable, and do not imply that fixture data is live platform data.
+
+After composing the complete report, call `save_report_draft` with the full Markdown draft. This action requires approval; do not claim that the report was saved until the tool succeeds.
+
+If approval is denied, expires, or the tool fails, still provide the complete report text in the conversation and clearly state that the draft was not saved.

packages/harness/src/index.ts

@@ -1,6 +1,8 @@
 export * from "./events/event-store.js";
 export * from "./events/in-memory-event-store.js";
 export * from "./errors/harness-error.js";
+export * from "./skills/skill-catalog.js";
+export * from "./skills/skill-event-inference.js";
 export * from "./tools/mock-ads-metrics.js";
 export * from "./tools/save-report-draft.js";
 export * from "./tools/tool-policy.js";

packages/harness/src/skills/skill-catalog.ts

@@ -0,0 +1,196 @@
+import { access, realpath, stat } from "node:fs/promises";
+import { realpathSync } from "node:fs";
+import { delimiter, isAbsolute, relative, resolve, sep } from "node:path";
+import {
+  DefaultResourceLoader,
+  SettingsManager,
+  type ResourceDiagnostic,
+  type Skill,
+} from "@earendil-works/pi-coding-agent";
+
+export type SkillDiagnostic = {
+  level: "warning" | "error";
+  message: string;
+  filePath?: string;
+};
+
+export type SkillCatalogEntry = {
+  name: string;
+  description: string;
+  source: string;
+  filePath: string;
+  diagnostics: SkillDiagnostic[];
+};
+
+export type IndexedSkill = {
+  name: string;
+  source: string;
+  canonicalFilePath: string;
+};
+
+export class SkillIndex {
+  readonly #skillsByPath = new Map<string, IndexedSkill>();
+
+  public constructor(skills: IndexedSkill[]) {
+    for (const skill of skills) {
+      this.#skillsByPath.set(canonicalPathSync(skill.canonicalFilePath), {
+        ...skill,
+        canonicalFilePath: canonicalPathSync(skill.canonicalFilePath),
+      });
+    }
+  }
+
+  public find(canonicalFilePath: string): IndexedSkill | undefined {
+    return this.#skillsByPath.get(canonicalFilePath);
+  }
+}
+
+export type SkillCatalogOptions = {
+  cwd: string;
+  agentDir: string;
+  extraDirsValue?: string;
+};
+
+export class SkillCatalog {
+  readonly #cwd: string;
+  readonly #agentDir: string;
+  readonly #extraDirsValue: string | undefined;
+  #entries: SkillCatalogEntry[] = [];
+  #indexedSkills: IndexedSkill[] = [];
+  #diagnostics: SkillDiagnostic[] = [];
+
+  public constructor(options: SkillCatalogOptions) {
+    this.#cwd = canonicalPathSync(options.cwd);
+    this.#agentDir = canonicalPathSync(options.agentDir);
+    this.#extraDirsValue = options.extraDirsValue;
+  }
+
+  public async reload(): Promise<void> {
+    const { paths: additionalSkillPaths, diagnostics: pathDiagnostics } =
+      await resolveExtraSkillPaths(this.#extraDirsValue);
+    const loader = new DefaultResourceLoader({
+      cwd: this.#cwd,
+      agentDir: this.#agentDir,
+      settingsManager: SettingsManager.inMemory(),
+      additionalSkillPaths,
+      noExtensions: true,
+      noPromptTemplates: true,
+      noThemes: true,
+      noContextFiles: true,
+    });
+
+    await loader.reload();
+    const loaded = loader.getSkills();
+    const loaderDiagnostics = loaded.diagnostics.map((diagnostic) =>
+      mapDiagnostic(diagnostic, this.#cwd),
+    );
+    this.#diagnostics = [...pathDiagnostics, ...loaderDiagnostics];
+    this.#indexedSkills = loaded.skills.map((skill) => ({
+      name: skill.name,
+      source: displaySource(skill),
+      canonicalFilePath: canonicalPathSync(skill.filePath),
+    }));
+    this.#entries = loaded.skills
+      .map((skill) => this.#toEntry(skill, loaded.diagnostics))
+      .sort((left, right) => left.name.localeCompare(right.name));
+  }
+
+  public list(): SkillCatalogEntry[] {
+    return this.#entries.map((entry) => ({
+      ...entry,
+      diagnostics: entry.diagnostics.map((diagnostic) => ({ ...diagnostic })),
+    }));
+  }
+
+  public index(): SkillIndex {
+    return new SkillIndex(this.#indexedSkills);
+  }
+
+  public readonly diagnostics = (): SkillDiagnostic[] => {
+    return this.#diagnostics.map((diagnostic) => ({ ...diagnostic }));
+  };
+
+  #toEntry(skill: Skill, diagnostics: ResourceDiagnostic[]): SkillCatalogEntry {
+    const canonicalFilePath = canonicalPathSync(skill.filePath);
+    return {
+      name: skill.name,
+      description: skill.description,
+      source: displaySource(skill),
+      filePath: safeDisplayPath(canonicalFilePath, this.#cwd),
+      diagnostics: diagnostics
+        .filter(
+          (diagnostic) =>
+            diagnostic.path !== undefined &&
+            canonicalPathSync(diagnostic.path) === canonicalFilePath,
+        )
+        .map((diagnostic) => mapDiagnostic(diagnostic, this.#cwd)),
+    };
+  }
+}
+
+async function resolveExtraSkillPaths(value?: string): Promise<{
+  paths: string[];
+  diagnostics: SkillDiagnostic[];
+}> {
+  if (value === undefined || value.trim() === "") {
+    return { paths: [], diagnostics: [] };
+  }
+
+  const paths: string[] = [];
+  const diagnostics: SkillDiagnostic[] = [];
+  const seen = new Set<string>();
+  for (const rawPath of value.split(delimiter).map((entry) => entry.trim()).filter(Boolean)) {
+    const resolved = resolve(rawPath);
+    try {
+      await access(resolved);
+      const pathStat = await stat(resolved);
+      if (!pathStat.isDirectory()) {
+        diagnostics.push({ level: "error", message: `Extra skill root is not a directory: ${resolved}` });
+        continue;
+      }
+      const canonical = await realpath(resolved);
+      if (!seen.has(canonical)) {
+        seen.add(canonical);
+        paths.push(canonical);
+      }
+    } catch {
+      diagnostics.push({ level: "error", message: `Extra skill root is unavailable: ${resolved}` });
+    }
+  }
+  return { paths, diagnostics };
+}
+
+function canonicalPathSync(filePath: string): string {
+  try {
+    return realpathSync(filePath);
+  } catch {
+    return resolve(filePath);
+  }
+}
+
+function displaySource(skill: Skill): string {
+  return skill.sourceInfo.scope === "temporary" ? skill.sourceInfo.source : skill.sourceInfo.scope;
+}
+
+function safeDisplayPath(filePath: string, cwd: string): string {
+  const relativePath = relative(cwd, filePath);
+  if (relativePath !== "" && !relativePath.startsWith(`..${sep}`) && relativePath !== ".." && !isAbsolute(relativePath)) {
+    return relativePath;
+  }
+  return `external/${skillBasename(filePath)}/SKILL.md`;
+}
+
+function skillBasename(filePath: string): string {
+  const segments = filePath.split(sep).filter(Boolean);
+  return segments.at(-2) ?? "skill";
+}
+
+function mapDiagnostic(diagnostic: ResourceDiagnostic, cwd: string): SkillDiagnostic {
+  return {
+    level: diagnostic.type === "error" ? "error" : "warning",
+    message: diagnostic.message,
+    ...(diagnostic.path === undefined
+      ? {}
+      : { filePath: safeDisplayPath(canonicalPathSync(diagnostic.path), cwd) }),
+  };
+}

packages/harness/src/skills/skill-event-inference.ts

@@ -0,0 +1,55 @@
+import { realpath } from "node:fs/promises";
+import type { EventStore } from "../events/event-store.js";
+import type { IndexedSkill, SkillIndex } from "./skill-catalog.js";
+
+export type SkillEventInferenceOptions = {
+  sessionId: string;
+  runId: string;
+  eventStore: EventStore;
+  skillIndex: SkillIndex;
+};
+
+export class SkillEventInference {
+  readonly #sessionId: string;
+  readonly #runId: string;
+  readonly #eventStore: EventStore;
+  readonly #skillIndex: SkillIndex;
+  readonly #pending = new Map<string, IndexedSkill>();
+
+  public constructor(options: SkillEventInferenceOptions) {
+    this.#sessionId = options.sessionId;
+    this.#runId = options.runId;
+    this.#eventStore = options.eventStore;
+    this.#skillIndex = options.skillIndex;
+  }
+
+  public async onReadRequested(input: { toolCallId: string; path: string }): Promise<void> {
+    let canonicalFilePath: string;
+    try {
+      canonicalFilePath = await realpath(input.path);
+    } catch {
+      return;
+    }
+    const skill = this.#skillIndex.find(canonicalFilePath);
+    if (skill === undefined) return;
+
+    this.#pending.set(input.toolCallId, skill);
+    this.#eventStore.append(this.#sessionId, {
+      type: "skill.requested",
+      runId: this.#runId,
+      payload: { name: skill.name, source: skill.source },
+    });
+  }
+
+  public onReadCompleted(input: { toolCallId: string; success: boolean }): void {
+    const skill = this.#pending.get(input.toolCallId);
+    this.#pending.delete(input.toolCallId);
+    if (skill === undefined || !input.success) return;
+
+    this.#eventStore.append(this.#sessionId, {
+      type: "skill.loaded",
+      runId: this.#runId,
+      payload: { name: skill.name, source: skill.source },
+    });
+  }
+}

packages/harness/tests/skills.test.ts

@@ -0,0 +1,162 @@
+import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { delimiter, join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import * as harness from "../src/index.js";
+
+type CatalogEntry = {
+  name: string;
+  description: string;
+  source: string;
+  filePath: string;
+  diagnostics: Array<{ level: string; message: string }>;
+};
+
+type Catalog = {
+  reload(): Promise<void>;
+  list(): CatalogEntry[];
+  index(): unknown;
+};
+
+type CatalogConstructor = new (options: {
+  cwd: string;
+  agentDir: string;
+  extraDirsValue?: string;
+}) => Catalog;
+
+let root = "";
+let agentDir = "";
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), "agent-studio-skills-"));
+  agentDir = join(root, ".runtime");
+  await mkdir(agentDir, { recursive: true });
+});
+
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true });
+});
+
+async function createSkill(relativeRoot: string, name: string, description = `Use when working with ${name}`): Promise<string> {
+  const directory = join(root, relativeRoot, name);
+  await mkdir(directory, { recursive: true });
+  const filePath = join(directory, "SKILL.md");
+  await writeFile(filePath, `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n\nFull secret skill body.\n`, "utf8");
+  return filePath;
+}
+
+function catalogConstructor(): CatalogConstructor {
+  const Constructor = Reflect.get(harness, "SkillCatalog") as CatalogConstructor | undefined;
+  expect(Constructor, "SkillCatalog must be exported").toBeDefined();
+  return Constructor!;
+}
+
+describe("SkillCatalog", () => {
+  it("discovers .agents, .pi, and delimiter-separated extra roots through Pi", async () => {
+    await createSkill(".agents/skills", "ads-analysis");
+    await createSkill(".pi/skills", "report-writer");
+    const extraA = join(root, "extra-a");
+    const extraB = join(root, "extra-b");
+    await createSkill("extra-a", "extra-one");
+    await createSkill("extra-b", "extra-two");
+
+    const catalog = new (catalogConstructor())({
+      cwd: root,
+      agentDir,
+      extraDirsValue: `${extraA}${delimiter}${extraB}${delimiter}${extraA}`,
+    });
+    await catalog.reload();
+
+    expect(catalog.list().map((skill) => skill.name)).toEqual([
+      "ads-analysis",
+      "extra-one",
+      "extra-two",
+      "report-writer",
+    ]);
+  });
+
+  it("returns only safe catalog metadata and never Skill bodies", async () => {
+    await createSkill(".agents/skills", "ads-analysis", "Use when analyzing mock ad metrics");
+    const catalog = new (catalogConstructor())({ cwd: root, agentDir });
+    await catalog.reload();
+
+    const serialized = JSON.stringify(catalog.list());
+    expect(serialized).toContain("Use when analyzing mock ad metrics");
+    expect(serialized).not.toContain("Full secret skill body");
+    expect(catalog.list()[0]?.filePath).toBe(".agents/skills/ads-analysis/SKILL.md");
+    expect(catalog.list()[0]?.filePath.startsWith("/")).toBe(false);
+  });
+
+  it("keeps running and reports diagnostics for invalid and missing roots", async () => {
+    const invalidDir = join(root, ".agents/skills/invalid");
+    await mkdir(invalidDir, { recursive: true });
+    await writeFile(join(invalidDir, "SKILL.md"), "# missing frontmatter", "utf8");
+    const missing = join(root, "missing-extra");
+    const catalog = new (catalogConstructor())({ cwd: root, agentDir, extraDirsValue: missing });
+
+    await expect(catalog.reload()).resolves.toBeUndefined();
+    expect(catalog.list().flatMap((skill) => skill.diagnostics).length).toBeGreaterThanOrEqual(0);
+    const diagnostics = Reflect.get(catalog, "diagnostics") as (() => Array<{ level: string; message: string }>) | undefined;
+    expect(diagnostics?.().some((entry) => entry.level === "error" && entry.message.includes("missing-extra"))).toBe(true);
+    await expect(access(agentDir)).resolves.toBeUndefined();
+  });
+});
+
+describe("SkillEventInference", () => {
+  it("emits requested and loaded only for a successful canonical Skill read", async () => {
+    const skillPath = await createSkill(".agents/skills", "ads-analysis");
+    const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
+      listAfter(sessionId: string): { events: Array<{ type: string }> };
+    };
+    const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
+    const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
+      onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
+      onReadCompleted(input: { toolCallId: string; success: boolean }): void;
+    };
+    expect(Index, "SkillIndex must be exported").toBeDefined();
+    expect(Inference, "SkillEventInference must be exported").toBeDefined();
+    const store = new storeConstructor();
+    const inference = new Inference({
+      sessionId: "ses_1",
+      runId: "run_1",
+      eventStore: store,
+      skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
+    });
+
+    await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
+    inference.onReadCompleted({ toolCallId: "call_1", success: true });
+    expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual([
+      "skill.requested",
+      "skill.loaded",
+    ]);
+  });
+
+  it("does not emit loaded after a failed read or for a same-named unrelated file", async () => {
+    const skillPath = await createSkill(".agents/skills", "ads-analysis");
+    const unrelated = join(root, "other", "SKILL.md");
+    await mkdir(join(root, "other"));
+    await writeFile(unrelated, "unrelated", "utf8");
+    const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
+      listAfter(sessionId: string): { events: Array<{ type: string }> };
+    };
+    const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
+    const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
+      onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
+      onReadCompleted(input: { toolCallId: string; success: boolean }): void;
+    };
+    const store = new storeConstructor();
+    const inference = new Inference({
+      sessionId: "ses_1",
+      runId: "run_1",
+      eventStore: store,
+      skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
+    });
+
+    await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
+    inference.onReadCompleted({ toolCallId: "call_1", success: false });
+    await inference.onReadRequested({ toolCallId: "call_2", path: unrelated });
+    inference.onReadCompleted({ toolCallId: "call_2", success: true });
+
+    expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual(["skill.requested"]);
+  });
+});