feat(harness): discover skills progressively

2026-09-07 15:43:39 +08:00
parent d229ac0f0d
commit ec0686cd56
6 changed files with 456 additions and 0 deletions
@@ -0,0 +1,162 @@
+import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { delimiter, join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import * as harness from "../src/index.js";
+
+type CatalogEntry = {
+  name: string;
+  description: string;
+  source: string;
+  filePath: string;
+  diagnostics: Array<{ level: string; message: string }>;
+};
+
+type Catalog = {
+  reload(): Promise<void>;
+  list(): CatalogEntry[];
+  index(): unknown;
+};
+
+type CatalogConstructor = new (options: {
+  cwd: string;
+  agentDir: string;
+  extraDirsValue?: string;
+}) => Catalog;
+
+let root = "";
+let agentDir = "";
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), "agent-studio-skills-"));
+  agentDir = join(root, ".runtime");
+  await mkdir(agentDir, { recursive: true });
+});
+
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true });
+});
+
+async function createSkill(relativeRoot: string, name: string, description = `Use when working with ${name}`): Promise<string> {
+  const directory = join(root, relativeRoot, name);
+  await mkdir(directory, { recursive: true });
+  const filePath = join(directory, "SKILL.md");
+  await writeFile(filePath, `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n\nFull secret skill body.\n`, "utf8");
+  return filePath;
+}
+
+function catalogConstructor(): CatalogConstructor {
+  const Constructor = Reflect.get(harness, "SkillCatalog") as CatalogConstructor | undefined;
+  expect(Constructor, "SkillCatalog must be exported").toBeDefined();
+  return Constructor!;
+}
+
+describe("SkillCatalog", () => {
+  it("discovers .agents, .pi, and delimiter-separated extra roots through Pi", async () => {
+    await createSkill(".agents/skills", "ads-analysis");
+    await createSkill(".pi/skills", "report-writer");
+    const extraA = join(root, "extra-a");
+    const extraB = join(root, "extra-b");
+    await createSkill("extra-a", "extra-one");
+    await createSkill("extra-b", "extra-two");
+
+    const catalog = new (catalogConstructor())({
+      cwd: root,
+      agentDir,
+      extraDirsValue: `${extraA}${delimiter}${extraB}${delimiter}${extraA}`,
+    });
+    await catalog.reload();
+
+    expect(catalog.list().map((skill) => skill.name)).toEqual([
+      "ads-analysis",
+      "extra-one",
+      "extra-two",
+      "report-writer",
+    ]);
+  });
+
+  it("returns only safe catalog metadata and never Skill bodies", async () => {
+    await createSkill(".agents/skills", "ads-analysis", "Use when analyzing mock ad metrics");
+    const catalog = new (catalogConstructor())({ cwd: root, agentDir });
+    await catalog.reload();
+
+    const serialized = JSON.stringify(catalog.list());
+    expect(serialized).toContain("Use when analyzing mock ad metrics");
+    expect(serialized).not.toContain("Full secret skill body");
+    expect(catalog.list()[0]?.filePath).toBe(".agents/skills/ads-analysis/SKILL.md");
+    expect(catalog.list()[0]?.filePath.startsWith("/")).toBe(false);
+  });
+
+  it("keeps running and reports diagnostics for invalid and missing roots", async () => {
+    const invalidDir = join(root, ".agents/skills/invalid");
+    await mkdir(invalidDir, { recursive: true });
+    await writeFile(join(invalidDir, "SKILL.md"), "# missing frontmatter", "utf8");
+    const missing = join(root, "missing-extra");
+    const catalog = new (catalogConstructor())({ cwd: root, agentDir, extraDirsValue: missing });
+
+    await expect(catalog.reload()).resolves.toBeUndefined();
+    expect(catalog.list().flatMap((skill) => skill.diagnostics).length).toBeGreaterThanOrEqual(0);
+    const diagnostics = Reflect.get(catalog, "diagnostics") as (() => Array<{ level: string; message: string }>) | undefined;
+    expect(diagnostics?.().some((entry) => entry.level === "error" && entry.message.includes("missing-extra"))).toBe(true);
+    await expect(access(agentDir)).resolves.toBeUndefined();
+  });
+});
+
+describe("SkillEventInference", () => {
+  it("emits requested and loaded only for a successful canonical Skill read", async () => {
+    const skillPath = await createSkill(".agents/skills", "ads-analysis");
+    const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
+      listAfter(sessionId: string): { events: Array<{ type: string }> };
+    };
+    const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
+    const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
+      onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
+      onReadCompleted(input: { toolCallId: string; success: boolean }): void;
+    };
+    expect(Index, "SkillIndex must be exported").toBeDefined();
+    expect(Inference, "SkillEventInference must be exported").toBeDefined();
+    const store = new storeConstructor();
+    const inference = new Inference({
+      sessionId: "ses_1",
+      runId: "run_1",
+      eventStore: store,
+      skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
+    });
+
+    await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
+    inference.onReadCompleted({ toolCallId: "call_1", success: true });
+    expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual([
+      "skill.requested",
+      "skill.loaded",
+    ]);
+  });
+
+  it("does not emit loaded after a failed read or for a same-named unrelated file", async () => {
+    const skillPath = await createSkill(".agents/skills", "ads-analysis");
+    const unrelated = join(root, "other", "SKILL.md");
+    await mkdir(join(root, "other"));
+    await writeFile(unrelated, "unrelated", "utf8");
+    const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
+      listAfter(sessionId: string): { events: Array<{ type: string }> };
+    };
+    const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
+    const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
+      onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
+      onReadCompleted(input: { toolCallId: string; success: boolean }): void;
+    };
+    const store = new storeConstructor();
+    const inference = new Inference({
+      sessionId: "ses_1",
+      runId: "run_1",
+      eventStore: store,
+      skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
+    });
+
+    await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
+    inference.onReadCompleted({ toolCallId: "call_1", success: false });
+    await inference.onReadRequested({ toolCallId: "call_2", path: unrelated });
+    inference.onReadCompleted({ toolCallId: "call_2", success: true });
+
+    expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual(["skill.requested"]);
+  });
+});