218 lines
10 KiB
TypeScript
218 lines
10 KiB
TypeScript
import { access, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { delimiter, join } from "node:path";
|
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
import * as harness from "../src/index.js";
|
|
|
|
type CatalogEntry = {
|
|
name: string;
|
|
description: string;
|
|
source: string;
|
|
filePath: string;
|
|
diagnostics: Array<{ level: string; message: string }>;
|
|
};
|
|
|
|
type Catalog = {
|
|
reload(): Promise<void>;
|
|
list(): CatalogEntry[];
|
|
index(): unknown;
|
|
};
|
|
|
|
type CatalogConstructor = new (options: {
|
|
cwd: string;
|
|
agentDir: string;
|
|
extraDirsValue?: string;
|
|
}) => Catalog;
|
|
|
|
let root = "";
|
|
let agentDir = "";
|
|
|
|
beforeEach(async () => {
|
|
root = await mkdtemp(join(tmpdir(), "agent-studio-skills-"));
|
|
agentDir = join(root, ".runtime");
|
|
await mkdir(agentDir, { recursive: true });
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await rm(root, { recursive: true, force: true });
|
|
});
|
|
|
|
async function createSkill(relativeRoot: string, name: string, description = `Use when working with ${name}`): Promise<string> {
|
|
const directory = join(root, relativeRoot, name);
|
|
await mkdir(directory, { recursive: true });
|
|
const filePath = join(directory, "SKILL.md");
|
|
await writeFile(filePath, `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n\nFull secret skill body.\n`, "utf8");
|
|
return filePath;
|
|
}
|
|
|
|
function catalogConstructor(): CatalogConstructor {
|
|
const Constructor = Reflect.get(harness, "SkillCatalog") as CatalogConstructor | undefined;
|
|
expect(Constructor, "SkillCatalog must be exported").toBeDefined();
|
|
return Constructor!;
|
|
}
|
|
|
|
describe("SkillCatalog", () => {
|
|
it("keeps the editor's new content-review reference readable in the isolated Skill snapshot", async () => {
|
|
const original = await createSkill(".agents/skills", "swads-short-drama-editor");
|
|
const baseDir = join(root, ".agents/skills/swads-short-drama-editor");
|
|
await mkdir(join(baseDir, "references"));
|
|
await mkdir(join(baseDir, "scripts"));
|
|
for (const resource of harness.EDITOR_RESOURCES) await writeFile(join(baseDir, resource), "fixture resource");
|
|
await writeFile(join(baseDir, "references/tiktok-adult-content-policy.md"), "fixture selection and final review gate");
|
|
const catalog = new harness.SkillCatalog({ cwd: root, agentDir, resources: { [original]: [...harness.EDITOR_RESOURCES] } });
|
|
await catalog.reload();
|
|
const sessionRoot = join(root, "session");
|
|
await mkdir(sessionRoot);
|
|
const snapshot = await catalog.createSessionSnapshot(sessionRoot);
|
|
const editor = snapshot.skills.find((skill) => skill.name === "swads-short-drama-editor")!;
|
|
const workspace = await harness.SafeWorkspace.create(sessionRoot);
|
|
const policy = await workspace.resolveReadable(join(editor.baseDir, "references/tiktok-adult-content-policy.md"));
|
|
expect(await readFile(policy, "utf8")).toBe("fixture selection and final review gate");
|
|
});
|
|
|
|
it("copies only approved resources with relative links intact and rejects escaped resources", async () => {
|
|
const original = await createSkill(".agents/skills", "diagnosis");
|
|
const baseDir = join(root, ".agents/skills/diagnosis");
|
|
await mkdir(join(baseDir, "references"));
|
|
await writeFile(join(baseDir, "references/metrics.md"), "metric contract");
|
|
await writeFile(join(baseDir, ".env"), "do not copy");
|
|
const catalog = new harness.SkillCatalog({ cwd: root, agentDir,
|
|
resources: { [original]: ["references/metrics.md"] },
|
|
});
|
|
await catalog.reload();
|
|
const sessionRoot = join(root, "session");
|
|
await mkdir(sessionRoot);
|
|
const snapshot = await catalog.createSessionSnapshot(sessionRoot);
|
|
const copied = snapshot.skills.find((skill) => skill.name === "diagnosis")!;
|
|
expect(await readFile(join(copied.baseDir, "references/metrics.md"), "utf8")).toBe("metric contract");
|
|
await expect(access(join(copied.baseDir, ".env"))).rejects.toThrow();
|
|
await rm(join(baseDir, "references/metrics.md"));
|
|
await symlink(join(baseDir, ".env"), join(baseDir, "references/metrics.md"));
|
|
await expect(catalog.createSessionSnapshot(sessionRoot)).rejects.toThrow();
|
|
});
|
|
|
|
it("makes discovered Skill bodies readable inside the Session without admitting project paths", async () => {
|
|
const original = await createSkill(".agents/skills", "ads-analysis");
|
|
const sessionRoot = join(root, "session");
|
|
await mkdir(sessionRoot);
|
|
const catalog = new harness.SkillCatalog({ cwd: root, agentDir });
|
|
await catalog.reload();
|
|
const snapshot = await catalog.createSessionSnapshot(sessionRoot);
|
|
const workspace = await harness.SafeWorkspace.create(sessionRoot);
|
|
const copied = snapshot.skills.find((skill) => skill.name === "ads-analysis")!;
|
|
await expect(workspace.resolveReadable(copied.filePath)).resolves.toBe(copied.filePath);
|
|
expect(await readFile(copied.filePath, "utf8")).toContain("Full secret skill body.");
|
|
expect(snapshot.index.find(copied.filePath)?.name).toBe("ads-analysis");
|
|
expect(snapshot.index.find(original)).toBeUndefined();
|
|
await expect(workspace.resolveReadable(original)).rejects.toThrow();
|
|
});
|
|
|
|
it("discovers .agents, .pi, and delimiter-separated extra roots through Pi", async () => {
|
|
await createSkill(".agents/skills", "ads-analysis");
|
|
await createSkill(".pi/skills", "report-writer");
|
|
const extraA = join(root, "extra-a");
|
|
const extraB = join(root, "extra-b");
|
|
await createSkill("extra-a", "extra-one");
|
|
await createSkill("extra-b", "extra-two");
|
|
|
|
const catalog = new (catalogConstructor())({
|
|
cwd: root,
|
|
agentDir,
|
|
extraDirsValue: `${extraA}${delimiter}${extraB}${delimiter}${extraA}`,
|
|
});
|
|
await catalog.reload();
|
|
|
|
expect(catalog.list().map((skill) => skill.name)).toEqual([
|
|
"ads-analysis",
|
|
"extra-one",
|
|
"extra-two",
|
|
"report-writer",
|
|
]);
|
|
});
|
|
|
|
it("returns only safe catalog metadata and never Skill bodies", async () => {
|
|
await createSkill(".agents/skills", "ads-analysis", "Use when analyzing mock ad metrics");
|
|
const catalog = new (catalogConstructor())({ cwd: root, agentDir });
|
|
await catalog.reload();
|
|
|
|
const serialized = JSON.stringify(catalog.list());
|
|
expect(serialized).toContain("Use when analyzing mock ad metrics");
|
|
expect(serialized).not.toContain("Full secret skill body");
|
|
expect(catalog.list()[0]?.filePath).toBe(".agents/skills/ads-analysis/SKILL.md");
|
|
expect(catalog.list()[0]?.filePath.startsWith("/")).toBe(false);
|
|
});
|
|
|
|
it("keeps running and reports diagnostics for invalid and missing roots", async () => {
|
|
const invalidDir = join(root, ".agents/skills/invalid");
|
|
await mkdir(invalidDir, { recursive: true });
|
|
await writeFile(join(invalidDir, "SKILL.md"), "# missing frontmatter", "utf8");
|
|
const missing = join(root, "missing-extra");
|
|
const catalog = new (catalogConstructor())({ cwd: root, agentDir, extraDirsValue: missing });
|
|
|
|
await expect(catalog.reload()).resolves.toBeUndefined();
|
|
expect(catalog.list().flatMap((skill) => skill.diagnostics).length).toBeGreaterThanOrEqual(0);
|
|
const diagnostics = Reflect.get(catalog, "diagnostics") as (() => Array<{ level: string; message: string }>) | undefined;
|
|
expect(diagnostics?.().some((entry) => entry.level === "error" && entry.message.includes("missing-extra"))).toBe(true);
|
|
await expect(access(agentDir)).resolves.toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe("SkillEventInference", () => {
|
|
it("emits requested and loaded only for a successful canonical Skill read", async () => {
|
|
const skillPath = await createSkill(".agents/skills", "ads-analysis");
|
|
const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
|
|
listAfter(sessionId: string): { events: Array<{ type: string }> };
|
|
};
|
|
const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
|
|
const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
|
|
onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
|
|
onReadCompleted(input: { toolCallId: string; success: boolean }): void;
|
|
};
|
|
expect(Index, "SkillIndex must be exported").toBeDefined();
|
|
expect(Inference, "SkillEventInference must be exported").toBeDefined();
|
|
const store = new storeConstructor();
|
|
const inference = new Inference({
|
|
sessionId: "ses_1",
|
|
runId: "run_1",
|
|
eventStore: store,
|
|
skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
|
|
});
|
|
|
|
await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
|
|
inference.onReadCompleted({ toolCallId: "call_1", success: true });
|
|
expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual([
|
|
"skill.requested",
|
|
"skill.loaded",
|
|
]);
|
|
});
|
|
|
|
it("does not emit loaded after a failed read or for a same-named unrelated file", async () => {
|
|
const skillPath = await createSkill(".agents/skills", "ads-analysis");
|
|
const unrelated = join(root, "other", "SKILL.md");
|
|
await mkdir(join(root, "other"));
|
|
await writeFile(unrelated, "unrelated", "utf8");
|
|
const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
|
|
listAfter(sessionId: string): { events: Array<{ type: string }> };
|
|
};
|
|
const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
|
|
const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
|
|
onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
|
|
onReadCompleted(input: { toolCallId: string; success: boolean }): void;
|
|
};
|
|
const store = new storeConstructor();
|
|
const inference = new Inference({
|
|
sessionId: "ses_1",
|
|
runId: "run_1",
|
|
eventStore: store,
|
|
skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
|
|
});
|
|
|
|
await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
|
|
inference.onReadCompleted({ toolCallId: "call_1", success: false });
|
|
await inference.onReadRequested({ toolCallId: "call_2", path: unrelated });
|
|
inference.onReadCompleted({ toolCallId: "call_2", success: true });
|
|
|
|
expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual(["skill.requested"]);
|
|
});
|
|
});
|