feat(harness): discover skills progressively
This commit is contained in:
Wenxin Duan
@@ -1,6 +1,8 @@
|
||||
export * from "./events/event-store.js";
|
||||
export * from "./events/in-memory-event-store.js";
|
||||
export * from "./errors/harness-error.js";
|
||||
export * from "./skills/skill-catalog.js";
|
||||
export * from "./skills/skill-event-inference.js";
|
||||
export * from "./tools/mock-ads-metrics.js";
|
||||
export * from "./tools/save-report-draft.js";
|
||||
export * from "./tools/tool-policy.js";
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
import { access, realpath, stat } from "node:fs/promises";
|
||||
import { realpathSync } from "node:fs";
|
||||
import { delimiter, isAbsolute, relative, resolve, sep } from "node:path";
|
||||
import {
|
||||
DefaultResourceLoader,
|
||||
SettingsManager,
|
||||
type ResourceDiagnostic,
|
||||
type Skill,
|
||||
} from "@earendil-works/pi-coding-agent";
|
||||
|
||||
export type SkillDiagnostic = {
|
||||
level: "warning" | "error";
|
||||
message: string;
|
||||
filePath?: string;
|
||||
};
|
||||
|
||||
export type SkillCatalogEntry = {
|
||||
name: string;
|
||||
description: string;
|
||||
source: string;
|
||||
filePath: string;
|
||||
diagnostics: SkillDiagnostic[];
|
||||
};
|
||||
|
||||
export type IndexedSkill = {
|
||||
name: string;
|
||||
source: string;
|
||||
canonicalFilePath: string;
|
||||
};
|
||||
|
||||
export class SkillIndex {
|
||||
readonly #skillsByPath = new Map<string, IndexedSkill>();
|
||||
|
||||
public constructor(skills: IndexedSkill[]) {
|
||||
for (const skill of skills) {
|
||||
this.#skillsByPath.set(canonicalPathSync(skill.canonicalFilePath), {
|
||||
...skill,
|
||||
canonicalFilePath: canonicalPathSync(skill.canonicalFilePath),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public find(canonicalFilePath: string): IndexedSkill | undefined {
|
||||
return this.#skillsByPath.get(canonicalFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
export type SkillCatalogOptions = {
|
||||
cwd: string;
|
||||
agentDir: string;
|
||||
extraDirsValue?: string;
|
||||
};
|
||||
|
||||
export class SkillCatalog {
|
||||
readonly #cwd: string;
|
||||
readonly #agentDir: string;
|
||||
readonly #extraDirsValue: string | undefined;
|
||||
#entries: SkillCatalogEntry[] = [];
|
||||
#indexedSkills: IndexedSkill[] = [];
|
||||
#diagnostics: SkillDiagnostic[] = [];
|
||||
|
||||
public constructor(options: SkillCatalogOptions) {
|
||||
this.#cwd = canonicalPathSync(options.cwd);
|
||||
this.#agentDir = canonicalPathSync(options.agentDir);
|
||||
this.#extraDirsValue = options.extraDirsValue;
|
||||
}
|
||||
|
||||
public async reload(): Promise<void> {
|
||||
const { paths: additionalSkillPaths, diagnostics: pathDiagnostics } =
|
||||
await resolveExtraSkillPaths(this.#extraDirsValue);
|
||||
const loader = new DefaultResourceLoader({
|
||||
cwd: this.#cwd,
|
||||
agentDir: this.#agentDir,
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
additionalSkillPaths,
|
||||
noExtensions: true,
|
||||
noPromptTemplates: true,
|
||||
noThemes: true,
|
||||
noContextFiles: true,
|
||||
});
|
||||
|
||||
await loader.reload();
|
||||
const loaded = loader.getSkills();
|
||||
const loaderDiagnostics = loaded.diagnostics.map((diagnostic) =>
|
||||
mapDiagnostic(diagnostic, this.#cwd),
|
||||
);
|
||||
this.#diagnostics = [...pathDiagnostics, ...loaderDiagnostics];
|
||||
this.#indexedSkills = loaded.skills.map((skill) => ({
|
||||
name: skill.name,
|
||||
source: displaySource(skill),
|
||||
canonicalFilePath: canonicalPathSync(skill.filePath),
|
||||
}));
|
||||
this.#entries = loaded.skills
|
||||
.map((skill) => this.#toEntry(skill, loaded.diagnostics))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
}
|
||||
|
||||
public list(): SkillCatalogEntry[] {
|
||||
return this.#entries.map((entry) => ({
|
||||
...entry,
|
||||
diagnostics: entry.diagnostics.map((diagnostic) => ({ ...diagnostic })),
|
||||
}));
|
||||
}
|
||||
|
||||
public index(): SkillIndex {
|
||||
return new SkillIndex(this.#indexedSkills);
|
||||
}
|
||||
|
||||
public readonly diagnostics = (): SkillDiagnostic[] => {
|
||||
return this.#diagnostics.map((diagnostic) => ({ ...diagnostic }));
|
||||
};
|
||||
|
||||
#toEntry(skill: Skill, diagnostics: ResourceDiagnostic[]): SkillCatalogEntry {
|
||||
const canonicalFilePath = canonicalPathSync(skill.filePath);
|
||||
return {
|
||||
name: skill.name,
|
||||
description: skill.description,
|
||||
source: displaySource(skill),
|
||||
filePath: safeDisplayPath(canonicalFilePath, this.#cwd),
|
||||
diagnostics: diagnostics
|
||||
.filter(
|
||||
(diagnostic) =>
|
||||
diagnostic.path !== undefined &&
|
||||
canonicalPathSync(diagnostic.path) === canonicalFilePath,
|
||||
)
|
||||
.map((diagnostic) => mapDiagnostic(diagnostic, this.#cwd)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveExtraSkillPaths(value?: string): Promise<{
|
||||
paths: string[];
|
||||
diagnostics: SkillDiagnostic[];
|
||||
}> {
|
||||
if (value === undefined || value.trim() === "") {
|
||||
return { paths: [], diagnostics: [] };
|
||||
}
|
||||
|
||||
const paths: string[] = [];
|
||||
const diagnostics: SkillDiagnostic[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const rawPath of value.split(delimiter).map((entry) => entry.trim()).filter(Boolean)) {
|
||||
const resolved = resolve(rawPath);
|
||||
try {
|
||||
await access(resolved);
|
||||
const pathStat = await stat(resolved);
|
||||
if (!pathStat.isDirectory()) {
|
||||
diagnostics.push({ level: "error", message: `Extra skill root is not a directory: ${resolved}` });
|
||||
continue;
|
||||
}
|
||||
const canonical = await realpath(resolved);
|
||||
if (!seen.has(canonical)) {
|
||||
seen.add(canonical);
|
||||
paths.push(canonical);
|
||||
}
|
||||
} catch {
|
||||
diagnostics.push({ level: "error", message: `Extra skill root is unavailable: ${resolved}` });
|
||||
}
|
||||
}
|
||||
return { paths, diagnostics };
|
||||
}
|
||||
|
||||
function canonicalPathSync(filePath: string): string {
|
||||
try {
|
||||
return realpathSync(filePath);
|
||||
} catch {
|
||||
return resolve(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
function displaySource(skill: Skill): string {
|
||||
return skill.sourceInfo.scope === "temporary" ? skill.sourceInfo.source : skill.sourceInfo.scope;
|
||||
}
|
||||
|
||||
function safeDisplayPath(filePath: string, cwd: string): string {
|
||||
const relativePath = relative(cwd, filePath);
|
||||
if (relativePath !== "" && !relativePath.startsWith(`..${sep}`) && relativePath !== ".." && !isAbsolute(relativePath)) {
|
||||
return relativePath;
|
||||
}
|
||||
return `external/${skillBasename(filePath)}/SKILL.md`;
|
||||
}
|
||||
|
||||
function skillBasename(filePath: string): string {
|
||||
const segments = filePath.split(sep).filter(Boolean);
|
||||
return segments.at(-2) ?? "skill";
|
||||
}
|
||||
|
||||
function mapDiagnostic(diagnostic: ResourceDiagnostic, cwd: string): SkillDiagnostic {
|
||||
return {
|
||||
level: diagnostic.type === "error" ? "error" : "warning",
|
||||
message: diagnostic.message,
|
||||
...(diagnostic.path === undefined
|
||||
? {}
|
||||
: { filePath: safeDisplayPath(canonicalPathSync(diagnostic.path), cwd) }),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { realpath } from "node:fs/promises";
|
||||
import type { EventStore } from "../events/event-store.js";
|
||||
import type { IndexedSkill, SkillIndex } from "./skill-catalog.js";
|
||||
|
||||
export type SkillEventInferenceOptions = {
|
||||
sessionId: string;
|
||||
runId: string;
|
||||
eventStore: EventStore;
|
||||
skillIndex: SkillIndex;
|
||||
};
|
||||
|
||||
export class SkillEventInference {
|
||||
readonly #sessionId: string;
|
||||
readonly #runId: string;
|
||||
readonly #eventStore: EventStore;
|
||||
readonly #skillIndex: SkillIndex;
|
||||
readonly #pending = new Map<string, IndexedSkill>();
|
||||
|
||||
public constructor(options: SkillEventInferenceOptions) {
|
||||
this.#sessionId = options.sessionId;
|
||||
this.#runId = options.runId;
|
||||
this.#eventStore = options.eventStore;
|
||||
this.#skillIndex = options.skillIndex;
|
||||
}
|
||||
|
||||
public async onReadRequested(input: { toolCallId: string; path: string }): Promise<void> {
|
||||
let canonicalFilePath: string;
|
||||
try {
|
||||
canonicalFilePath = await realpath(input.path);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const skill = this.#skillIndex.find(canonicalFilePath);
|
||||
if (skill === undefined) return;
|
||||
|
||||
this.#pending.set(input.toolCallId, skill);
|
||||
this.#eventStore.append(this.#sessionId, {
|
||||
type: "skill.requested",
|
||||
runId: this.#runId,
|
||||
payload: { name: skill.name, source: skill.source },
|
||||
});
|
||||
}
|
||||
|
||||
public onReadCompleted(input: { toolCallId: string; success: boolean }): void {
|
||||
const skill = this.#pending.get(input.toolCallId);
|
||||
this.#pending.delete(input.toolCallId);
|
||||
if (skill === undefined || !input.success) return;
|
||||
|
||||
this.#eventStore.append(this.#sessionId, {
|
||||
type: "skill.loaded",
|
||||
runId: this.#runId,
|
||||
payload: { name: skill.name, source: skill.source },
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import * as harness from "../src/index.js";
|
||||
|
||||
type CatalogEntry = {
|
||||
name: string;
|
||||
description: string;
|
||||
source: string;
|
||||
filePath: string;
|
||||
diagnostics: Array<{ level: string; message: string }>;
|
||||
};
|
||||
|
||||
type Catalog = {
|
||||
reload(): Promise<void>;
|
||||
list(): CatalogEntry[];
|
||||
index(): unknown;
|
||||
};
|
||||
|
||||
type CatalogConstructor = new (options: {
|
||||
cwd: string;
|
||||
agentDir: string;
|
||||
extraDirsValue?: string;
|
||||
}) => Catalog;
|
||||
|
||||
let root = "";
|
||||
let agentDir = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "agent-studio-skills-"));
|
||||
agentDir = join(root, ".runtime");
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function createSkill(relativeRoot: string, name: string, description = `Use when working with ${name}`): Promise<string> {
|
||||
const directory = join(root, relativeRoot, name);
|
||||
await mkdir(directory, { recursive: true });
|
||||
const filePath = join(directory, "SKILL.md");
|
||||
await writeFile(filePath, `---\nname: ${name}\ndescription: ${description}\n---\n\n# ${name}\n\nFull secret skill body.\n`, "utf8");
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function catalogConstructor(): CatalogConstructor {
|
||||
const Constructor = Reflect.get(harness, "SkillCatalog") as CatalogConstructor | undefined;
|
||||
expect(Constructor, "SkillCatalog must be exported").toBeDefined();
|
||||
return Constructor!;
|
||||
}
|
||||
|
||||
describe("SkillCatalog", () => {
|
||||
it("discovers .agents, .pi, and delimiter-separated extra roots through Pi", async () => {
|
||||
await createSkill(".agents/skills", "ads-analysis");
|
||||
await createSkill(".pi/skills", "report-writer");
|
||||
const extraA = join(root, "extra-a");
|
||||
const extraB = join(root, "extra-b");
|
||||
await createSkill("extra-a", "extra-one");
|
||||
await createSkill("extra-b", "extra-two");
|
||||
|
||||
const catalog = new (catalogConstructor())({
|
||||
cwd: root,
|
||||
agentDir,
|
||||
extraDirsValue: `${extraA}${delimiter}${extraB}${delimiter}${extraA}`,
|
||||
});
|
||||
await catalog.reload();
|
||||
|
||||
expect(catalog.list().map((skill) => skill.name)).toEqual([
|
||||
"ads-analysis",
|
||||
"extra-one",
|
||||
"extra-two",
|
||||
"report-writer",
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns only safe catalog metadata and never Skill bodies", async () => {
|
||||
await createSkill(".agents/skills", "ads-analysis", "Use when analyzing mock ad metrics");
|
||||
const catalog = new (catalogConstructor())({ cwd: root, agentDir });
|
||||
await catalog.reload();
|
||||
|
||||
const serialized = JSON.stringify(catalog.list());
|
||||
expect(serialized).toContain("Use when analyzing mock ad metrics");
|
||||
expect(serialized).not.toContain("Full secret skill body");
|
||||
expect(catalog.list()[0]?.filePath).toBe(".agents/skills/ads-analysis/SKILL.md");
|
||||
expect(catalog.list()[0]?.filePath.startsWith("/")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps running and reports diagnostics for invalid and missing roots", async () => {
|
||||
const invalidDir = join(root, ".agents/skills/invalid");
|
||||
await mkdir(invalidDir, { recursive: true });
|
||||
await writeFile(join(invalidDir, "SKILL.md"), "# missing frontmatter", "utf8");
|
||||
const missing = join(root, "missing-extra");
|
||||
const catalog = new (catalogConstructor())({ cwd: root, agentDir, extraDirsValue: missing });
|
||||
|
||||
await expect(catalog.reload()).resolves.toBeUndefined();
|
||||
expect(catalog.list().flatMap((skill) => skill.diagnostics).length).toBeGreaterThanOrEqual(0);
|
||||
const diagnostics = Reflect.get(catalog, "diagnostics") as (() => Array<{ level: string; message: string }>) | undefined;
|
||||
expect(diagnostics?.().some((entry) => entry.level === "error" && entry.message.includes("missing-extra"))).toBe(true);
|
||||
await expect(access(agentDir)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("SkillEventInference", () => {
|
||||
it("emits requested and loaded only for a successful canonical Skill read", async () => {
|
||||
const skillPath = await createSkill(".agents/skills", "ads-analysis");
|
||||
const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
|
||||
listAfter(sessionId: string): { events: Array<{ type: string }> };
|
||||
};
|
||||
const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
|
||||
const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
|
||||
onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
|
||||
onReadCompleted(input: { toolCallId: string; success: boolean }): void;
|
||||
};
|
||||
expect(Index, "SkillIndex must be exported").toBeDefined();
|
||||
expect(Inference, "SkillEventInference must be exported").toBeDefined();
|
||||
const store = new storeConstructor();
|
||||
const inference = new Inference({
|
||||
sessionId: "ses_1",
|
||||
runId: "run_1",
|
||||
eventStore: store,
|
||||
skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
|
||||
});
|
||||
|
||||
await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
|
||||
inference.onReadCompleted({ toolCallId: "call_1", success: true });
|
||||
expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual([
|
||||
"skill.requested",
|
||||
"skill.loaded",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not emit loaded after a failed read or for a same-named unrelated file", async () => {
|
||||
const skillPath = await createSkill(".agents/skills", "ads-analysis");
|
||||
const unrelated = join(root, "other", "SKILL.md");
|
||||
await mkdir(join(root, "other"));
|
||||
await writeFile(unrelated, "unrelated", "utf8");
|
||||
const storeConstructor = Reflect.get(harness, "InMemoryEventStore") as new () => {
|
||||
listAfter(sessionId: string): { events: Array<{ type: string }> };
|
||||
};
|
||||
const Index = Reflect.get(harness, "SkillIndex") as new (skills: Array<{ name: string; source: string; canonicalFilePath: string }>) => unknown;
|
||||
const Inference = Reflect.get(harness, "SkillEventInference") as unknown as new (options: Record<string, unknown>) => {
|
||||
onReadRequested(input: { toolCallId: string; path: string }): Promise<void>;
|
||||
onReadCompleted(input: { toolCallId: string; success: boolean }): void;
|
||||
};
|
||||
const store = new storeConstructor();
|
||||
const inference = new Inference({
|
||||
sessionId: "ses_1",
|
||||
runId: "run_1",
|
||||
eventStore: store,
|
||||
skillIndex: new Index([{ name: "ads-analysis", source: "project", canonicalFilePath: skillPath }]),
|
||||
});
|
||||
|
||||
await inference.onReadRequested({ toolCallId: "call_1", path: skillPath });
|
||||
inference.onReadCompleted({ toolCallId: "call_1", success: false });
|
||||
await inference.onReadRequested({ toolCallId: "call_2", path: unrelated });
|
||||
inference.onReadCompleted({ toolCallId: "call_2", success: true });
|
||||
|
||||
expect(store.listAfter("ses_1").events.map((event) => event.type)).toEqual(["skill.requested"]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user