feat(harness): discover skills progressively

packages/harness/src/index.ts

@@ -1,6 +1,8 @@
 export * from "./events/event-store.js";
 export * from "./events/in-memory-event-store.js";
 export * from "./errors/harness-error.js";
+export * from "./skills/skill-catalog.js";
+export * from "./skills/skill-event-inference.js";
 export * from "./tools/mock-ads-metrics.js";
 export * from "./tools/save-report-draft.js";
 export * from "./tools/tool-policy.js";

packages/harness/src/skills/skill-catalog.ts

@@ -0,0 +1,196 @@
+import { access, realpath, stat } from "node:fs/promises";
+import { realpathSync } from "node:fs";
+import { delimiter, isAbsolute, relative, resolve, sep } from "node:path";
+import {
+  DefaultResourceLoader,
+  SettingsManager,
+  type ResourceDiagnostic,
+  type Skill,
+} from "@earendil-works/pi-coding-agent";
+
+export type SkillDiagnostic = {
+  level: "warning" | "error";
+  message: string;
+  filePath?: string;
+};
+
+export type SkillCatalogEntry = {
+  name: string;
+  description: string;
+  source: string;
+  filePath: string;
+  diagnostics: SkillDiagnostic[];
+};
+
+export type IndexedSkill = {
+  name: string;
+  source: string;
+  canonicalFilePath: string;
+};
+
+export class SkillIndex {
+  readonly #skillsByPath = new Map<string, IndexedSkill>();
+
+  public constructor(skills: IndexedSkill[]) {
+    for (const skill of skills) {
+      this.#skillsByPath.set(canonicalPathSync(skill.canonicalFilePath), {
+        ...skill,
+        canonicalFilePath: canonicalPathSync(skill.canonicalFilePath),
+      });
+    }
+  }
+
+  public find(canonicalFilePath: string): IndexedSkill | undefined {
+    return this.#skillsByPath.get(canonicalFilePath);
+  }
+}
+
+export type SkillCatalogOptions = {
+  cwd: string;
+  agentDir: string;
+  extraDirsValue?: string;
+};
+
+export class SkillCatalog {
+  readonly #cwd: string;
+  readonly #agentDir: string;
+  readonly #extraDirsValue: string | undefined;
+  #entries: SkillCatalogEntry[] = [];
+  #indexedSkills: IndexedSkill[] = [];
+  #diagnostics: SkillDiagnostic[] = [];
+
+  public constructor(options: SkillCatalogOptions) {
+    this.#cwd = canonicalPathSync(options.cwd);
+    this.#agentDir = canonicalPathSync(options.agentDir);
+    this.#extraDirsValue = options.extraDirsValue;
+  }
+
+  public async reload(): Promise<void> {
+    const { paths: additionalSkillPaths, diagnostics: pathDiagnostics } =
+      await resolveExtraSkillPaths(this.#extraDirsValue);
+    const loader = new DefaultResourceLoader({
+      cwd: this.#cwd,
+      agentDir: this.#agentDir,
+      settingsManager: SettingsManager.inMemory(),
+      additionalSkillPaths,
+      noExtensions: true,
+      noPromptTemplates: true,
+      noThemes: true,
+      noContextFiles: true,
+    });
+
+    await loader.reload();
+    const loaded = loader.getSkills();
+    const loaderDiagnostics = loaded.diagnostics.map((diagnostic) =>
+      mapDiagnostic(diagnostic, this.#cwd),
+    );
+    this.#diagnostics = [...pathDiagnostics, ...loaderDiagnostics];
+    this.#indexedSkills = loaded.skills.map((skill) => ({
+      name: skill.name,
+      source: displaySource(skill),
+      canonicalFilePath: canonicalPathSync(skill.filePath),
+    }));
+    this.#entries = loaded.skills
+      .map((skill) => this.#toEntry(skill, loaded.diagnostics))
+      .sort((left, right) => left.name.localeCompare(right.name));
+  }
+
+  public list(): SkillCatalogEntry[] {
+    return this.#entries.map((entry) => ({
+      ...entry,
+      diagnostics: entry.diagnostics.map((diagnostic) => ({ ...diagnostic })),
+    }));
+  }
+
+  public index(): SkillIndex {
+    return new SkillIndex(this.#indexedSkills);
+  }
+
+  public readonly diagnostics = (): SkillDiagnostic[] => {
+    return this.#diagnostics.map((diagnostic) => ({ ...diagnostic }));
+  };
+
+  #toEntry(skill: Skill, diagnostics: ResourceDiagnostic[]): SkillCatalogEntry {
+    const canonicalFilePath = canonicalPathSync(skill.filePath);
+    return {
+      name: skill.name,
+      description: skill.description,
+      source: displaySource(skill),
+      filePath: safeDisplayPath(canonicalFilePath, this.#cwd),
+      diagnostics: diagnostics
+        .filter(
+          (diagnostic) =>
+            diagnostic.path !== undefined &&
+            canonicalPathSync(diagnostic.path) === canonicalFilePath,
+        )
+        .map((diagnostic) => mapDiagnostic(diagnostic, this.#cwd)),
+    };
+  }
+}
+
+async function resolveExtraSkillPaths(value?: string): Promise<{
+  paths: string[];
+  diagnostics: SkillDiagnostic[];
+}> {
+  if (value === undefined || value.trim() === "") {
+    return { paths: [], diagnostics: [] };
+  }
+
+  const paths: string[] = [];
+  const diagnostics: SkillDiagnostic[] = [];
+  const seen = new Set<string>();
+  for (const rawPath of value.split(delimiter).map((entry) => entry.trim()).filter(Boolean)) {
+    const resolved = resolve(rawPath);
+    try {
+      await access(resolved);
+      const pathStat = await stat(resolved);
+      if (!pathStat.isDirectory()) {
+        diagnostics.push({ level: "error", message: `Extra skill root is not a directory: ${resolved}` });
+        continue;
+      }
+      const canonical = await realpath(resolved);
+      if (!seen.has(canonical)) {
+        seen.add(canonical);
+        paths.push(canonical);
+      }
+    } catch {
+      diagnostics.push({ level: "error", message: `Extra skill root is unavailable: ${resolved}` });
+    }
+  }
+  return { paths, diagnostics };
+}
+
+function canonicalPathSync(filePath: string): string {
+  try {
+    return realpathSync(filePath);
+  } catch {
+    return resolve(filePath);
+  }
+}
+
+function displaySource(skill: Skill): string {
+  return skill.sourceInfo.scope === "temporary" ? skill.sourceInfo.source : skill.sourceInfo.scope;
+}
+
+function safeDisplayPath(filePath: string, cwd: string): string {
+  const relativePath = relative(cwd, filePath);
+  if (relativePath !== "" && !relativePath.startsWith(`..${sep}`) && relativePath !== ".." && !isAbsolute(relativePath)) {
+    return relativePath;
+  }
+  return `external/${skillBasename(filePath)}/SKILL.md`;
+}
+
+function skillBasename(filePath: string): string {
+  const segments = filePath.split(sep).filter(Boolean);
+  return segments.at(-2) ?? "skill";
+}
+
+function mapDiagnostic(diagnostic: ResourceDiagnostic, cwd: string): SkillDiagnostic {
+  return {
+    level: diagnostic.type === "error" ? "error" : "warning",
+    message: diagnostic.message,
+    ...(diagnostic.path === undefined
+      ? {}
+      : { filePath: safeDisplayPath(canonicalPathSync(diagnostic.path), cwd) }),
+  };
+}

packages/harness/src/skills/skill-event-inference.ts

@@ -0,0 +1,55 @@
+import { realpath } from "node:fs/promises";
+import type { EventStore } from "../events/event-store.js";
+import type { IndexedSkill, SkillIndex } from "./skill-catalog.js";
+
+export type SkillEventInferenceOptions = {
+  sessionId: string;
+  runId: string;
+  eventStore: EventStore;
+  skillIndex: SkillIndex;
+};
+
+export class SkillEventInference {
+  readonly #sessionId: string;
+  readonly #runId: string;
+  readonly #eventStore: EventStore;
+  readonly #skillIndex: SkillIndex;
+  readonly #pending = new Map<string, IndexedSkill>();
+
+  public constructor(options: SkillEventInferenceOptions) {
+    this.#sessionId = options.sessionId;
+    this.#runId = options.runId;
+    this.#eventStore = options.eventStore;
+    this.#skillIndex = options.skillIndex;
+  }
+
+  public async onReadRequested(input: { toolCallId: string; path: string }): Promise<void> {
+    let canonicalFilePath: string;
+    try {
+      canonicalFilePath = await realpath(input.path);
+    } catch {
+      return;
+    }
+    const skill = this.#skillIndex.find(canonicalFilePath);
+    if (skill === undefined) return;
+
+    this.#pending.set(input.toolCallId, skill);
+    this.#eventStore.append(this.#sessionId, {
+      type: "skill.requested",
+      runId: this.#runId,
+      payload: { name: skill.name, source: skill.source },
+    });
+  }
+
+  public onReadCompleted(input: { toolCallId: string; success: boolean }): void {
+    const skill = this.#pending.get(input.toolCallId);
+    this.#pending.delete(input.toolCallId);
+    if (skill === undefined || !input.success) return;
+
+    this.#eventStore.append(this.#sessionId, {
+      type: "skill.loaded",
+      runId: this.#runId,
+      payload: { name: skill.name, source: skill.source },
+    });
+  }
+}