diff --git a/packages/harness/package.json b/packages/harness/package.json index c2a074b..a29694e 100644 --- a/packages/harness/package.json +++ b/packages/harness/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@agent-studio/shared": "workspace:*", + "@earendil-works/pi-ai": "0.85.1", "@earendil-works/pi-coding-agent": "^0.85.1", "sharp": "^0.35.4", "typebox": "1.3.7" diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index c3b6b47..dd19787 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -3,6 +3,10 @@ export * from "./approvals/in-memory-approval-broker.js"; export * from "./events/event-store.js"; export * from "./events/in-memory-event-store.js"; export * from "./errors/harness-error.js"; +export * from "./models/base-url-policy.js"; +export * from "./models/in-memory-model-connection-store.js"; +export * from "./models/model-connection-store.js"; +export * from "./models/pi-model-runtime-factory.js"; export * from "./security/redact.js"; export * from "./skills/skill-catalog.js"; export * from "./skills/skill-event-inference.js"; diff --git a/packages/harness/src/models/base-url-policy.ts b/packages/harness/src/models/base-url-policy.ts new file mode 100644 index 0000000..c287918 --- /dev/null +++ b/packages/harness/src/models/base-url-policy.ts @@ -0,0 +1,157 @@ +import { lookup } from "node:dns/promises"; +import { BlockList, isIP } from "node:net"; +import { HarnessError } from "../errors/harness-error.js"; + +export type ResolvedAddress = { address: string; family: 4 | 6 }; +export type AddressResolver = (hostname: string) => Promise; +export type BaseUrlPolicyOptions = { + allowLocalhostHttp?: boolean; + allowPrivateNetwork?: boolean; +}; +export type ValidatedBaseUrl = { + normalizedUrl: string; + addresses: string[]; + redirectPolicy: "same-origin"; +}; + +const alwaysBlocked = createAlwaysBlockedList(); +const privateBlocked = createPrivateBlockedList(); +const sensitiveQueryNames = new Set([ + "apikey", + "authorization", + "token", + "secret", + "password", + "credential", +]); + +export async function validateBaseUrl( + value: string, + options: BaseUrlPolicyOptions = {}, + resolver: AddressResolver = defaultResolver, +): Promise { + let url: URL; + try { + url = new URL(value); + } catch { + throw notAllowed("Base URL is malformed"); + } + + const hostname = stripIpv6Brackets(url.hostname); + const localhostHttp = url.protocol === "http:" && isLocalhostName(hostname) && options.allowLocalhostHttp === true; + if (url.protocol !== "https:" && !localhostHttp) { + throw notAllowed("Base URL must use HTTPS"); + } + if (url.username !== "" || url.password !== "") { + throw notAllowed("Base URL must not contain user information"); + } + if (url.hash !== "") throw notAllowed("Base URL must not contain a fragment"); + for (const key of url.searchParams.keys()) { + const normalized = key.toLowerCase().replaceAll(/[^a-z0-9]/g, ""); + if ([...sensitiveQueryNames].some((name) => normalized === name || normalized.endsWith(name))) { + throw notAllowed("Base URL must not contain credentials in its query"); + } + } + if (hostname === "") throw notAllowed("Base URL must contain a hostname"); + + const addresses = isIP(hostname) === 0 + ? await resolveAddresses(hostname, resolver) + : [{ address: hostname, family: isIP(hostname) as 4 | 6 }]; + const uniqueAddresses = [...new Set(addresses.map(({ address }) => stripIpv6Brackets(address)))]; + if (uniqueAddresses.length === 0) throw notAllowed("Base URL hostname did not resolve"); + for (const address of uniqueAddresses) { + assertAllowedAddress(address, options.allowPrivateNetwork === true); + } + + return { + normalizedUrl: url.toString().replace(/\/$/, url.pathname === "/" ? "" : "/"), + addresses: uniqueAddresses, + redirectPolicy: "same-origin", + }; +} + +async function defaultResolver(hostname: string): Promise { + const results = await lookup(hostname, { all: true, verbatim: true }); + return results.flatMap(({ address, family }) => + family === 4 || family === 6 ? [{ address, family }] : [], + ); +} + +async function resolveAddresses(hostname: string, resolver: AddressResolver): Promise { + try { + return await resolver(hostname); + } catch { + throw notAllowed("Base URL hostname could not be resolved"); + } +} + +function assertAllowedAddress(address: string, allowPrivateNetwork: boolean): void { + const familyNumber = isIP(address); + if (familyNumber === 0) throw notAllowed("Resolver returned an invalid IP address"); + const family = familyNumber === 4 ? "ipv4" : "ipv6"; + if (alwaysBlocked.check(address, family)) throw notAllowed("Base URL resolves to a restricted address"); + if (!allowPrivateNetwork && privateBlocked.check(address, family)) { + throw notAllowed("Base URL resolves to a private address"); + } +} + +function isLocalhostName(hostname: string): boolean { + const normalized = hostname.toLowerCase().replace(/\.$/, ""); + return normalized === "localhost" || normalized.endsWith(".localhost"); +} + +function stripIpv6Brackets(value: string): string { + return value.startsWith("[") && value.endsWith("]") ? value.slice(1, -1) : value; +} + +function notAllowed(message: string): HarnessError { + return new HarnessError("BASE_URL_NOT_ALLOWED", message); +} + +function createAlwaysBlockedList(): BlockList { + const list = new BlockList(); + addSubnets(list, "ipv4", [ + ["0.0.0.0", 8], + ["169.254.0.0", 16], + ["224.0.0.0", 4], + ["240.0.0.0", 4], + ]); + addSubnets(list, "ipv6", [ + ["::", 128], + ["::ffff:0.0.0.0", 104], + ["::ffff:169.254.0.0", 112], + ["::ffff:224.0.0.0", 100], + ["::ffff:240.0.0.0", 100], + ["fe80::", 10], + ["ff00::", 8], + ]); + list.addAddress("fd00:ec2::254", "ipv6"); + return list; +} + +function createPrivateBlockedList(): BlockList { + const list = new BlockList(); + addSubnets(list, "ipv4", [ + ["10.0.0.0", 8], + ["100.64.0.0", 10], + ["127.0.0.0", 8], + ["172.16.0.0", 12], + ["192.168.0.0", 16], + ["198.18.0.0", 15], + ]); + addSubnets(list, "ipv6", [ + ["::1", 128], + ["::ffff:10.0.0.0", 104], + ["::ffff:100.64.0.0", 106], + ["::ffff:127.0.0.0", 104], + ["::ffff:172.16.0.0", 108], + ["::ffff:192.168.0.0", 112], + ["::ffff:198.18.0.0", 111], + ["fc00::", 7], + ]); + return list; +} + +function addSubnets(list: BlockList, family: "ipv4" | "ipv6", entries: Array<[string, number]>): void { + for (const [network, prefix] of entries) list.addSubnet(network, prefix, family); +} diff --git a/packages/harness/src/models/in-memory-model-connection-store.ts b/packages/harness/src/models/in-memory-model-connection-store.ts new file mode 100644 index 0000000..f98c680 --- /dev/null +++ b/packages/harness/src/models/in-memory-model-connection-store.ts @@ -0,0 +1,115 @@ +import { + ModelConnectionInputSchema, + RedactedModelConnectionSchema, + type ModelConnectionInput, + type RedactedModelConnection, +} from "@agent-studio/shared"; +import { HarnessError } from "../errors/harness-error.js"; +import { validateBaseUrl, type ValidatedBaseUrl } from "./base-url-policy.js"; +import type { ModelConnectionStore, ResolvedModelConnection } from "./model-connection-store.js"; + +type ConnectionMetadata = Omit; +type UrlValidator = (url: string) => Promise; + +export type InMemoryModelConnectionStoreOptions = { + environment?: ModelConnectionInput; + validateUrl?: UrlValidator; +}; + +export class InMemoryModelConnectionStore implements ModelConnectionStore { + readonly #metadata = new Map(); + readonly #credentials = new Map(); + readonly #environmentMetadata: ConnectionMetadata | undefined; + readonly #environmentKey: string | undefined; + readonly #validateUrl: UrlValidator; + + public constructor(options: InMemoryModelConnectionStoreOptions = {}) { + if (options.environment === undefined) { + this.#environmentMetadata = undefined; + this.#environmentKey = undefined; + } else { + const parsed = ModelConnectionInputSchema.parse(options.environment); + assertProviderApi(parsed); + const { apiKey, ...metadata } = parsed; + this.#environmentMetadata = metadata; + this.#environmentKey = apiKey; + } + this.#validateUrl = options.validateUrl ?? (async (url) => validateBaseUrl(url)); + } + + public async configure(sessionId: string, input: ModelConnectionInput): Promise { + const parsed = ModelConnectionInputSchema.parse(input); + assertProviderApi(parsed); + let normalized = parsed; + if (parsed.baseUrl !== undefined) { + const result = await this.#validateUrl(parsed.baseUrl); + normalized = { ...parsed, baseUrl: result.normalizedUrl }; + } + const { apiKey, ...metadata } = normalized; + this.#metadata.set(sessionId, metadata); + this.#credentials.delete(sessionId); + if (apiKey !== undefined) this.#credentials.set(sessionId, apiKey); + } + + public getRedacted(sessionId: string): RedactedModelConnection | undefined { + const resolved = this.#resolveOptional(sessionId); + if (resolved === undefined) return undefined; + const key = resolved.apiKey; + const safeConnection = { ...resolved.connection }; + delete safeConnection.apiKey; + return RedactedModelConnectionSchema.parse({ + ...safeConnection, + configured: true, + source: resolved.source, + ...(key === undefined ? {} : { keyHint: keyHint(key) }), + }); + } + + public resolveConnection(sessionId: string): ResolvedModelConnection { + const resolved = this.#resolveOptional(sessionId); + if (resolved !== undefined) return resolved; + throw new HarnessError("MODEL_NOT_CONFIGURED", `No model is configured for session ${sessionId}`); + } + + public clearCredentials(sessionId: string): void { + this.#credentials.delete(sessionId); + this.#metadata.delete(sessionId); + } + + #resolveOptional(sessionId: string): ResolvedModelConnection | undefined { + const metadata = this.#metadata.get(sessionId); + if (metadata !== undefined) { + const apiKey = this.#credentials.get(sessionId); + return { + connection: { ...metadata, ...(apiKey === undefined ? {} : { apiKey }) }, + ...(apiKey === undefined ? {} : { apiKey }), + source: "session", + }; + } + if (this.#environmentMetadata === undefined) return undefined; + return { + connection: { + ...this.#environmentMetadata, + ...(this.#environmentKey === undefined ? {} : { apiKey: this.#environmentKey }), + }, + ...(this.#environmentKey === undefined ? {} : { apiKey: this.#environmentKey }), + source: "environment", + }; + } +} + +function keyHint(apiKey: string): string { + return `••••${apiKey.slice(-4)}`; +} + +function assertProviderApi(input: ModelConnectionInput): void { + const mismatch = + (input.providerId === "anthropic" && input.api !== "anthropic-messages") || + (input.providerId === "openai" && input.api === "anthropic-messages"); + if (mismatch) { + throw new HarnessError( + "MODEL_API_MISMATCH", + `Provider ${input.providerId} is incompatible with API ${input.api}`, + ); + } +} diff --git a/packages/harness/src/models/model-connection-store.ts b/packages/harness/src/models/model-connection-store.ts new file mode 100644 index 0000000..8334944 --- /dev/null +++ b/packages/harness/src/models/model-connection-store.ts @@ -0,0 +1,14 @@ +import type { ModelConnectionInput, RedactedModelConnection } from "@agent-studio/shared"; + +export type ResolvedModelConnection = { + connection: ModelConnectionInput; + apiKey?: string; + source: "session" | "environment"; +}; + +export interface ModelConnectionStore { + configure(sessionId: string, input: ModelConnectionInput): Promise; + getRedacted(sessionId: string): RedactedModelConnection | undefined; + resolveConnection(sessionId: string): ResolvedModelConnection; + clearCredentials(sessionId: string): void; +} diff --git a/packages/harness/src/models/pi-model-runtime-factory.ts b/packages/harness/src/models/pi-model-runtime-factory.ts new file mode 100644 index 0000000..9cc35e5 --- /dev/null +++ b/packages/harness/src/models/pi-model-runtime-factory.ts @@ -0,0 +1,40 @@ +import { InMemoryCredentialStore } from "@earendil-works/pi-ai"; +import { ModelRuntime } from "@earendil-works/pi-coding-agent"; +import type { ResolvedModelConnection } from "./model-connection-store.js"; + +export async function createPiModelRuntime( + resolved: ResolvedModelConnection, + signal?: AbortSignal, +): Promise { + const credentials = new InMemoryCredentialStore(); + const runtime = await ModelRuntime.create({ + credentials, + modelsPath: null, + allowModelNetwork: false, + refreshOnCreate: false, + ...(signal === undefined ? {} : { signal }), + }); + const { connection } = resolved; + runtime.registerProvider(connection.providerId, { + name: connection.displayName ?? connection.providerId, + api: connection.api, + ...(connection.baseUrl === undefined ? {} : { baseUrl: connection.baseUrl }), + models: [ + { + id: connection.modelId, + name: connection.displayName ?? connection.modelId, + api: connection.api, + ...(connection.baseUrl === undefined ? {} : { baseUrl: connection.baseUrl }), + reasoning: false, + input: connection.supportsImages ? ["text", "image"] : ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: connection.contextWindow ?? 32_000, + maxTokens: connection.maxTokens ?? 4_096, + }, + ], + }); + if (resolved.apiKey !== undefined) { + await runtime.setRuntimeApiKey(connection.providerId, resolved.apiKey, signal === undefined ? {} : { signal }); + } + return runtime; +} diff --git a/packages/harness/tests/base-url-policy.test.ts b/packages/harness/tests/base-url-policy.test.ts new file mode 100644 index 0000000..fcc1e2d --- /dev/null +++ b/packages/harness/tests/base-url-policy.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import * as harness from "../src/index.js"; + +type Resolver = (hostname: string) => Promise>; +type Validate = ( + url: string, + options?: { allowLocalhostHttp?: boolean; allowPrivateNetwork?: boolean }, + resolver?: Resolver, +) => Promise<{ normalizedUrl: string; addresses: string[]; redirectPolicy: "same-origin" }>; + +function validate(): Validate { + const value = Reflect.get(harness, "validateBaseUrl") as Validate | undefined; + expect(value, "validateBaseUrl must be exported").toBeDefined(); + return value!; +} + +const publicResolver: Resolver = () => Promise.resolve([{ address: "93.184.216.34", family: 4 }]); + +describe("validateBaseUrl", () => { + it("normalizes an HTTPS URL and pins all validated addresses", async () => { + const result = await validate()("https://api.example.com/v1", {}, () => Promise.resolve([ + { address: "2606:2800:220:1:248:1893:25c8:1946", family: 6 }, + { address: "93.184.216.34", family: 4 }, + ])); + + expect(result).toEqual({ + normalizedUrl: "https://api.example.com/v1", + addresses: ["2606:2800:220:1:248:1893:25c8:1946", "93.184.216.34"], + redirectPolicy: "same-origin", + }); + }); + + it.each([ + "http://169.254.169.254/latest/meta-data", + "https://127.0.0.1/v1", + "file:///etc/passwd", + "https://user:pass@example.com/v1", + "https://example.com/v1#secret", + "https://example.com/v1?api_key=secret", + ])("rejects restricted base URL %s", async (url) => { + await expect(validate()(url, {}, publicResolver)).rejects.toMatchObject({ + code: "BASE_URL_NOT_ALLOWED", + }); + }); + + it.each(["10.0.0.1", "100.64.0.1", "169.254.10.2", "172.16.0.1", "192.168.1.1", "::1", "::ffff:127.0.0.1", "fc00::1", "fe80::1"])( + "rejects a restricted resolved address %s", + async (address) => { + const family = address.includes(":") ? 6 : 4; + await expect( + validate()("https://api.example.com", {}, () => Promise.resolve([{ address, family }])), + ).rejects.toMatchObject({ code: "BASE_URL_NOT_ALLOWED" }); + }, + ); + + it("permits explicit localhost HTTP only in development mode", async () => { + const result = await validate()("http://localhost:11434/v1", { allowLocalhostHttp: true, allowPrivateNetwork: true }, () => Promise.resolve([ + { address: "127.0.0.1", family: 4 }, + ])); + expect(result.normalizedUrl).toBe("http://localhost:11434/v1"); + }); + + it("never permits the metadata address even when private networks are enabled", async () => { + await expect( + validate()("https://metadata.internal", { allowPrivateNetwork: true }, () => Promise.resolve([ + { address: "169.254.169.254", family: 4 }, + ])), + ).rejects.toMatchObject({ code: "BASE_URL_NOT_ALLOWED" }); + }); +}); diff --git a/packages/harness/tests/model-connection.test.ts b/packages/harness/tests/model-connection.test.ts new file mode 100644 index 0000000..3426775 --- /dev/null +++ b/packages/harness/tests/model-connection.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import * as harness from "../src/index.js"; + +type Input = { + providerId: string; + api: "openai-responses" | "openai-completions" | "anthropic-messages"; + apiKey?: string; + baseUrl?: string; + modelId: string; + displayName?: string; + supportsImages: boolean; + contextWindow?: number; + maxTokens?: number; +}; +type Resolved = { connection: Input; apiKey?: string; source: "session" | "environment" }; +type Store = { + configure(sessionId: string, input: Input): Promise; + getRedacted(sessionId: string): Record; + resolveConnection(sessionId: string): Resolved; + clearCredentials(sessionId: string): void; +}; + +function input(overrides: Partial = {}): Input { + return { + providerId: "training-openai", + api: "openai-responses", + apiKey: "sk-test-secret", + baseUrl: "https://api.example.com/v1", + modelId: "training-model", + displayName: "Training Model", + supportsImages: true, + contextWindow: 32_000, + maxTokens: 4_096, + ...overrides, + }; +} + +function createStore(environment?: Input): Store { + const Constructor = Reflect.get(harness, "InMemoryModelConnectionStore") as unknown as + | (new (options?: Record) => Store) + | undefined; + expect(Constructor, "InMemoryModelConnectionStore must be exported").toBeDefined(); + return new Constructor!({ + ...(environment === undefined ? {} : { environment }), + validateUrl: (url: string) => Promise.resolve({ normalizedUrl: url, addresses: ["93.184.216.34"], redirectPolicy: "same-origin" }), + }); +} + +describe("InMemoryModelConnectionStore", () => { + it("never returns the raw key", async () => { + const store = createStore(); + await store.configure("ses_1", input()); + + const json = JSON.stringify(store.getRedacted("ses_1")); + expect(json).not.toContain("sk-test-secret"); + expect(store.getRedacted("ses_1").keyHint).toBe("••••cret"); + }); + + it("uses session config before environment and falls back after an idempotent clear", async () => { + const store = createStore(input({ providerId: "env-provider", apiKey: "env-secret", modelId: "env-model" })); + await store.configure("ses_1", input({ apiKey: "session-secret" })); + expect(store.resolveConnection("ses_1")).toMatchObject({ source: "session", apiKey: "session-secret" }); + + store.clearCredentials("ses_1"); + store.clearCredentials("ses_1"); + expect(store.resolveConnection("ses_1")).toMatchObject({ source: "environment", apiKey: "env-secret" }); + }); + + it("rotates keys without exposing the old credential", async () => { + const store = createStore(); + await store.configure("ses_1", input({ apiKey: "first-secret" })); + await store.configure("ses_1", input({ apiKey: "second-secret" })); + + expect(store.resolveConnection("ses_1").apiKey).toBe("second-secret"); + expect(JSON.stringify(store.getRedacted("ses_1"))).not.toContain("first-secret"); + }); + + it("fails explicitly when no connection is configured", () => { + const store = createStore(); + expect(() => store.resolveConnection("ses_missing")).toThrowError( + expect.objectContaining({ code: "MODEL_NOT_CONFIGURED" }), + ); + }); + + it.each([ + input({ providerId: "anthropic", api: "openai-responses" }), + input({ providerId: "openai", api: "anthropic-messages" }), + ])("rejects a built-in provider/API mismatch", async (invalid) => { + await expect(createStore().configure("ses_1", invalid)).rejects.toMatchObject({ + code: "MODEL_API_MISMATCH", + }); + }); + + it("creates a Pi runtime with only the configured model and in-memory credentials", async () => { + const createRuntime = Reflect.get(harness, "createPiModelRuntime") as + | ((resolved: Resolved) => Promise<{ getModel(providerId: string, modelId: string): unknown; getAuth(providerId: string): Promise }>) + | undefined; + expect(createRuntime, "createPiModelRuntime must be exported").toBeDefined(); + const resolved = { connection: input(), apiKey: "runtime-secret", source: "session" as const }; + + const runtime = await createRuntime!(resolved); + + expect(runtime.getModel("training-openai", "training-model")).toBeDefined(); + expect(await runtime.getAuth("training-openai")).toMatchObject({ auth: { apiKey: "runtime-secret" } }); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d7d96da..0b13752 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,6 +99,9 @@ importers: '@agent-studio/shared': specifier: workspace:* version: link:../shared + '@earendil-works/pi-ai': + specifier: 0.85.1 + version: 0.85.1(ws@8.21.3)(zod@4.5.4) '@earendil-works/pi-coding-agent': specifier: ^0.85.1 version: 0.85.1(ws@8.21.3)(zod@4.5.4) @@ -3446,7 +3449,7 @@ snapshots: '@aws-sdk/types': 3.974.5 '@smithy/core': 3.33.3 '@smithy/fetch-http-handler': 5.8.0 - '@smithy/node-http-handler': 4.7.3 + '@smithy/node-http-handler': 4.12.1 '@smithy/types': 4.18.0 tslib: 2.8.1