feat: add durable CMS-free image video subsystem and UI components

This commit is contained in:
Wenxin Duan
2026-09-15 01:04:09 +08:00
parent 9349317eb8
commit 40e03e393c
14 changed files with 795 additions and 0 deletions
+49
View File
@@ -0,0 +1,49 @@
import { createReadStream } from "node:fs";
import { HarnessError, ImageVideoDraftSchema, taskIdSchema } from "@agent-studio/harness";
import type { FastifyInstance, FastifyRequest } from "fastify";
import { z } from "zod";
import type { AppDependencies } from "../app.js";
import { requireSession } from "./sessions.js";
export function registerImageVideoRoutes(app: FastifyInstance, dependencies: AppDependencies) {
const context = (request: FastifyRequest) => {
const host = request.headers.host ?? "";
const ports = dependencies.imageVideoPorts ?? [80, 3001, 5173, 4173];
let url: URL;
try { url = new URL(`http://${host}`); } catch { throw new HarnessError("I2V_ORIGIN_DENIED", "Local origin required"); }
if (!["localhost", "127.0.0.1", "[::1]"].includes(url.hostname) || !ports.includes(Number(url.port || 80)) || request.headers["sec-fetch-site"] === "cross-site" || (request.headers.origin !== undefined && request.headers.origin !== url.origin)) throw new HarnessError("I2V_ORIGIN_DENIED", "Image video tasks are available only from the local page");
if (!dependencies.imageVideo) throw new HarnessError("I2V_TASK_NOT_FOUND", "Image video service unavailable");
return dependencies.imageVideo;
};
app.get("/api/image-video/tasks", async (request, reply) => {
const service = context(request); void reply.header("Cache-Control", "no-store");
const history = await service.list();
return { tasks: await Promise.all(history.map(task => service.refresh(task.id))) };
});
app.get("/api/image-video/tasks/:taskId", async (request, reply) => {
const service = context(request); const { taskId } = z.object({ taskId: taskIdSchema }).parse(request.params);
void reply.header("Cache-Control", "no-store"); return { task: await service.refresh(taskId) };
});
app.post("/api/sessions/:sessionId/image-video/tasks", async (request, reply) => {
const service = context(request); const { sessionId } = z.object({ sessionId: z.string().regex(/^ses_[\w-]+$/) }).parse(request.params);
requireSession(dependencies, sessionId);
if (!request.headers["content-type"]?.startsWith("application/json")) throw new HarnessError("I2V_JSON_REQUIRED", "Draft creation requires JSON");
void reply.header("Cache-Control", "no-store"); return { task: await service.create(sessionId, ImageVideoDraftSchema.parse(request.body)) };
});
for (const kind of ["reference", "video"] as const) app.get(`/api/image-video/tasks/:taskId/${kind}`, async (request, reply) => {
const service = context(request); const { taskId } = z.object({ taskId: taskIdSchema }).parse(request.params);
const query = z.object({ download: z.literal("1").optional() }).strict().parse(request.query);
const media = await service.media(taskId, kind);
void reply.header("Cache-Control", "no-store").header("X-Content-Type-Options", "nosniff").header("Cross-Origin-Resource-Policy", "same-origin").header("Accept-Ranges", "bytes").type(media.contentType);
if (query.download === "1") void reply.header("Content-Disposition", `attachment; filename="${taskId}${kind === "video" ? ".mp4" : ".image"}"`);
const range = request.headers.range;
if (range !== undefined) {
const match = /^bytes=(\d*)-(\d*)$/.exec(range); let start = 0; let end = media.size - 1;
if (match?.[1]) { start = Number(match[1]); if (match[2]) end = Math.min(Number(match[2]), end); }
else if (match?.[2]) start = Math.max(0, media.size - Number(match[2]));
if (!match || (!match[1] && !match[2]) || !Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end || start >= media.size || end < 0) return reply.code(416).header("Content-Range", `bytes */${String(media.size)}`).send();
return reply.code(206).header("Content-Range", `bytes ${String(start)}-${String(end)}/${String(media.size)}`).header("Content-Length", end - start + 1).send(createReadStream(media.path, { start, end }));
}
return reply.header("Content-Length", media.size).send(createReadStream(media.path));
});
}
+44
View File
@@ -0,0 +1,44 @@
import { describe, expect, it } from "vitest";
import { buildApp, type AppDependencies } from "../src/app.js";
import { ImageVideoService, FileAttachmentStore } from "@agent-studio/harness";
import type { ImageVideoPublicTask } from "@agent-studio/harness";
import { mkdtemp, mkdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Readable } from "node:stream";
async function fixture() {
const root = await mkdtemp(join(tmpdir(), "i2v-api-")); await mkdir(join(root, "ses_1"));
const attachments = new FileAttachmentStore({ sessionsRoot: root });
const image = await attachments.add("ses_1", { name: "pixel.png", mediaType: "image/png", stream: Readable.from(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAE0lEQVQImWP4z8DwnwGM/zMwAAAf7gP9qS/A4gAAAABJRU5ErkJggg==", "base64")) });
const service = new ImageVideoService({ root: join(root, "tasks"), attachments, remote: {} as never, collect: () => Promise.reject(new Error("No external operations permitted")) });
const dependencies = { imageVideo: service, sessionRegistry: { get: (id: string) => id === "ses_1" ? { sessionId: id } : undefined, disposeAll: async () => {} }, skillCatalog: { list: () => [] }, modelStore: {}, attachmentStore: attachments, approvalBroker: {} } as unknown as AppDependencies;
return { app: buildApp(dependencies), service, draft: { attachmentId: image.attachmentId, title: "Reference", prompt: "Natural motion and ambient sound", durationSeconds: 10 } };
}
describe("image video local API", () => {
it("creates locally, persists history independently from Sessions and serves registered reference ranges", async () => {
const { app, service, draft } = await fixture();
try {
const response = await app.inject({ method: "POST", url: "/api/sessions/ses_1/image-video/tasks", payload: draft });
expect(response.statusCode).toBe(200); const task = response.json<{ task: ImageVideoPublicTask }>().task;
expect(task.state).toBe("draft");
expect((await app.inject({ method: "GET", url: "/api/image-video/tasks" })).json<{ tasks: ImageVideoPublicTask[] }>().tasks).toHaveLength(1);
const ranged = await app.inject({ method: "GET", url: task.referenceUrl, headers: { range: "bytes=0-3" } });
expect(ranged.statusCode).toBe(206); expect(ranged.rawPayload.length).toBe(4); expect(ranged.headers["cache-control"]).toBe("no-store");
expect((await app.inject({ method: "GET", url: task.referenceUrl, headers: { range: "bytes=999999-" } })).statusCode).toBe(416);
expect((await app.inject({ method: "GET", url: `/api/image-video/tasks/${task.id}/video` })).statusCode).toBe(404);
expect((await service.list())[0]?.id).toBe(task.id);
} finally { await app.close(); }
});
it("rejects foreign origins, rebinding hosts, unknown sessions, arbitrary media paths and mutation types", async () => {
const { app, draft } = await fixture();
try {
for (const headers of [{ host: "evil.example", origin: "http://evil.example" }, { origin: "https://evil.example" }, { "sec-fetch-site": "cross-site" }]) expect((await app.inject({ method: "GET", url: "/api/image-video/tasks", headers })).statusCode).toBeGreaterThanOrEqual(400);
expect((await app.inject({ method: "POST", url: "/api/sessions/ses_other/image-video/tasks", payload: draft })).statusCode).toBe(404);
expect((await app.inject({ method: "POST", url: "/api/sessions/ses_1/image-video/tasks", payload: "text", headers: { "content-type": "text/plain" } })).statusCode).toBe(400);
expect((await app.inject({ method: "GET", url: "/api/image-video/tasks/secret.env/reference" })).statusCode).toBe(400);
expect((await app.inject({ method: "POST", url: "/api/sessions/ses_1/image-video/tasks", payload: { ...draft, path: "/etc/passwd" } })).statusCode).toBe(400);
} finally { await app.close(); }
});
});
@@ -0,0 +1,86 @@
import { useEffect, useRef, useState } from "react";
export type ImageVideoTask = { id: string; title: string; prompt: string; durationSeconds: number; state: string; referenceUrl: string; videoUrl?: string; jobId?: string; error?: string; validation?: { videoSeconds?: number; width?: number; height?: number; fps?: number } };
export function isImageVideoTask(value: unknown): value is ImageVideoTask {
if (!value || typeof value !== "object") return false;
const task = value as Partial<ImageVideoTask>;
return typeof task.id === "string" && /^i2v_[0-9a-f-]{36}$/.test(task.id) && typeof task.title === "string" && typeof task.prompt === "string" && typeof task.state === "string" && typeof task.durationSeconds === "number" && task.referenceUrl === `/api/image-video/tasks/${task.id}/reference`;
}
const stateNames: Record<string, string> = { draft: "本地草稿 · 待图片上传审批", initializing_upload: "准备上传", uploading: "上传中", finalizing: "入库确认中", importing: "参考图入库中", awaiting_submission: "参考图已入库 · 待生成审批", submitting: "提交中", queued: "排队中", running: "生成中", validating: "成片技术校验中", ready: "生成预览 · 待视听检查", failed: "失败", unknown_submission: "提交状态未知 · 禁止重投", unknown_upload: "上传状态未知 · 禁止重复导入" };
export function ImageVideoTaskCard({ task }: { task: ImageVideoTask }) {
if (!isImageVideoTask(task)) return null;
const video = task.state === "ready" && task.videoUrl === `/api/image-video/tasks/${task.id}/video` ? task.videoUrl : undefined;
return <article className="i2v-task">
<img className="i2v-reference" src={task.referenceUrl} alt={`${task.title} 参考图`} loading="lazy" />
<div><h3>{task.title}</h3><p role="status">{stateNames[task.state] ?? "未知状态 · 请核实"}</p><p>{task.durationSeconds} 秒 · H3 ref2va · 9:16 · 768p</p><p>{task.prompt}</p>
{task.jobId ? <p>生成任务:<code>{task.jobId}</code></p> : null}
{task.error ? <p role="alert">{task.error}</p> : null}
{video ? <><video aria-label="生成预览播放器" src={video} controls playsInline preload="metadata" /><a className="secondary-button" href={`${video}?download=1`} download>下载 MP4</a><p>技术验证通过,待视听检查;音轨存在不能证明声音内容正确。</p></> : null}
</div>
</article>;
}
async function jsonRequest(url: string, init?: RequestInit): Promise<Record<string, unknown>> {
const response = await fetch(url, { cache: "no-store", ...init });
if (!response.ok) throw new Error("图生视频请求失败,请检查连接或任务状态。");
return await response.json() as Record<string, unknown>;
}
export function ImageVideoPanel({ sessionId, onDraft }: { sessionId?: string; onDraft?: (task: ImageVideoTask) => void }) {
const [tasks, setTasks] = useState<ImageVideoTask[]>([]); const [loading, setLoading] = useState(true); const [error, setError] = useState<string>();
const [file, setFile] = useState<File>(); const [title, setTitle] = useState(""); const [prompt, setPrompt] = useState(""); const [duration, setDuration] = useState(10); const [saving, setSaving] = useState(false);
const [reference, setReference] = useState<string>(); const mounted = useRef(true);
useEffect(() => {
mounted.current = true; const controller = new AbortController(); let active = false;
const refresh = async () => { if (active) return; active = true; try { const data = await jsonRequest("/api/image-video/tasks", { signal: controller.signal }); if (!controller.signal.aborted) { setTasks(Array.isArray(data.tasks) ? data.tasks.filter(isImageVideoTask) : []); setError(undefined); } } catch { if (!controller.signal.aborted) setError("无法读取任务历史,请检查本机服务。"); } finally { active = false; if (!controller.signal.aborted) setLoading(false); } };
void refresh(); const timer = window.setInterval(() => void refresh(), 30_000);
return () => { mounted.current = false; controller.abort(); window.clearInterval(timer); };
}, []);
useEffect(() => { if (!file) { setReference(undefined); return; } const url = URL.createObjectURL(file); setReference(url); return () => URL.revokeObjectURL(url); }, [file]);
const create = async () => {
if (!file || !sessionId || saving || !title.trim() || !prompt.trim()) return;
setSaving(true); setError(undefined);
try {
const body = new FormData(); body.append("file", file);
const upload = await jsonRequest(`/api/sessions/${encodeURIComponent(sessionId)}/attachments`, { method: "POST", body });
const attachment = upload.attachment as { attachmentId?: unknown } | undefined;
if (typeof attachment?.attachmentId !== "string") throw new Error("附件接收失败。");
const result = await jsonRequest(`/api/sessions/${encodeURIComponent(sessionId)}/image-video/tasks`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ attachmentId: attachment.attachmentId, title, prompt, durationSeconds: duration }) });
if (!isImageVideoTask(result.task)) throw new Error("任务响应无效。");
const task = result.task;
if (mounted.current) { setTasks(current => [task, ...current]); onDraft?.(task); }
} catch (error) { if (mounted.current) setError(error instanceof Error ? error.message : "准备任务失败。"); } finally { if (mounted.current) setSaving(false); }
};
return <section className="i2v-panel" aria-label="图生视频工作区">
<h2>图片 → 有声视频</h2><p>单张参考图,H3 ref2va。无需 CMS。默认竖屏、自然速度、有环境声;10 秒是目标,实际支持尚待验证。</p>
{sessionId && onDraft ? <form onSubmit={event => { event.preventDefault(); void create(); }}>
<label>参考图片<input aria-label="参考图片" type="file" accept="image/png,image/jpeg,image/webp" disabled={saving} onChange={event => setFile(event.currentTarget.files?.[0])} /></label>
{reference ? <img className="i2v-reference" src={reference} alt="本机参考图片预览" /> : null}
<label>标题<input aria-label="标题" value={title} maxLength={120} disabled={saving} onChange={event => setTitle(event.currentTarget.value)} /></label>
<label>动作与声音<textarea aria-label="动作与声音" placeholder="例如:人物自然转向窗外,镜头缓慢推近,有雨声。" value={prompt} maxLength={4000} disabled={saving} onChange={event => setPrompt(event.currentTarget.value)} /></label>
<label>目标时长(秒)<input aria-label="目标时长(秒)" type="number" min="0.001" max="60" step="0.001" value={duration} disabled={saving} onChange={event => setDuration(Number(event.currentTarget.value))} /></label>
<p>准备仅保存到本机。随后在聊天中分别确认图片外传与 H3 生成。此缩略图不会传给聊天模型;H3 将在批准后接收图片。费用未知。</p>
<button className="secondary-button" disabled={saving || !file || !title.trim() || !prompt.trim() || duration <= 0 || duration > 60} type="submit">{saving ? "保存中…" : "准备本地任务"}</button>
</form> : null}
{loading ? <p role="status">正在读取持久任务…</p> : null}
{error ? <p role="alert">{error}</p> : null}
{!loading && tasks.length === 0 ? <p>尚无图生视频任务。</p> : null}
<div className="i2v-history">{tasks.map(task => <ImageVideoTaskCard key={task.id} task={task} />)}</div>
</section>;
}
export function ImageVideoApprovalDetails({ taskId, onReady }: { taskId: unknown; onReady: (ready: boolean) => void }) {
const [task, setTask] = useState<ImageVideoTask>(); const [error, setError] = useState(false);
useEffect(() => {
const controller = new AbortController(); onReady(false); setTask(undefined); setError(false);
if (typeof taskId !== "string" || !/^i2v_[0-9a-f-]{36}$/.test(taskId)) { setError(true); return; }
void jsonRequest(`/api/image-video/tasks/${taskId}`, { signal: controller.signal }).then(result => {
if (!isImageVideoTask(result.task) || result.task.id !== taskId) throw new Error("Task mismatch");
if (!controller.signal.aborted) { setTask(result.task); onReady(true); }
}).catch(() => { if (!controller.signal.aborted) setError(true); });
return () => controller.abort();
}, [taskId, onReady]);
if (error) return <p role="alert">无法读取本次冻结参数,审批已禁用。请拒绝并核实任务。</p>;
return task ? <ImageVideoTaskCard task={task} /> : <p role="status">读取本次参考图和冻结参数…</p>;
}
+53
View File
@@ -0,0 +1,53 @@
import { act, render, screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ApprovalDialog } from "../src/components/approvals/ApprovalDialog.js";
import { ImageVideoPanel, ImageVideoTaskCard } from "../src/components/tools/ImageVideoPanel.js";
const id = "i2v_11111111-1111-1111-1111-111111111111";
const task = { id, title: "Window", prompt: "Turn naturally toward the window; ambient rain", durationSeconds: 10, state: "queued", referenceUrl: `/api/image-video/tasks/${id}/reference`, createdAt: "2026-09-15T00:00:00Z", updatedAt: "2026-09-15T00:00:00Z" };
afterEach(() => { vi.unstubAllGlobals(); vi.useRealTimers(); });
describe("image video UI", () => {
it("restores queued tasks, polls at 30 seconds, and cleans up after unmount", async () => {
vi.useFakeTimers();
const fetcher = vi.fn(() => Promise.resolve(new Response(JSON.stringify({ tasks: [task] }))));
vi.stubGlobal("fetch", fetcher); const view = render(<ImageVideoPanel />);
await act(() => vi.advanceTimersByTimeAsync(0));
expect(screen.getByText("Window")).toBeVisible(); expect(screen.getByText(/排队中/)).toBeVisible();
await act(() => vi.advanceTimersByTimeAsync(29_999)); expect(fetcher).toHaveBeenCalledTimes(1);
await act(() => vi.advanceTimersByTimeAsync(1)); expect(fetcher).toHaveBeenCalledTimes(2);
view.unmount(); await act(() => vi.advanceTimersByTimeAsync(60_000)); expect(fetcher).toHaveBeenCalledTimes(2);
});
it("creates a draft from a local image without needing model image capability", async () => {
const onDraft = vi.fn(); const requests: { url: string; body: unknown }[] = [];
vi.stubGlobal("fetch", (url: string, init?: RequestInit) => { requests.push({ url, body: init?.body }); return Promise.resolve(new Response(JSON.stringify(url.includes("attachments") ? { attachment: { attachmentId: "att_local" } } : init?.method === "POST" ? { task } : { tasks: [] }))); });
render(<ImageVideoPanel sessionId="ses_1" onDraft={onDraft} />); const user = userEvent.setup();
await user.upload(screen.getByLabelText("参考图片"), new File(["test"], "test.png", { type: "image/png" }));
await user.type(screen.getByLabelText("标题"), "Window"); await user.type(screen.getByLabelText("动作与声音"), task.prompt);
await user.click(screen.getByRole("button", { name: "准备本地任务" }));
await waitFor(() => expect(onDraft).toHaveBeenCalledWith(expect.objectContaining({ id })));
expect(requests.some(request => request.url.includes("/sessions/ses_1/attachments"))).toBe(true);
expect(requests.filter(request => request.url.endsWith("/image-video/tasks") && typeof request.body === "string")[0]?.body).toContain('"attachmentId":"att_local"');
expect(requests.some(request => /submit|upload_reference/.test(request.url))).toBe(false);
});
it("renders a native player/download only for this registered task video", () => {
const view = render(<ImageVideoTaskCard task={{ ...task, state: "ready", videoUrl: `/api/image-video/tasks/${id}/video` }} />);
expect(screen.getByLabelText("生成预览播放器")).toHaveAttribute("controls");
expect(screen.getByRole("link", { name: "下载 MP4" })).toHaveAttribute("href", `/api/image-video/tasks/${id}/video?download=1`);
expect(screen.getByRole("status")).toHaveTextContent("待视听检查");
view.rerender(<ImageVideoTaskCard task={{ ...task, state: "ready", videoUrl: "https://evil.example/video" }} />);
expect(screen.queryByRole("link", { name: "下载 MP4" })).not.toBeInTheDocument();
});
it("requires an unchecked confirmation after showing the frozen reference/prompt/duration", async () => {
vi.stubGlobal("fetch", () => Promise.resolve(new Response(JSON.stringify({ task }))));
const approve = vi.fn();
render(<ApprovalDialog request={{ approvalId: "ap_i2v", toolCallId: "tc", toolName: "i2v_submit", arguments: { taskId: id }, riskLevel: "medium", expiresAt: "2099-01-01T00:00:00Z", status: "pending" }} onApprove={approve} onDeny={vi.fn()} />);
expect(await screen.findByText(task.prompt)).toBeVisible();
expect(screen.getByRole("img")).toHaveAttribute("src", task.referenceUrl);
expect(screen.getByText(/10 秒/)).toBeVisible(); expect(screen.getByText(/费用未知/)).toBeVisible();
const button = screen.getByRole("button", { name: "批准 H3 生成" }); expect(button).toBeDisabled();
await userEvent.click(screen.getByRole("checkbox")); expect(button).toBeEnabled();
await userEvent.click(button); expect(approve).toHaveBeenCalledWith("ap_i2v");
});
});
+17
View File
@@ -0,0 +1,17 @@
# SW Ads image-to-video in Agent Studio
The Skills pane contains “图片 → 有声视频”. Add one JPEG, PNG or WebP reference, a title, the exact motion/sound prompt and a duration (10 seconds initially). “准备本地任务” receives the image through the existing Session attachment validator, copies normalized image bytes to private durable storage, and fills the chat composer with the registered task ID. Send that prepared message to use the original `swads-image-to-video` Skill. No CMS login or vision-enabled chat model is required. The local thumbnail itself is not transmitted to the chat model.
Two separate page approvals are required. Reference upload sends the frozen image to SW Ads storage and creates a `review_pending`, unassigned reference asset. Generation sends that asset and the frozen prompt to H3 `ref2va`, 9:16, 768p with the exact requested duration. Each dialog loads the registered image, prompt and duration and starts with an unchecked confirmation. Refusal causes no operation of that type. Costs and arbitrary duration support are not assumed. There is no provider fallback, automatic repair, account assignment, moderation approval or publication.
Only four controlled tools are exposed: `i2v_create_task`, `i2v_upload_reference`, `i2v_submit` and `i2v_status`. The latter three accept only a task ID owned by that Session. The analytics MCP allowlist remains separate. The server discovers current input schemas and the current MCP account’s `tenant_id` and `platform`; ambiguity blocks upload. Original Skill snapshots include SKILL.md, three references and scripts/media.py.
The durable history is stored under `AGENT_DIR/image-video` (default `.agent-studio-runtime/image-video`). It persists independently of chat Sessions. Returning to the local page lists registered tasks and polls at most every 30 seconds per task, including after restart. Old tasks cannot be reassigned to a new Session for mutation. Only one server process may own a runtime directory. Each operation serializes its task, validates a strict versioned record and reference hash, and atomically persists revisioned intents before network mutations. Unknown upload/submission outcomes are quarantined; never delete their records and retry blindly. A returned generation receipt is retained separately before updating its task state, so stale results cannot silently overwrite a newer revision.
The local history and media routes are `/api/image-video/tasks`, `/api/image-video/tasks/:taskId`, and the registered `reference` / `video` endpoints. Draft creation is `/api/sessions/:sessionId/image-video/tasks`. All require a loopback Host with an allowed local port, reject cross-site requests and mismatched Origin, and media/history are `no-store`. No endpoint accepts a remote media URL or arbitrary filesystem path. Media supports HTTP Range and download.
MP4 collection uses the registered job’s single `video/mp4` artifact only, public HTTPS DNS validation with the actual connection pinned to the validated IP, no redirects, bounded bytes and elapsed time. The original Skill verifier runs with every FFmpeg/FFprobe subprocess restricted to file/pipe protocols and the MP4 demuxer. Full decode, sound presence, portrait geometry, positive frame rate and exact video duration are checked; audio container packet rounding has a 50 ms tolerance. A local SHA-256 binds the validated file to the registered player/download endpoint. A successful technical preview remains “待视听检查”; no human visual or audio approval is inferred.
Server configuration: `SWADS_MCP_TOKEN` stays server-only; optional `IMAGE_VIDEO_SKILL_DIR` overrides the original Skill directory; `PYTHON_EXECUTABLE` may select the installed local Python. Existing bundled FFmpeg/FFprobe are reused. Never place credentials in task prompts or browser input.
Validation boundary: automated tests use synthetic local references and fake provider boundary responses. Read-only live discovery confirms current input schemas and identity structure; current tools do not advertise output schemas. The adapter conservatively accepts documented top-level upload fields (`upload_id`, `object_key`, `upload_url`), import `creative_factory_asset_id`, submit `job_id`, and status `status` plus `artifacts[{content_type,url}]`; unsupported or ambiguous envelopes stop safely. Actual upload/finalization/status envelopes must still be checked with an explicitly approved live reference. A real target-duration H3 generation, actual browser playback, download hash match and human sound/visual review are required before claiming end-to-end real generation success. Synthetic test video and jsdom component evidence do not satisfy that final gate.
@@ -0,0 +1,29 @@
import { z } from "zod";
export const ImageVideoDraftSchema = z.object({ attachmentId: z.string().regex(/^att_[\w-]+$/), title: z.string().trim().min(1).max(120), prompt: z.string().trim().min(1).max(4000), durationSeconds: z.number().positive().max(60) }).strict();
export type ImageVideoDraft = z.infer<typeof ImageVideoDraftSchema>;
export const taskIdSchema = z.string().regex(/^i2v_[0-9a-f-]{36}$/);
export const validationSchema = z.object({ videoSeconds: z.number().positive(), audioSeconds: z.number().positive(), width: z.number().int().positive(), height: z.number().int().positive(), fps: z.number().positive(), decoded: z.literal(true), sha256: z.string().regex(/^[a-f0-9]{64}$/), visualReview: z.literal("pending"), audioReview: z.literal("pending") }).strict();
export type ImageVideoValidation = z.infer<typeof validationSchema>;
export const recordSchema = z.object({
version: z.literal(1), id: taskIdSchema, ownerSessionId: z.string().regex(/^ses_[\w-]+$/), revision: z.number().int().nonnegative(),
draft: ImageVideoDraftSchema, requestHash: z.string().regex(/^[a-f0-9]{64}$/), referenceHash: z.string().regex(/^[a-f0-9]{64}$/), referenceType: z.enum(["image/jpeg", "image/png", "image/webp"]),
state: z.enum(["draft", "initializing_upload", "uploading", "finalizing", "importing", "awaiting_submission", "submitting", "queued", "running", "validating", "ready", "failed", "unknown_submission", "unknown_upload"]),
createdAt: z.string().datetime(), updatedAt: z.string().datetime(), lastPollAt: z.number().optional(),
identity: z.object({ tenantId: z.string().min(1), platform: z.string().min(1) }).strict().optional(),
upload: z.object({ uploadId: z.string().min(1), objectKey: z.string().min(1), uploadUrl: z.string().url() }).strict().optional(),
assetId: z.string().min(1).optional(), jobId: z.string().min(1).max(200).optional(), artifactUrl: z.string().url().optional(),
error: z.string().max(500).optional(), validation: validationSchema.optional(),
}).strict();
export type ImageVideoRecord = z.infer<typeof recordSchema>;
export type ImageVideoPublicTask = { id: string; title: string; prompt: string; durationSeconds: number; state: string; createdAt: string; updatedAt: string; referenceUrl: string; videoUrl?: string; jobId?: string; error?: string; validation?: ImageVideoValidation };
export interface ImageVideoRemote {
preflight(): Promise<{ tenantId: string; platform: string }>;
initialize(record: ImageVideoRecord, bytes: Buffer): Promise<{ uploadId: string; objectKey: string; uploadUrl: string }>;
put(record: ImageVideoRecord, bytes: Buffer): Promise<void>;
finalize(record: ImageVideoRecord): Promise<void>;
importStatus(record: ImageVideoRecord): Promise<{ state: "pending" | "ready" | "failed"; assetId?: string }>;
submit(record: ImageVideoRecord): Promise<{ jobId: string }>;
status(record: ImageVideoRecord): Promise<{ state: "queued" | "running" | "succeeded" | "failed"; artifactUrl?: string }>;
}
export const IMAGE_VIDEO_RESOURCES = ["references/selection.md", "references/swads.md", "references/local-processing.md", "scripts/media.py"];
+108
View File
@@ -0,0 +1,108 @@
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
import { z } from "zod";
import { Check } from "typebox/value";
import type { TSchema } from "typebox";
import { HarnessError } from "../errors/harness-error.js";
import { SWADS_MCP_URL } from "../tools/swads-mcp.js";
import type { ImageVideoRecord, ImageVideoRemote } from "./contracts.js";
import { imageVideoTransfer, validateImageVideoUrl } from "./media.js";
const names = ["swads_whoami", "creative_upload_from_url", "creative_import_status", "video_h3_submit", "video_job_status"] as const;
const nonempty = z.string().min(1).max(2048);
export function assertImageVideoArguments(schema: Record<string, unknown>, args: Record<string, unknown>): void {
const properties = z.record(z.string(), z.unknown()).parse(schema.properties ?? {});
if (Object.keys(args).some(key => !(key in properties)) || !Check(schema as TSchema, args)) throw new HarnessError("I2V_SCHEMA_CHANGED", "SW Ads request schema changed; operation blocked");
}
export function imageVideoResponse(value: unknown): Record<string, unknown> {
try {
const envelope = z.object({ isError: z.boolean().optional(), structuredContent: z.unknown().optional(), content: z.array(z.object({ type: z.string(), text: z.string().optional() })).optional() }).parse(value);
if (envelope.isError) throw new Error("Remote error");
const texts = envelope.content?.filter(part => part.type === "text");
const result = envelope.structuredContent ?? (texts?.length === 1 && texts[0]?.text ? JSON.parse(texts[0].text) as unknown : undefined);
return z.record(z.string(), z.unknown()).parse(result);
} catch { throw new HarnessError("I2V_RESPONSE_UNSUPPORTED", "SW Ads response contract is unclear; retain the task and inspect its receipt"); }
}
/** Dedicated capability set. The analytics client's allowlist is never broadened. */
export class ImageVideoMcpRemote implements ImageVideoRemote {
#connection: Promise<Client> | undefined;
#schemas = new Map<string, Record<string, unknown>>();
constructor(private readonly options: { token: string }) {}
private async connect() {
if (!this.options.token.trim()) throw new HarnessError("SWADS_NOT_CONFIGURED", "SWADS_MCP_TOKEN is not configured on the server");
this.#connection ??= (async () => {
const client = new Client({ name: "agent-studio-image-video", version: "1.0.0" });
try { await client.connect(new StreamableHTTPClientTransport(new URL(SWADS_MCP_URL), { requestInit: { redirect: "error", headers: { Authorization: `Bearer ${this.options.token}` } } }) as Transport, { timeout: 20_000 }); return client; }
catch { await client.close().catch(() => undefined); this.#connection = undefined; throw new HarnessError("I2V_CONNECTION_FAILED", "SW Ads connection failed"); }
})();
return this.#connection;
}
async close() { const client = await this.#connection?.catch(() => undefined); this.#connection = undefined; await client?.close(); }
async preflight() {
const client = await this.connect(); this.#schemas.clear();
let cursor: string | undefined;
do {
const page = await client.listTools(cursor ? { cursor } : {}, { timeout: 20_000 });
for (const tool of page.tools) if (names.some(name => name === tool.name)) this.#schemas.set(tool.name, tool.inputSchema);
cursor = page.nextCursor;
} while (cursor);
for (const name of names) if (!this.#schemas.has(name)) throw new HarnessError("I2V_CAPABILITY_MISSING", `Required capability unavailable: ${name}`);
const identity = z.object({ access: z.object({ current_account: z.object({ tenant_id: nonempty, platform: nonempty }) }) }).safeParse(await this.call("swads_whoami", {}));
if (!identity.success) throw new HarnessError("I2V_IDENTITY_AMBIGUOUS", "Current MCP tenant/platform is not unambiguous; upload blocked");
return { tenantId: identity.data.access.current_account.tenant_id, platform: identity.data.access.current_account.platform };
}
private async call(name: typeof names[number], args: Record<string, unknown>): Promise<Record<string, unknown>> {
if (!names.includes(name)) throw new HarnessError("I2V_TOOL_DENIED", "Tool unavailable");
const schema = this.#schemas.get(name);
// Status collection after restart discovers the same live capability set.
if (!schema) { await this.preflight(); return this.call(name, args); }
assertImageVideoArguments(schema, args);
try {
const result = await (await this.connect()).callTool({ name, arguments: args }, undefined, { timeout: 60_000 });
return imageVideoResponse(result);
} catch { throw new HarnessError("I2V_REMOTE_FAILED", "SW Ads request failed or its response is unsupported; inspect saved task state"); }
}
private uploadArgs(r: ImageVideoRecord) {
if (!r.identity) throw new HarnessError("I2V_IDENTITY_AMBIGUOUS", "Missing approved upload identity");
return { tenant_id: r.identity.tenantId, platforms: [r.identity.platform], platform: r.identity.platform, media_kind: "image", review_status: "review_pending", asset_role: "reference" };
}
async initialize(r: ImageVideoRecord, bytes: Buffer) {
const ext = r.referenceType.split("/")[1];
const receipt = z.object({ upload_id: nonempty, object_key: nonempty, upload_url: z.string().url() }).parse(await this.call("creative_upload_from_url", { ...this.uploadArgs(r), file_name: `${r.id}.${ext}`, content_type: r.referenceType, size_bytes: bytes.length, content_sha256: r.referenceHash }));
return { uploadId: receipt.upload_id, objectKey: receipt.object_key, uploadUrl: receipt.upload_url };
}
async put(r: ImageVideoRecord, bytes: Buffer) { if (!r.upload) throw new Error("Missing receipt"); await imageVideoTransfer(r.upload.uploadUrl, { bytes, contentType: r.referenceType }); }
async finalize(r: ImageVideoRecord) {
if (!r.upload) throw new Error("Missing receipt");
const result = await this.call("creative_upload_from_url", { ...this.uploadArgs(r), upload_id: r.upload.uploadId, object_key: r.upload.objectKey });
if (result.status !== "finalizing") throw new HarnessError("I2V_RESPONSE_UNSUPPORTED", "Finalization receipt is unclear; poll the saved upload ID");
}
async importStatus(r: ImageVideoRecord) {
if (!r.upload) throw new Error("Missing receipt");
const value = await this.call("creative_import_status", { upload_id: r.upload.uploadId });
if (typeof value.creative_factory_asset_id === "string" && value.creative_factory_asset_id) return { state: "ready" as const, assetId: value.creative_factory_asset_id };
if (value.status === "failed") return { state: "failed" as const };
if (["pending", "finalizing", "processing"].includes(String(value.status))) return { state: "pending" as const };
throw new HarnessError("I2V_RESPONSE_UNSUPPORTED", "Import response has no unambiguous asset receipt");
}
async submit(r: ImageVideoRecord) {
if (!r.assetId) throw new Error("Missing reference asset");
const result = z.object({ job_id: nonempty }).parse(await this.call("video_h3_submit", { prompt: r.draft.prompt, title: r.draft.title, reference_asset_ids: [r.assetId], aspect_ratio: "9:16", duration_seconds: r.draft.durationSeconds, resolution: "768p", h3_mode: "ref2va", idempotency_key: `${r.id}/${r.requestHash}`, skill_id: "swads-image-to-video" }));
return { jobId: result.job_id };
}
async status(r: ImageVideoRecord) {
if (!r.jobId) throw new Error("Missing job");
const value = await this.call("video_job_status", { job_id: r.jobId });
if (value.job_id !== undefined && value.job_id !== r.jobId) throw new HarnessError("I2V_RESPONSE_UNSUPPORTED", "Returned job does not match registered job");
if (["pending", "queued"].includes(String(value.status))) return { state: "queued" as const };
if (["running", "processing"].includes(String(value.status))) return { state: "running" as const };
if (["failed", "rejected", "cancelled"].includes(String(value.status))) return { state: "failed" as const };
if (!["succeeded", "completed", "success"].includes(String(value.status))) throw new HarnessError("I2V_RESPONSE_UNSUPPORTED", "Unknown generation state");
const artifacts = z.array(z.object({ content_type: z.string(), url: z.string().url() })).parse(value.artifacts).filter(artifact => artifact.content_type === "video/mp4");
if (artifacts.length !== 1 || !artifacts[0]) throw new HarnessError("I2V_RESPONSE_UNSUPPORTED", "Expected one registered MP4 artifact");
await validateImageVideoUrl(artifacts[0].url);
return { state: "succeeded" as const, artifactUrl: artifacts[0].url };
}
}
+75
View File
@@ -0,0 +1,75 @@
import { request } from "node:https";
import { isIP } from "node:net";
import { readFile, writeFile, rename } from "node:fs/promises";
import { join } from "node:path";
import { z } from "zod";
import { HarnessError } from "../errors/harness-error.js";
import { validateBaseUrl, type AddressResolver } from "../models/base-url-policy.js";
import { mediaRuntime, runMedia } from "../media/media-worker.js";
import { validationSchema } from "./contracts.js";
import { sha256 } from "./service.js";
export async function validateImageVideoUrl(value: string, resolver?: AddressResolver) {
let url: URL;
try { url = new URL(value); } catch { throw new HarnessError("I2V_URL_DENIED", "Invalid media URL"); }
// The storage provider may require a signed query. Only remove it from host validation.
const hostOnly = new URL(url); hostOnly.search = "";
const checked = await validateBaseUrl(hostOnly.href, {}, resolver);
const address = checked.addresses[0];
if (!address) throw new HarnessError("I2V_URL_DENIED", "No public media address");
return { url, address };
}
/** HTTPS with DNS pinned to the validated address; redirects and oversized bodies fail closed. */
export async function imageVideoTransfer(value: string, options: { bytes?: Buffer; contentType?: string; maxBytes?: number } = {}): Promise<Buffer> {
const { url, address } = await validateImageVideoUrl(value);
return new Promise((resolve, reject) => {
const error = () => new HarnessError("I2V_TRANSFER_FAILED", "Media transfer failed or exceeded its bounds");
const req = request(url, {
family: isIP(address),
method: options.bytes ? "PUT" : "GET",
headers: options.bytes ? { "Content-Type": options.contentType ?? "application/octet-stream", "Content-Length": options.bytes.length } : {},
lookup: (_hostname, _options, callback) => callback(null, address, isIP(address)),
}, response => {
if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 300) { response.destroy(); req.destroy(error()); return; }
const limit = options.maxBytes ?? (options.bytes ? 1024 * 1024 : 256 * 1024 * 1024);
if (Number(response.headers["content-length"] ?? 0) > limit) { response.destroy(); req.destroy(error()); return; }
if (!options.bytes && !/^video\/mp4(?:;|$)/i.test(response.headers["content-type"] ?? "")) { response.destroy(); req.destroy(error()); return; }
let total = 0; const chunks: Buffer[] = [];
response.on("data", (chunk: Buffer) => { total += chunk.length; if (total > limit) req.destroy(error()); else chunks.push(chunk); });
response.once("error", () => reject(error()));
response.once("end", () => { clearTimeout(timer); resolve(Buffer.concat(chunks)); });
});
const timer = setTimeout(() => req.destroy(error()), 120_000);
req.once("error", () => { clearTimeout(timer); reject(error()); });
req.end(options.bytes);
});
}
export function parseImageVideoValidation(value: unknown, duration: number) {
const raw = z.object({ video_seconds: z.number().positive(), audio_seconds: z.number().positive(), width: z.number().positive(), height: z.number().positive(), fps: z.string(), decoded: z.literal(true), sha256: z.string() }).parse(value);
const parts = raw.fps.split("/"); const fps = Number(parts[0]) / Number(parts[1] ?? 1);
if (Math.abs(raw.video_seconds - duration) > 0.002 || Math.abs(raw.audio_seconds - duration) > 0.05 || Math.abs(raw.width / raw.height - 9 / 16) > 0.01 || fps > 120) throw new HarnessError("I2V_VALIDATION_FAILED", "H3 output does not match approved duration or portrait dimensions");
return validationSchema.parse({ videoSeconds: raw.video_seconds, audioSeconds: raw.audio_seconds, width: raw.width, height: raw.height, fps, decoded: true, sha256: raw.sha256, visualReview: "pending", audioReview: "pending" });
}
export async function verifyImageVideo(path: string, directory: string, duration: number, options: { skillRoot: string; python: string }) {
const { ffmpeg, ffprobe } = await mediaRuntime();
// Patch the module object used by the unmodified original Skill. Every subprocess is
// restricted, including ffprobe, full decode and the audio-peak ffmpeg invocation.
const wrapper = `import runpy,sys,subprocess\noriginal=subprocess.run\nallowed=set(sys.argv[2:4])\ndef local_run(command,*args,**kwargs):\n if not isinstance(command,list) or command[0] not in allowed: raise RuntimeError('Unexpected media command')\n return original([command[0],'-protocol_whitelist','file,pipe','-format_whitelist','mov']+command[1:],*args,**kwargs)\nsubprocess.run=local_run\nscript=sys.argv[1]\nsys.argv=[script,'--ffmpeg',sys.argv[2],'--ffprobe',sys.argv[3],'verify','--input',sys.argv[4],'--minimum',sys.argv[5]]\nrunpy.run_path(script,run_name='__main__')`;
const result: unknown = JSON.parse(await runMedia(options.python, ["-c", wrapper, join(options.skillRoot, "scripts/media.py"), ffmpeg, ffprobe, path, String(duration)], directory, new AbortController().signal, 180_000));
return parseImageVideoValidation(result, duration);
}
export function createImageVideoCollector(options: { skillRoot: string; python: string }) {
return async (url: string, directory: string, duration: number) => {
const bytes = await imageVideoTransfer(url);
if (bytes.length < 16 || bytes.subarray(4, 8).toString("ascii") !== "ftyp") throw new HarnessError("I2V_INVALID_MP4", "Artifact is not an MP4 file");
const path = join(directory, "candidate.mp4"); await writeFile(path, bytes, { mode: 0o600 });
const validation = await verifyImageVideo(path, directory, duration, options);
if (validation.sha256 !== sha256(await readFile(path))) throw new HarnessError("I2V_INVALID_MP4", "Validated media changed");
await rename(path, join(directory, "video.mp4"));
return validation;
};
}
+147
View File
@@ -0,0 +1,147 @@
import { createHash, randomUUID } from "node:crypto";
import { lstat, mkdir, open, readFile, readdir, rename, stat } from "node:fs/promises";
import { join, resolve } from "node:path";
import type { AttachmentStore } from "../attachments/attachment-store.js";
import { HarnessError } from "../errors/harness-error.js";
import { ImageVideoDraftSchema, recordSchema, taskIdSchema, validationSchema, type ImageVideoDraft, type ImageVideoPublicTask, type ImageVideoRecord, type ImageVideoRemote, type ImageVideoValidation } from "./contracts.js";
const locks = new Map<string, Promise<unknown>>();
export const sha256 = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex");
const fail = (code: string, message: string): never => { throw new HarnessError(code, message); };
export type ImageVideoOptions = { root: string; attachments: Pick<AttachmentStore, "get" | "openNormalized">; remote: ImageVideoRemote; collect: (url: string, directory: string, duration: number) => Promise<ImageVideoValidation> };
/** One process owns this private store. Per-task locks are shared by service instances. */
export class ImageVideoService {
readonly root: string;
constructor(private readonly options: ImageVideoOptions) { this.root = resolve(options.root); }
async create(sessionId: string, input: ImageVideoDraft): Promise<ImageVideoPublicTask> {
const draft = ImageVideoDraftSchema.parse(input);
const attachment = this.options.attachments.get(sessionId, draft.attachmentId);
if (!attachment || attachment.sessionId !== sessionId) return fail("I2V_TASK_NOT_FOUND", "Reference image not found in this Session");
const bytes = await this.options.attachments.openNormalized(sessionId, draft.attachmentId);
const now = new Date().toISOString(); const id = `i2v_${randomUUID()}`;
const record = recordSchema.parse({ version: 1, id, ownerSessionId: sessionId, revision: 0, draft, referenceHash: sha256(bytes), requestHash: sha256(JSON.stringify(draft)), referenceType: attachment.mediaType, state: "draft", createdAt: now, updatedAt: now });
await mkdir(join(this.root, id), { recursive: true, mode: 0o700 });
await this.atomic(join(this.root, id, "reference"), bytes);
await this.atomic(join(this.root, id, "record.json"), JSON.stringify(record));
return this.public(record);
}
async list(): Promise<ImageVideoPublicTask[]> {
const entries = await readdir(this.root).catch(() => []);
const tasks = await Promise.all(entries.filter(id => taskIdSchema.safeParse(id).success).map(id => this.view(id).catch(() => undefined)));
return tasks.filter((task): task is ImageVideoPublicTask => task !== undefined).sort((a, b) => b.createdAt.localeCompare(a.createdAt));
}
async view(id: string): Promise<ImageVideoPublicTask> { return this.lock(id, async () => this.public(await this.read(id))); }
async owned(sessionId: string, id: string): Promise<ImageVideoPublicTask> { return this.lock(id, async () => { const r = await this.read(id); this.owner(r, sessionId); return this.public(r); }); }
async upload(sessionId: string, id: string): Promise<ImageVideoPublicTask> {
return this.lock(id, async () => {
const r = await this.read(id); this.owner(r, sessionId);
if (r.state === "unknown_upload") return fail("I2V_UNKNOWN_UPLOAD", "Upload outcome unknown; inspect the saved receipt before any new import");
if (r.state !== "draft") return this.public(r);
// Read-only discovery and identity validation happen before any remote mutation intent.
r.identity = await this.options.remote.preflight();
const bytes = await this.reference(r);
r.state = "initializing_upload"; await this.save(r);
try {
r.upload = await this.options.remote.initialize(r, bytes); r.state = "uploading"; await this.save(r);
await this.options.remote.put(r, bytes); r.state = "finalizing"; await this.save(r);
await this.options.remote.finalize(r); r.state = "importing"; await this.save(r);
} catch { r.state = "unknown_upload"; r.error = "参考图上传或入库结果未知;禁止重复导入。请核实本任务收据。"; await this.save(r); }
return this.public(r);
});
}
async submit(sessionId: string, id: string): Promise<ImageVideoPublicTask> {
return this.lock(id, async () => {
const r = await this.read(id); this.owner(r, sessionId);
if (r.jobId) return this.public(r);
if (r.state === "unknown_submission") return fail("I2V_UNKNOWN_SUBMISSION", "Submission outcome unknown; no safe retry or new billing request is allowed");
if (r.state !== "awaiting_submission" || !r.assetId) return fail("I2V_NOT_READY", "Reference import must be ready before generation approval");
await this.options.remote.preflight(); await this.reference(r);
r.state = "submitting"; await this.save(r);
try {
// Deliberately independent of chat cancellation: retain any returned receipt.
const receipt = await this.options.remote.submit(r);
if (!receipt.jobId) throw new Error("Missing job ID");
await this.atomic(join(this.root, id, "submission-receipt.json"), JSON.stringify({ version: 1, id, revision: r.revision, requestHash: r.requestHash, jobId: receipt.jobId }));
r.jobId = receipt.jobId; r.state = "queued"; delete r.lastPollAt; await this.save(r);
} catch { r.state = "unknown_submission"; r.error = "生成提交结果未知;禁止重投。请人工核实,避免重复计费。"; await this.save(r); }
return this.public(r);
});
}
async refresh(id: string): Promise<ImageVideoPublicTask> {
return this.lock(id, async () => {
const r = await this.read(id);
if (r.state === "ready" || r.state === "failed" || Date.now() - (r.lastPollAt ?? 0) < 30_000) return this.public(r);
if (!r.jobId && !(r.upload && ["importing", "unknown_upload"].includes(r.state))) return this.public(r);
r.lastPollAt = Date.now(); await this.save(r);
try {
if (!r.jobId) {
const result = await this.options.remote.importStatus(r);
if (result.state === "ready" && result.assetId) { r.assetId = result.assetId; r.state = "awaiting_submission"; delete r.error; }
else if (result.state === "failed") { r.state = "failed"; r.error = "参考图入库失败。"; }
} else {
const result = await this.options.remote.status(r);
if (result.state === "succeeded") {
if (!result.artifactUrl) throw new Error("No unambiguous MP4 artifact");
r.artifactUrl = result.artifactUrl; r.state = "validating"; await this.save(r);
r.validation = validationSchema.parse(await this.options.collect(result.artifactUrl, join(this.root, id), r.draft.durationSeconds));
if (Math.abs(r.validation.videoSeconds - r.draft.durationSeconds) > 0.002 || Math.abs(r.validation.audioSeconds - r.draft.durationSeconds) > 0.05) throw new Error("Duration mismatch");
r.state = "ready"; delete r.error;
} else { r.state = result.state; if (result.state === "failed") r.error = "H3 生成失败;未自动重投。"; }
}
} catch {
if (r.state === "validating") r.state = "failed";
r.error = r.state === "failed" ? "成片技术验证失败:请核实 MP4、精确时长、尺寸、帧率和音轨;未自动重投。" : "状态响应暂不可用或契约不明确;保留原任务,下次只读查询。";
}
await this.save(r); return this.public(r);
});
}
async media(id: string, kind: "reference" | "video") {
return this.lock(id, async () => {
const r = await this.read(id);
if (kind === "video" && (r.state !== "ready" || !r.validation)) return fail("I2V_MEDIA_NOT_FOUND", "Validated video not ready");
const path = join(this.root, id, kind === "reference" ? "reference" : "video.mp4");
await this.regular(path); const bytes = await readFile(path);
if (sha256(bytes) !== (kind === "reference" ? r.referenceHash : r.validation?.sha256)) return fail("I2V_MEDIA_INVALID", "Registered media hash mismatch");
return { path, size: (await stat(path)).size, contentType: kind === "reference" ? r.referenceType : "video/mp4" };
});
}
private owner(r: ImageVideoRecord, sessionId: string) { if (r.ownerSessionId !== sessionId) fail("I2V_TASK_NOT_FOUND", "Task not found in this Session"); }
private public(r: ImageVideoRecord): ImageVideoPublicTask {
return { id: r.id, title: r.draft.title, prompt: r.draft.prompt, durationSeconds: r.draft.durationSeconds, state: r.state, createdAt: r.createdAt, updatedAt: r.updatedAt, referenceUrl: `/api/image-video/tasks/${r.id}/reference`, ...(r.jobId ? { jobId: r.jobId } : {}), ...(r.error ? { error: r.error } : {}), ...(r.validation ? { validation: r.validation } : {}), ...(r.state === "ready" ? { videoUrl: `/api/image-video/tasks/${r.id}/video` } : {}) };
}
private async reference(r: ImageVideoRecord) { await this.regular(join(this.root, r.id, "reference")); const bytes = await readFile(join(this.root, r.id, "reference")); if (sha256(bytes) !== r.referenceHash) fail("I2V_MEDIA_INVALID", "Reference changed after draft"); return bytes; }
private async regular(path: string) { const info = await lstat(path); if (!info.isFile() || info.isSymbolicLink()) fail("I2V_RECORD_INVALID", "Unsafe registered file"); }
private async read(id: string): Promise<ImageVideoRecord> {
if (!taskIdSchema.safeParse(id).success) return fail("I2V_TASK_NOT_FOUND", "Task not found");
const path = join(this.root, id, "record.json");
const dir = await lstat(join(this.root, id)).catch(() => undefined);
if (!dir || !dir.isDirectory() || dir.isSymbolicLink()) return fail("I2V_TASK_NOT_FOUND", "Task not found");
try {
await this.regular(path); const r = recordSchema.parse(JSON.parse(await readFile(path, "utf8")));
if (r.id !== id || r.requestHash !== sha256(JSON.stringify(r.draft))) throw new Error("Binding mismatch");
if (r.state === "submitting") { r.state = "unknown_submission"; await this.save(r); }
if (["initializing_upload", "uploading", "finalizing"].includes(r.state)) { r.state = "unknown_upload"; await this.save(r); }
return r;
} catch { return fail("I2V_RECORD_INVALID", "Task record is invalid; manual recovery required"); }
}
private async save(r: ImageVideoRecord) {
const path = join(this.root, r.id, "record.json");
const previous = recordSchema.parse(JSON.parse(await readFile(path, "utf8")));
if (previous.revision !== r.revision) return fail("I2V_STALE_REVISION", "Stale task result rejected");
r.revision++; r.updatedAt = new Date().toISOString();
await this.atomic(path, JSON.stringify(recordSchema.parse(r)));
}
private async atomic(path: string, content: Buffer | string) { const temp = `${path}.${randomUUID()}.tmp`; const file = await open(temp, "wx", 0o600); try { await file.writeFile(content); await file.sync(); } finally { await file.close(); } await rename(temp, path); }
private async lock<T>(id: string, operation: () => Promise<T>): Promise<T> {
const key = `${this.root}/${id}`; const previous = locks.get(key) ?? Promise.resolve();
const work = previous.catch(() => undefined).then(operation); locks.set(key, work);
try { return await work; } finally { if (locks.get(key) === work) locks.delete(key); }
}
}
+13
View File
@@ -0,0 +1,13 @@
import { defineTool } from "@earendil-works/pi-coding-agent";
import { Type } from "typebox";
import type { ImageVideoService } from "./service.js";
const task = Type.Object({ taskId: Type.String({ pattern: "^i2v_[0-9a-f-]{36}$" }) }, { additionalProperties: false });
export function createImageVideoTools(service: ImageVideoService, sessionId: string) {
const output = (details: unknown) => ({ content: [{ type: "text" as const, text: JSON.stringify(details) }], details });
return [
defineTool({ name: "i2v_create_task", label: "Prepare local image video draft", description: "Read swads-image-to-video SKILL.md and references first. Copy this Session's existing attachment into a durable local draft. No external transmission. Use the user's exact title, prompt and duration; default portrait 9:16, H3 ref2va,768p. A form-created taskId already exists: do not create it again. Image display does not send it to the chat model. Never require CMS.", parameters: Type.Object({ attachmentId: Type.String(), title: Type.String(), prompt: Type.String(), durationSeconds: Type.Number() }, { additionalProperties: false }), async execute(_id, args) { return output(await service.create(sessionId, args)); } }),
defineTool({ name: "i2v_upload_reference", label: "Approve reference transfer", description: "Separate explicit page approval: transfer this registered draft's frozen image to SW Ads storage and create review_pending, unassigned reference asset. No account assignment or publishing. Arguments accept only taskId. Refusal performs no transfer. Wait for import status before requesting generation approval.", parameters: task, async execute(_id, args) { return output(await service.upload(sessionId, args.taskId)); } }),
defineTool({ name: "i2v_submit", label: "Approve H3 generation", description: "Separate explicit page approval: submit this registered task's exact frozen prompt, reference asset and target duration once to H3 ref2va,9:16,768p. Cost unknown; target duration support unverified. Never modify duration, retry unknown submission, or switch provider. Returns durable jobId immediately; UI polls the original job without an LLM. Technical preview is not human-reviewed.", parameters: task, async execute(_id, args) { return output(await service.submit(sessionId, args.taskId)); } }),
defineTool({ name: "i2v_status", label: "Read registered image video task", description: "Read only this Session's registered task. Poll at most every 30 seconds; preserve queued or unknown state. Does not submit or approve. The webpage restores task history after restart.", parameters: task, async execute(_id, args) { await service.owned(sessionId, args.taskId); return output(await service.refresh(args.taskId)); } }),
];
}
@@ -0,0 +1,31 @@
import { describe, expect, it } from "vitest";
import * as harness from "../src/index.js";
describe("image video remote boundary", () => {
it("rejects incompatible live tool input types and new required fields before calling them", () => {
expect(harness).toHaveProperty("assertImageVideoArguments");
const schema = { type: "object", properties: { duration_seconds: { type: "number" }, reference_asset_ids: { anyOf: [{ type: "array", items: { type: "string" } }, { type: "null" }] } }, required: ["duration_seconds"] };
expect(() => harness.assertImageVideoArguments(schema, { duration_seconds: 10, reference_asset_ids: ["asset_1"] })).not.toThrow();
expect(() => harness.assertImageVideoArguments(schema, { duration_seconds: "10" })).toThrow();
expect(() => harness.assertImageVideoArguments(schema, { duration_seconds: 10, external_account_id: "unapproved" })).toThrow();
expect(() => harness.assertImageVideoArguments({ ...schema, required: ["new_field"] }, { duration_seconds: 10 })).toThrow();
});
it("accepts signed HTTPS queries privately but rejects credentials, private DNS and localhost", async () => {
expect(harness).toHaveProperty("validateImageVideoUrl");
const result = await harness.validateImageVideoUrl("https://cdn.example/video?token=private", () => Promise.resolve([{ address: "8.8.8.8", family: 4 }]));
expect(result.url.search).toBe("?token=private"); expect(result.address).toBe("8.8.8.8");
for (const url of ["http://cdn.example/v", "https://user:pass@cdn.example/v", "https://127.0.0.1/v", "https://[::ffff:127.0.0.1]/v"]) await expect(harness.validateImageVideoUrl(url)).rejects.toBeDefined();
await expect(harness.validateImageVideoUrl("https://cdn.example/v", () => Promise.resolve([{ address: "192.168.1.2", family: 4 }]))).rejects.toBeDefined();
});
it("extracts only the documented response envelope and rejects malformed or error responses", () => {
expect(harness).toHaveProperty("imageVideoResponse");
expect(harness.imageVideoResponse({ content: [{ type: "text", text: '{"job_id":"j1"}' }] })).toEqual({ job_id: "j1" });
for (const value of [{ isError: true, content: [{ type: "text", text: "token=private" }] }, { content: [{ type: "text", text: "not JSON" }] }, { structuredContent: [] }]) expect(() => harness.imageVideoResponse(value)).toThrow();
});
it("refuses a false-duration or incomplete technical validation instead of releasing a video", () => {
expect(harness).toHaveProperty("parseImageVideoValidation");
const result = { video_seconds: 10, audio_seconds: 10, width: 720, height: 1280, fps: "24", decoded: true, sha256: "a".repeat(64) };
expect(harness.parseImageVideoValidation(result, 10).visualReview).toBe("pending");
for (const changed of [{ video_seconds: 7.292 }, { audio_seconds: 0 }, { decoded: false }, { width: 1280, height: 720 }, { fps: "0" }]) expect(() => harness.parseImageVideoValidation({ ...result, ...changed }, 10)).toThrow();
});
});
@@ -0,0 +1,12 @@
import { describe, expect, it } from "vitest";
import * as harness from "../src/index.js";
describe("image video tool permissions", () => {
it("separates local draft/status from separately approved upload and generation", () => {
expect(harness.decideToolPolicy("i2v_create_task")).toBe("allow");
expect(harness.decideToolPolicy("i2v_status")).toBe("allow");
expect(harness.decideToolPolicy("i2v_upload_reference")).toBe("ask");
expect(harness.decideToolPolicy("i2v_submit")).toBe("ask");
expect(harness.decideToolPolicy("video_h3_submit")).toBe("deny");
expect(harness.decideToolPolicy("creative_upload_from_url")).toBe("deny");
});
});
@@ -0,0 +1,33 @@
import { EventEmitter } from "node:events";
import { describe, expect, it, vi } from "vitest";
const response = vi.hoisted(() => ({ status: 200, type: "video/mp4", body: Buffer.from("mock-MP4"), connectedAddress: "" }));
vi.mock("node:dns/promises", () => ({ lookup: () => Promise.resolve([{ address: "8.8.8.8", family: 4 }]) }));
vi.mock("node:https", () => ({ request: (_url: URL, options: { family?: number; lookup: (name: string, options: { all?: boolean }, callback: (error: Error | null, addresses: string | { address: string }[]) => void) => void }, receive: (value: EventEmitter) => void) => {
const req = new EventEmitter() as EventEmitter & { end: () => void; destroy: (error?: Error) => void };
let destroyed = false;
req.destroy = (error) => { destroyed = true; if (error) queueMicrotask(() => req.emit("error", error)); };
req.end = () => queueMicrotask(() => {
// Node 22 requests all addresses unless a concrete address family is supplied.
const all = options.family !== 4 && options.family !== 6;
options.lookup("cdn.example", { all }, (error, addresses) => {
if (error || (all && !Array.isArray(addresses))) { req.destroy(new Error("Invalid IP callback shape")); return; }
response.connectedAddress = typeof addresses === "string" ? addresses : addresses[0]?.address ?? "";
const incoming = Object.assign(new EventEmitter(), { statusCode: response.status, headers: { "content-type": response.type, "content-length": response.body.length }, destroy() {} });
receive(incoming); if (!destroyed) { incoming.emit("data", response.body); incoming.emit("end"); }
});
});
return req;
} }));
import { imageVideoTransfer } from "../src/image-video/media.js";
describe("bounded pinned HTTPS media transfer", () => {
it("connects with the checked public IP using Node's actual lookup callback contract", async () => {
response.status = 200; response.type = "video/mp4";
expect(await imageVideoTransfer("https://cdn.example/video?token=private")).toEqual(Buffer.from("mock-MP4"));
expect(response.connectedAddress).toBe("8.8.8.8");
});
it("rejects redirects, HTML masquerading as a video and oversized bodies", async () => {
response.status = 302; await expect(imageVideoTransfer("https://cdn.example/video")).rejects.toThrow();
response.status = 200; response.type = "text/html"; await expect(imageVideoTransfer("https://cdn.example/video")).rejects.toThrow();
response.type = "video/mp4"; await expect(imageVideoTransfer("https://cdn.example/video", { maxBytes: 2 })).rejects.toThrow();
});
});
@@ -0,0 +1,98 @@
import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { Readable } from "node:stream";
import sharp from "sharp";
import { describe, expect, it } from "vitest";
import * as harness from "../src/index.js";
import type { ImageVideoRemote, ImageVideoRecord } from "../src/image-video/contracts.js";
async function fixture() {
expect(harness).toHaveProperty("ImageVideoService");
const root = await mkdtemp(join(tmpdir(), "i2v-test-"));
await mkdir(join(root, "ses_owner"));
const attachments = new harness.FileAttachmentStore({ sessionsRoot: root });
const attachment = await attachments.add("ses_owner", { name: "test.png", mediaType: "image/png", stream: Readable.from(await sharp({ create: { width: 8, height: 8, channels: 3, background: "red" } }).png().toBuffer()) });
const calls: string[] = [];
const remote: ImageVideoRemote = {
preflight() { calls.push("identity"); return Promise.resolve({ tenantId: "tenant", platform: "tiktok" }); },
initialize() { calls.push("initialize"); return Promise.resolve({ uploadId: "upload_1", objectKey: "obj", uploadUrl: "https://public.example/put?token=private" }); },
put() { calls.push("put"); return Promise.resolve(); }, finalize() { calls.push("finalize"); return Promise.resolve(); },
importStatus() { calls.push("import"); return Promise.resolve({ state: "ready", assetId: "asset_1" }); },
submit() { calls.push("submit"); return Promise.resolve({ jobId: "remote_job_1" }); },
status() { calls.push("status"); return Promise.resolve({ state: "queued" }); },
};
const options = { root: join(root, "tasks"), attachments, remote, collect: () => Promise.reject(new Error("No fake video")) };
const service = new harness.ImageVideoService(options);
const draft = { attachmentId: attachment.attachmentId, title: "Motion", prompt: "Natural movement with ambient sound", durationSeconds: 10 };
return { service, options, draft, calls, remote, root };
}
describe("durable image video tasks", () => {
it("copies a local reference without transmitting it and hides private bindings", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
expect(f.calls).toEqual([]); expect(task.state).toBe("draft");
await f.options.attachments.removeAll("ses_owner");
expect((await f.service.media(task.id, "reference")).size).toBeGreaterThan(0);
expect(JSON.stringify(await f.service.list())).not.toMatch(/ownerSessionId|uploadUrl|objectKey|referenceHash|\/private\//);
});
it("rejects foreign owners before any external operation", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
await expect(f.service.upload("ses_other", task.id)).rejects.toMatchObject({ code: "I2V_TASK_NOT_FOUND" });
await expect(f.service.submit("ses_other", task.id)).rejects.toMatchObject({ code: "I2V_TASK_NOT_FOUND" });
expect(f.calls).toEqual([]);
});
it("requires upload first and coalesces concurrent submission, restoring only the known job", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
await expect(f.service.submit("ses_owner", task.id)).rejects.toMatchObject({ code: "I2V_NOT_READY" });
await f.service.upload("ses_owner", task.id); await f.service.refresh(task.id);
const [a, b] = await Promise.all([f.service.submit("ses_owner", task.id), f.service.submit("ses_owner", task.id)]);
expect(a.jobId).toBe("remote_job_1"); expect(b.jobId).toBe(a.jobId);
const restored = new harness.ImageVideoService(f.options);
expect((await restored.view(task.id)).jobId).toBe(a.jobId);
await restored.submit("ses_owner", task.id);
expect(f.calls.filter(call => call === "submit")).toHaveLength(1);
expect(JSON.stringify(await restored.view(task.id))).not.toContain("private");
});
it("isolates unknown submissions after timeout and after process interruption", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
await f.service.upload("ses_owner", task.id); await f.service.refresh(task.id);
f.remote.submit = () => { f.calls.push("submit"); return Promise.reject(new Error("token=private")); };
expect((await f.service.submit("ses_owner", task.id)).state).toBe("unknown_submission");
await expect(new harness.ImageVideoService(f.options).submit("ses_owner", task.id)).rejects.toMatchObject({ code: "I2V_UNKNOWN_SUBMISSION" });
expect(f.calls.filter(call => call === "submit")).toHaveLength(1);
const file = join(f.options.root, task.id, "record.json"); const record = JSON.parse(await readFile(file, "utf8")) as ImageVideoRecord;
record.state = "submitting"; await writeFile(file, JSON.stringify(record));
expect((await new harness.ImageVideoService(f.options).view(task.id)).state).toBe("unknown_submission");
});
it("never reinitializes uncertain uploads and blocks stale persisted revisions", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
f.remote.initialize = () => { f.calls.push("initialize"); return Promise.reject(new Error("timeout")); };
expect((await f.service.upload("ses_owner", task.id)).state).toBe("unknown_upload");
await expect(f.service.upload("ses_owner", task.id)).rejects.toMatchObject({ code: "I2V_UNKNOWN_UPLOAD" });
expect(f.calls.filter(call => call === "initialize")).toHaveLength(1);
const file = join(f.options.root, task.id, "record.json"); const record = JSON.parse(await readFile(file, "utf8")) as ImageVideoRecord;
record.requestHash = "invalid"; await writeFile(file, JSON.stringify(record));
await expect(f.service.view(task.id)).rejects.toMatchObject({ code: "I2V_RECORD_INVALID" });
});
it("throttles read-only polling even across instances and accepts no arbitrary identifiers", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
await f.service.upload("ses_owner", task.id); await f.service.refresh(task.id); await f.service.submit("ses_owner", task.id);
await Promise.all([f.service.refresh(task.id), new harness.ImageVideoService(f.options).refresh(task.id)]);
expect(f.calls.filter(call => call === "status")).toHaveLength(1);
await expect(f.service.view("../../etc/passwd")).rejects.toMatchObject({ code: "I2V_TASK_NOT_FOUND" });
});
it("rejects a late submit result from overwriting a newer revision, while retaining its known receipt", async () => {
const f = await fixture(); const task = await f.service.create("ses_owner", f.draft);
await f.service.upload("ses_owner", task.id); await f.service.refresh(task.id);
f.remote.submit = async () => {
const file = join(f.options.root, task.id, "record.json"); const newer = JSON.parse(await readFile(file, "utf8")) as ImageVideoRecord;
newer.revision += 1; newer.state = "unknown_submission"; await writeFile(file, JSON.stringify(newer));
return { jobId: "late_job" };
};
await expect(f.service.submit("ses_owner", task.id)).rejects.toMatchObject({ code: "I2V_STALE_REVISION" });
expect((await f.service.view(task.id)).state).toBe("unknown_submission");
const receipt = JSON.parse(await readFile(join(f.options.root, task.id, "submission-receipt.json"), "utf8")) as { jobId: string };
expect(receipt.jobId).toBe("late_job");
});
});