From 40e03e393c53665eb1da1ae3456985c04acd7d65 Mon Sep 17 00:00:00 2001 From: Wenxin Duan Date: Tue, 15 Sep 2026 01:04:09 +0800 Subject: [PATCH] feat: add durable CMS-free image video subsystem and UI components --- apps/server/src/routes/image-video.ts | 49 ++++++ apps/server/tests/image-video-api.test.ts | 44 ++++++ .../src/components/tools/ImageVideoPanel.tsx | 86 ++++++++++ apps/web/tests/image-video.test.tsx | 53 +++++++ docs/image-to-video-web.md | 17 ++ packages/harness/src/image-video/contracts.ts | 29 ++++ packages/harness/src/image-video/mcp.ts | 108 +++++++++++++ packages/harness/src/image-video/media.ts | 75 +++++++++ packages/harness/src/image-video/service.ts | 147 ++++++++++++++++++ packages/harness/src/image-video/tools.ts | 13 ++ .../tests/image-video-boundary.test.ts | 31 ++++ .../harness/tests/image-video-tools.test.ts | 12 ++ .../tests/image-video-transfer.test.ts | 33 ++++ packages/harness/tests/image-video.test.ts | 98 ++++++++++++ 14 files changed, 795 insertions(+) create mode 100644 apps/server/src/routes/image-video.ts create mode 100644 apps/server/tests/image-video-api.test.ts create mode 100644 apps/web/src/components/tools/ImageVideoPanel.tsx create mode 100644 apps/web/tests/image-video.test.tsx create mode 100644 docs/image-to-video-web.md create mode 100644 packages/harness/src/image-video/contracts.ts create mode 100644 packages/harness/src/image-video/mcp.ts create mode 100644 packages/harness/src/image-video/media.ts create mode 100644 packages/harness/src/image-video/service.ts create mode 100644 packages/harness/src/image-video/tools.ts create mode 100644 packages/harness/tests/image-video-boundary.test.ts create mode 100644 packages/harness/tests/image-video-tools.test.ts create mode 100644 packages/harness/tests/image-video-transfer.test.ts create mode 100644 packages/harness/tests/image-video.test.ts diff --git a/apps/server/src/routes/image-video.ts b/apps/server/src/routes/image-video.ts new file mode 100644 index 0000000..04ad844 --- /dev/null +++ b/apps/server/src/routes/image-video.ts @@ -0,0 +1,49 @@ +import { createReadStream } from "node:fs"; +import { HarnessError, ImageVideoDraftSchema, taskIdSchema } from "@agent-studio/harness"; +import type { FastifyInstance, FastifyRequest } from "fastify"; +import { z } from "zod"; +import type { AppDependencies } from "../app.js"; +import { requireSession } from "./sessions.js"; + +export function registerImageVideoRoutes(app: FastifyInstance, dependencies: AppDependencies) { + const context = (request: FastifyRequest) => { + const host = request.headers.host ?? ""; + const ports = dependencies.imageVideoPorts ?? [80, 3001, 5173, 4173]; + let url: URL; + try { url = new URL(`http://${host}`); } catch { throw new HarnessError("I2V_ORIGIN_DENIED", "Local origin required"); } + if (!["localhost", "127.0.0.1", "[::1]"].includes(url.hostname) || !ports.includes(Number(url.port || 80)) || request.headers["sec-fetch-site"] === "cross-site" || (request.headers.origin !== undefined && request.headers.origin !== url.origin)) throw new HarnessError("I2V_ORIGIN_DENIED", "Image video tasks are available only from the local page"); + if (!dependencies.imageVideo) throw new HarnessError("I2V_TASK_NOT_FOUND", "Image video service unavailable"); + return dependencies.imageVideo; + }; + app.get("/api/image-video/tasks", async (request, reply) => { + const service = context(request); void reply.header("Cache-Control", "no-store"); + const history = await service.list(); + return { tasks: await Promise.all(history.map(task => service.refresh(task.id))) }; + }); + app.get("/api/image-video/tasks/:taskId", async (request, reply) => { + const service = context(request); const { taskId } = z.object({ taskId: taskIdSchema }).parse(request.params); + void reply.header("Cache-Control", "no-store"); return { task: await service.refresh(taskId) }; + }); + app.post("/api/sessions/:sessionId/image-video/tasks", async (request, reply) => { + const service = context(request); const { sessionId } = z.object({ sessionId: z.string().regex(/^ses_[\w-]+$/) }).parse(request.params); + requireSession(dependencies, sessionId); + if (!request.headers["content-type"]?.startsWith("application/json")) throw new HarnessError("I2V_JSON_REQUIRED", "Draft creation requires JSON"); + void reply.header("Cache-Control", "no-store"); return { task: await service.create(sessionId, ImageVideoDraftSchema.parse(request.body)) }; + }); + for (const kind of ["reference", "video"] as const) app.get(`/api/image-video/tasks/:taskId/${kind}`, async (request, reply) => { + const service = context(request); const { taskId } = z.object({ taskId: taskIdSchema }).parse(request.params); + const query = z.object({ download: z.literal("1").optional() }).strict().parse(request.query); + const media = await service.media(taskId, kind); + void reply.header("Cache-Control", "no-store").header("X-Content-Type-Options", "nosniff").header("Cross-Origin-Resource-Policy", "same-origin").header("Accept-Ranges", "bytes").type(media.contentType); + if (query.download === "1") void reply.header("Content-Disposition", `attachment; filename="${taskId}${kind === "video" ? ".mp4" : ".image"}"`); + const range = request.headers.range; + if (range !== undefined) { + const match = /^bytes=(\d*)-(\d*)$/.exec(range); let start = 0; let end = media.size - 1; + if (match?.[1]) { start = Number(match[1]); if (match[2]) end = Math.min(Number(match[2]), end); } + else if (match?.[2]) start = Math.max(0, media.size - Number(match[2])); + if (!match || (!match[1] && !match[2]) || !Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end || start >= media.size || end < 0) return reply.code(416).header("Content-Range", `bytes */${String(media.size)}`).send(); + return reply.code(206).header("Content-Range", `bytes ${String(start)}-${String(end)}/${String(media.size)}`).header("Content-Length", end - start + 1).send(createReadStream(media.path, { start, end })); + } + return reply.header("Content-Length", media.size).send(createReadStream(media.path)); + }); +} diff --git a/apps/server/tests/image-video-api.test.ts b/apps/server/tests/image-video-api.test.ts new file mode 100644 index 0000000..77faeb8 --- /dev/null +++ b/apps/server/tests/image-video-api.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { buildApp, type AppDependencies } from "../src/app.js"; +import { ImageVideoService, FileAttachmentStore } from "@agent-studio/harness"; +import type { ImageVideoPublicTask } from "@agent-studio/harness"; +import { mkdtemp, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "i2v-api-")); await mkdir(join(root, "ses_1")); + const attachments = new FileAttachmentStore({ sessionsRoot: root }); + const image = await attachments.add("ses_1", { name: "pixel.png", mediaType: "image/png", stream: Readable.from(Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAE0lEQVQImWP4z8DwnwGM/zMwAAAf7gP9qS/A4gAAAABJRU5ErkJggg==", "base64")) }); + const service = new ImageVideoService({ root: join(root, "tasks"), attachments, remote: {} as never, collect: () => Promise.reject(new Error("No external operations permitted")) }); + const dependencies = { imageVideo: service, sessionRegistry: { get: (id: string) => id === "ses_1" ? { sessionId: id } : undefined, disposeAll: async () => {} }, skillCatalog: { list: () => [] }, modelStore: {}, attachmentStore: attachments, approvalBroker: {} } as unknown as AppDependencies; + return { app: buildApp(dependencies), service, draft: { attachmentId: image.attachmentId, title: "Reference", prompt: "Natural motion and ambient sound", durationSeconds: 10 } }; +} + +describe("image video local API", () => { + it("creates locally, persists history independently from Sessions and serves registered reference ranges", async () => { + const { app, service, draft } = await fixture(); + try { + const response = await app.inject({ method: "POST", url: "/api/sessions/ses_1/image-video/tasks", payload: draft }); + expect(response.statusCode).toBe(200); const task = response.json<{ task: ImageVideoPublicTask }>().task; + expect(task.state).toBe("draft"); + expect((await app.inject({ method: "GET", url: "/api/image-video/tasks" })).json<{ tasks: ImageVideoPublicTask[] }>().tasks).toHaveLength(1); + const ranged = await app.inject({ method: "GET", url: task.referenceUrl, headers: { range: "bytes=0-3" } }); + expect(ranged.statusCode).toBe(206); expect(ranged.rawPayload.length).toBe(4); expect(ranged.headers["cache-control"]).toBe("no-store"); + expect((await app.inject({ method: "GET", url: task.referenceUrl, headers: { range: "bytes=999999-" } })).statusCode).toBe(416); + expect((await app.inject({ method: "GET", url: `/api/image-video/tasks/${task.id}/video` })).statusCode).toBe(404); + expect((await service.list())[0]?.id).toBe(task.id); + } finally { await app.close(); } + }); + it("rejects foreign origins, rebinding hosts, unknown sessions, arbitrary media paths and mutation types", async () => { + const { app, draft } = await fixture(); + try { + for (const headers of [{ host: "evil.example", origin: "http://evil.example" }, { origin: "https://evil.example" }, { "sec-fetch-site": "cross-site" }]) expect((await app.inject({ method: "GET", url: "/api/image-video/tasks", headers })).statusCode).toBeGreaterThanOrEqual(400); + expect((await app.inject({ method: "POST", url: "/api/sessions/ses_other/image-video/tasks", payload: draft })).statusCode).toBe(404); + expect((await app.inject({ method: "POST", url: "/api/sessions/ses_1/image-video/tasks", payload: "text", headers: { "content-type": "text/plain" } })).statusCode).toBe(400); + expect((await app.inject({ method: "GET", url: "/api/image-video/tasks/secret.env/reference" })).statusCode).toBe(400); + expect((await app.inject({ method: "POST", url: "/api/sessions/ses_1/image-video/tasks", payload: { ...draft, path: "/etc/passwd" } })).statusCode).toBe(400); + } finally { await app.close(); } + }); +}); diff --git a/apps/web/src/components/tools/ImageVideoPanel.tsx b/apps/web/src/components/tools/ImageVideoPanel.tsx new file mode 100644 index 0000000..6081f94 --- /dev/null +++ b/apps/web/src/components/tools/ImageVideoPanel.tsx @@ -0,0 +1,86 @@ +import { useEffect, useRef, useState } from "react"; + +export type ImageVideoTask = { id: string; title: string; prompt: string; durationSeconds: number; state: string; referenceUrl: string; videoUrl?: string; jobId?: string; error?: string; validation?: { videoSeconds?: number; width?: number; height?: number; fps?: number } }; +export function isImageVideoTask(value: unknown): value is ImageVideoTask { + if (!value || typeof value !== "object") return false; + const task = value as Partial; + return typeof task.id === "string" && /^i2v_[0-9a-f-]{36}$/.test(task.id) && typeof task.title === "string" && typeof task.prompt === "string" && typeof task.state === "string" && typeof task.durationSeconds === "number" && task.referenceUrl === `/api/image-video/tasks/${task.id}/reference`; +} +const stateNames: Record = { draft: "本地草稿 · 待图片上传审批", initializing_upload: "准备上传", uploading: "上传中", finalizing: "入库确认中", importing: "参考图入库中", awaiting_submission: "参考图已入库 · 待生成审批", submitting: "提交中", queued: "排队中", running: "生成中", validating: "成片技术校验中", ready: "生成预览 · 待视听检查", failed: "失败", unknown_submission: "提交状态未知 · 禁止重投", unknown_upload: "上传状态未知 · 禁止重复导入" }; + +export function ImageVideoTaskCard({ task }: { task: ImageVideoTask }) { + if (!isImageVideoTask(task)) return null; + const video = task.state === "ready" && task.videoUrl === `/api/image-video/tasks/${task.id}/video` ? task.videoUrl : undefined; + return
+ {`${task.title} +

{task.title}

{stateNames[task.state] ?? "未知状态 · 请核实"}

{task.durationSeconds} 秒 · H3 ref2va · 9:16 · 768p

{task.prompt}

+ {task.jobId ?

生成任务：{task.jobId}

: null} + {task.error ?

{task.error}

: null} + {video ? <>
+
; +} + +async function jsonRequest(url: string, init?: RequestInit): Promise> { + const response = await fetch(url, { cache: "no-store", ...init }); + if (!response.ok) throw new Error("图生视频请求失败，请检查连接或任务状态。"); + return await response.json() as Record; +} + +export function ImageVideoPanel({ sessionId, onDraft }: { sessionId?: string; onDraft?: (task: ImageVideoTask) => void }) { + const [tasks, setTasks] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(); + const [file, setFile] = useState(); const [title, setTitle] = useState(""); const [prompt, setPrompt] = useState(""); const [duration, setDuration] = useState(10); const [saving, setSaving] = useState(false); + const [reference, setReference] = useState(); const mounted = useRef(true); + useEffect(() => { + mounted.current = true; const controller = new AbortController(); let active = false; + const refresh = async () => { if (active) return; active = true; try { const data = await jsonRequest("/api/image-video/tasks", { signal: controller.signal }); if (!controller.signal.aborted) { setTasks(Array.isArray(data.tasks) ? data.tasks.filter(isImageVideoTask) : []); setError(undefined); } } catch { if (!controller.signal.aborted) setError("无法读取任务历史，请检查本机服务。"); } finally { active = false; if (!controller.signal.aborted) setLoading(false); } }; + void refresh(); const timer = window.setInterval(() => void refresh(), 30_000); + return () => { mounted.current = false; controller.abort(); window.clearInterval(timer); }; + }, []); + useEffect(() => { if (!file) { setReference(undefined); return; } const url = URL.createObjectURL(file); setReference(url); return () => URL.revokeObjectURL(url); }, [file]); + const create = async () => { + if (!file || !sessionId || saving || !title.trim() || !prompt.trim()) return; + setSaving(true); setError(undefined); + try { + const body = new FormData(); body.append("file", file); + const upload = await jsonRequest(`/api/sessions/${encodeURIComponent(sessionId)}/attachments`, { method: "POST", body }); + const attachment = upload.attachment as { attachmentId?: unknown } | undefined; + if (typeof attachment?.attachmentId !== "string") throw new Error("附件接收失败。"); + const result = await jsonRequest(`/api/sessions/${encodeURIComponent(sessionId)}/image-video/tasks`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ attachmentId: attachment.attachmentId, title, prompt, durationSeconds: duration }) }); + if (!isImageVideoTask(result.task)) throw new Error("任务响应无效。"); + const task = result.task; + if (mounted.current) { setTasks(current => [task, ...current]); onDraft?.(task); } + } catch (error) { if (mounted.current) setError(error instanceof Error ? error.message : "准备任务失败。"); } finally { if (mounted.current) setSaving(false); } + }; + return
+

图片 → 有声视频

单张参考图，H3 ref2va。无需 CMS。默认竖屏、自然速度、有环境声；10 秒是目标，实际支持尚待验证。

+ {sessionId && onDraft ?
{ event.preventDefault(); void create(); }}> + + {reference ? 本机参考图片预览 : null} + +