51 lines
3.7 KiB
JavaScript
51 lines
3.7 KiB
JavaScript
import { createInterface } from "node:readline";
|
|
import { createRequire } from "node:module";
|
|
const require = createRequire(new URL("../packages/harness/package.json", import.meta.url));
|
|
const sharp = require("sharp");
|
|
const reader = createInterface({ input: process.stdin });
|
|
console.log("Provide gateway API key on hidden stdin. Only synthetic images will be sent.");
|
|
let key = (await reader[Symbol.asyncIterator]().next()).value?.trim();
|
|
reader.close();
|
|
if (!key) process.exit(1);
|
|
const model = "Qwen3.8-27B-FP8";
|
|
const shapes = [
|
|
'<circle cx="110" cy="180" r="60" fill="red"/><rect x="230" y="120" width="120" height="120" fill="blue"/><polygon points="470,115 410,240 530,240" fill="green"/>',
|
|
'<rect x="50" y="120" width="120" height="120" fill="green"/><polygon points="290,115 230,240 350,240" fill="red"/><circle cx="470" cy="180" r="60" fill="blue"/>',
|
|
];
|
|
const images = await Promise.all(shapes.map(async (shape) => {
|
|
const png = await sharp(Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="600" height="360"><rect width="600" height="360" fill="white"/>${shape}</svg>`)).png().toBuffer();
|
|
return `data:image/png;base64,${png.toString("base64")}`;
|
|
}));
|
|
const prompt = 'Describe each image separately. Return only JSON {"images":[[{"color":"...","shape":"..."}]]}. One inner array per image, list objects from left to right. Use lowercase English color names and shape names circle, square, or triangle. Do not guess if you cannot see the images.';
|
|
const expected = [[{color:"red",shape:"circle"},{color:"blue",shape:"square"},{color:"green",shape:"triangle"}], [{color:"green",shape:"square"},{color:"red",shape:"triangle"},{color:"blue",shape:"circle"}]];
|
|
try {
|
|
for (const api of ["responses", "chat/completions"]) {
|
|
const content = api === "responses"
|
|
? [{type:"input_text",text:prompt}, ...images.map((image_url) => ({type:"input_image",image_url}))]
|
|
: [{type:"text",text:prompt}, ...images.map((url) => ({type:"image_url",image_url:{url}}))];
|
|
const body = api === "responses"
|
|
? {model,input:[{role:"user",content}],max_output_tokens:2048,stream:false}
|
|
: {model,messages:[{role:"user",content}],max_tokens:2048,stream:false};
|
|
const response = await fetch(`https://gateway.pencil-stub.com/v1/${api}`, {
|
|
method:"POST", redirect:"error", signal:AbortSignal.timeout(90_000),
|
|
headers:{Authorization:`Bearer ${key}`,"Content-Type":"application/json"},body:JSON.stringify(body),
|
|
});
|
|
const result = await response.json();
|
|
if (!response.ok) {
|
|
console.log(JSON.stringify({api,httpStatus:response.status,error:String(result.error?.message ?? result.message ?? "Request failed").replaceAll(key,"[REDACTED]").slice(0,600)}));
|
|
continue;
|
|
}
|
|
const answer = api === "responses"
|
|
? (result.output ?? []).flatMap((item) => item.content ?? []).filter((item) => item.type === "output_text").map((item) => item.text).join("")
|
|
: result.choices?.[0]?.message?.content ?? "";
|
|
let parsed;
|
|
try { parsed = JSON.parse(answer.replace(/^\s*```(?:json)?\s*/, "").replace(/\s*```\s*$/, "")); } catch { /* Retain the bounded answer for diagnosis. */ }
|
|
const passed = Array.isArray(parsed?.images) && parsed.images.length === expected.length && expected.every((objects,i) => Array.isArray(parsed.images[i]) && parsed.images[i].length === objects.length && objects.every((item,j) => parsed.images[i][j]?.color === item.color && parsed.images[i][j]?.shape === item.shape));
|
|
console.log(JSON.stringify({api,httpStatus:response.status,model:result.model,passed,answer:answer.replaceAll(key,"[REDACTED]").slice(0,1500),expected,privateMediaSent:false}));
|
|
if (passed) break;
|
|
}
|
|
} catch (error) {
|
|
console.error(JSON.stringify({status:"probe_failed",type:error.name}));
|
|
process.exitCode = 1;
|
|
} finally { key = undefined; }
|