diff --git a/skills-manifest.json b/skills-manifest.json index e7a0a823c0..de75b0ba2a 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -50,8 +50,8 @@ "files": 12 }, "media-use": { - "hash": "332f69eaeabc945f", - "files": 156 + "hash": "28948e640a5d133a", + "files": 158 }, "motion-graphics": { "hash": "de8ec5464028ec67", diff --git a/skills/media-use/audio/scripts/lib/heygen.mjs b/skills/media-use/audio/scripts/lib/heygen.mjs index ed784ffd40..0886aead98 100644 --- a/skills/media-use/audio/scripts/lib/heygen.mjs +++ b/skills/media-use/audio/scripts/lib/heygen.mjs @@ -1,3 +1,4 @@ +import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs"; // heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio // pipeline. The credential resolver matches the hyperframes CLI auth: first // usable source wins — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/ @@ -125,7 +126,7 @@ export async function heygenJSON(path, { method = "GET", headers = {}, body } = // Download a (presigned) URL to destPath; returns byte length. export async function downloadTo(url, destPath) { - const res = await fetch(url); + const res = await fetchMedia(url); if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`); const bytes = Buffer.from(await res.arrayBuffer()); mkdirSync(dirname(destPath), { recursive: true }); diff --git a/skills/media-use/audio/scripts/lib/tts.mjs b/skills/media-use/audio/scripts/lib/tts.mjs index 8bb004e138..e341bf8edb 100644 --- a/skills/media-use/audio/scripts/lib/tts.mjs +++ b/skills/media-use/audio/scripts/lib/tts.mjs @@ -1,3 +1,4 @@ +import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs"; // tts.mjs — multi-provider TTS for the media audio engine. The provider chain, // auto-detected from env, is the one documented in ../SKILL.md: // @@ -312,7 +313,7 @@ export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, d if (!inner.audio_url) { return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" }; } - const res = await fetchImpl(inner.audio_url); + const res = await fetchMedia(inner.audio_url, { fetchImpl }); if (!res.ok) { return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` }; } diff --git a/skills/media-use/scripts/lib/freeze.mjs b/skills/media-use/scripts/lib/freeze.mjs index 2527f6626d..f4d8609701 100644 --- a/skills/media-use/scripts/lib/freeze.mjs +++ b/skills/media-use/scripts/lib/freeze.mjs @@ -1,3 +1,4 @@ +import { fetchMedia, isPublicMediaUrl } from "./media-fetch.mjs"; import { writeFileSync, copyFileSync, mkdirSync } from "node:fs"; import { dirname } from "node:path"; @@ -7,7 +8,7 @@ const MAX_FREEZE_BYTES = 256 * 1024 * 1024; export async function freezeUrl(url, destPath) { const where = String(url).slice(0, 80); - const res = await fetch(url); + const res = await fetchMedia(url); if (!res.ok) throw new Error(`freeze failed: HTTP ${res.status} for ${where}`); // Fail fast on an advertised oversize body before reading a single byte. @@ -47,15 +48,6 @@ const PLATFORM_HOSTS = /(^|\.)(youtube\.com|youtu\.be|vimeo\.com|tiktok\.com|instagram\.com|twitter\.com|x\.com|facebook\.com|dailymotion\.com)$/i; const MEDIA_EXT = /\.(mp3|wav|m4a|aac|ogg|flac|mp4|mov|webm|mkv|png|jpe?g|webp|gif|svg|avif)$/i; -// SSRF guard (m11): a user-supplied --from URL must not point at the local host -// or a private network. Blocks loopback/localhost, RFC1918, link-local, and the -// IPv6 equivalents on the literal hostname. -// ponytail: literal-host check only; a DNS name that *resolves* to a private IP -// (rebinding) still passes — add resolve-then-check if --from ever fetches from -// untrusted hostnames at scale. -const PRIVATE_HOST = - /^(localhost|.*\.local|.*\.internal|127\.|10\.|0\.|169\.254\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.|\[?(::1|::ffff:127\.|f[cd][0-9a-f]{2}:|fe80:))/i; - export function isDirectMediaUrl(u) { let url; try { @@ -65,6 +57,6 @@ export function isDirectMediaUrl(u) { } if (url.protocol !== "http:" && url.protocol !== "https:") return false; if (PLATFORM_HOSTS.test(url.hostname)) return false; - if (PRIVATE_HOST.test(url.hostname)) return false; + if (!isPublicMediaUrl(url)) return false; return MEDIA_EXT.test(url.pathname); } diff --git a/skills/media-use/scripts/lib/heygen-video-provider.test.mjs b/skills/media-use/scripts/lib/heygen-video-provider.test.mjs index d1b87fd35f..35f2e001a2 100644 --- a/skills/media-use/scripts/lib/heygen-video-provider.test.mjs +++ b/skills/media-use/scripts/lib/heygen-video-provider.test.mjs @@ -36,7 +36,7 @@ async function freshGenerate() { return module.heygenVideoGenerate; } -async function listenVideoServer() { +async function listenVideoServer(t) { const server = http.createServer((req, res) => { if (req.url !== "/video.mp4") { res.writeHead(404).end(); @@ -51,23 +51,33 @@ async function listenVideoServer() { await new Promise((resolve) => server.listen(0, resolve)); const address = server.address(); assert.ok(address && typeof address !== "string"); - return { - server, - url: `http://127.0.0.1:${address.port}/video.mp4`, - }; + // Keep the provider URL public; only the test transport reaches the local fixture. + const url = "https://media.example.com/video.mp4"; + const originalFetch = globalThis.fetch; + t.mock.method(globalThis, "fetch", (requested, options) => { + assert.equal(requested, url); + assert.equal(options.redirect, "manual"); + return originalFetch(`http://127.0.0.1:${address.port}/video.mp4`, options); + }); + return { server, url }; } -async function listenFailingVideoServer() { +async function listenFailingVideoServer(t) { const server = http.createServer((req, res) => { res.writeHead(500).end(); }); await new Promise((resolve) => server.listen(0, resolve)); const address = server.address(); assert.ok(address && typeof address !== "string"); - return { - server, - url: `http://127.0.0.1:${address.port}/video.mp4`, - }; + // Keep the provider URL public; only the test transport reaches the local fixture. + const url = "https://media.example.com/video.mp4"; + const originalFetch = globalThis.fetch; + t.mock.method(globalThis, "fetch", (requested, options) => { + assert.equal(requested, url); + assert.equal(options.redirect, "manual"); + return originalFetch(`http://127.0.0.1:${address.port}/video.mp4`, options); + }); + return { server, url }; } function closeServer(server) { @@ -144,8 +154,8 @@ function bodyFromInvocation(invocation) { return JSON.parse(invocation.slice(start + marker.length)); } -test("downloads a generated avatar video and returns the generated MP4 result", async () => { - const { server, url } = await listenVideoServer(); +test("downloads a generated avatar video and returns the generated MP4 result", async (t) => { + const { server, url } = await listenVideoServer(t); let localPath; try { await withFakeHeygen( @@ -191,8 +201,8 @@ test("downloads a generated avatar video and returns the generated MP4 result", } }); -test("tags video creation but not avatar or voice discovery", async () => { - const { server, url } = await listenVideoServer(); +test("tags video creation but not avatar or voice discovery", async (t) => { + const { server, url } = await listenVideoServer(t); let localPath; try { await withFakeHeygen( @@ -218,8 +228,8 @@ test("tags video creation but not avatar or voice discovery", async () => { } }); -test("uses explicit avatar and voice overrides without discovery", async () => { - const { server, url } = await listenVideoServer(); +test("uses explicit avatar and voice overrides without discovery", async (t) => { + const { server, url } = await listenVideoServer(t); let localPath; try { await withFakeHeygen( @@ -249,8 +259,8 @@ test("uses explicit avatar and voice overrides without discovery", async () => { } }); -test("caches discovered avatar and voice IDs for the process", async () => { - const { server, url } = await listenVideoServer(); +test("caches discovered avatar and voice IDs for the process", async (t) => { + const { server, url } = await listenVideoServer(t); const localPaths = new Set(); try { await withFakeHeygen( @@ -349,8 +359,8 @@ test("onboards and returns null when avatar/voice discovery itself is unauthenti }); }); -test("download failure after a successful create returns null and logs a diagnostic", async () => { - const { server, url } = await listenFailingVideoServer(); +test("download failure after a successful create returns null and logs a diagnostic", async (t) => { + const { server, url } = await listenFailingVideoServer(t); try { await withFakeHeygen({ response: JSON.stringify({ data: { video_url: url } }) }, async () => { const heygenVideoGenerate = await freshGenerate(); diff --git a/skills/media-use/scripts/lib/logo-provider.mjs b/skills/media-use/scripts/lib/logo-provider.mjs index d4028af94f..05f910670e 100644 --- a/skills/media-use/scripts/lib/logo-provider.mjs +++ b/skills/media-use/scripts/lib/logo-provider.mjs @@ -1,3 +1,4 @@ +import { fetchMedia } from "./media-fetch.mjs"; // Official brand marks — the `logo` type's provider tiers, tried in registry // order. Every tier was verified against a 54-brand stress test (2026-07, // 100% cascade hit). Hit counts below are a snapshot of that run — they @@ -105,13 +106,13 @@ export function faviconDomainFor(entity) { } async function fetchJson(url) { - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + const res = await fetchMedia(url, { signal: AbortSignal.timeout(10_000) }); if (!res.ok) return null; return res.json(); } async function urlExists(url) { - const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(10_000) }); + const res = await fetchMedia(url, { method: "HEAD", signal: AbortSignal.timeout(10_000) }); return res.ok; } @@ -196,7 +197,7 @@ export async function faviconSearch(intent, ctx = {}) { const url = `https://icons.duckduckgo.com/ip3/${domain}.ico`; let body; try { - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + const res = await fetchMedia(url, { signal: AbortSignal.timeout(10_000) }); if (!res.ok) return null; body = Buffer.from(await res.arrayBuffer()); } catch { diff --git a/skills/media-use/scripts/lib/logo-provider.test.mjs b/skills/media-use/scripts/lib/logo-provider.test.mjs index 5863d409b9..fc710825c5 100644 --- a/skills/media-use/scripts/lib/logo-provider.test.mjs +++ b/skills/media-use/scripts/lib/logo-provider.test.mjs @@ -135,3 +135,20 @@ test("the real logo cascade falls through tier by tier to the first hit", async assert.ok(res, "cascade must land on the favicon tier"); assert.equal(res.metadata.provider, "favicon.ddg"); }); + +for (const search of [simpleIconsSearch, githubAvatarSearch]) { + test(`${search.name} rejects private HEAD redirects`, async (t) => { + const seen = []; + t.mock.method(globalThis, "fetch", async (url, options) => { + seen.push(url); + assert.equal(options.method, "HEAD"); + assert.ok(options.signal); + return options.redirect === "manual" + ? new Response(null, { status: 302, headers: { location: "http://127.0.0.1/private" } }) + : new Response(null, { status: 200 }); + }); + assert.equal(await search("vercel logo"), null); + assert.equal(seen.length, 1); + assert.ok(!seen[0].includes("127.0.0.1")); + }); +} diff --git a/skills/media-use/scripts/lib/media-fetch.mjs b/skills/media-use/scripts/lib/media-fetch.mjs new file mode 100644 index 0000000000..830e41618a --- /dev/null +++ b/skills/media-use/scripts/lib/media-fetch.mjs @@ -0,0 +1,71 @@ +// Media downloads use public HTTP(S) URLs. Validate every redirect target; +// a provider result must meet the same host policy as a direct ingest URL. +// Public HTTPS-to-HTTP redirects are allowed, matching direct HTTP support. +// This is a literal-host policy, not DNS pinning: DNS resolution remains trusted. + +import { BlockList, isIP } from "node:net"; + +const blocked = new BlockList(); +for (const [network, prefix] of [ + ["0.0.0.0", 8], + ["10.0.0.0", 8], + ["100.64.0.0", 10], + ["127.0.0.0", 8], + ["169.254.0.0", 16], + ["172.16.0.0", 12], + ["192.0.0.0", 24], + ["192.0.2.0", 24], + ["192.88.99.0", 24], + ["192.168.0.0", 16], + ["198.18.0.0", 15], + ["198.51.100.0", 24], + ["203.0.113.0", 24], + ["224.0.0.0", 4], + ["240.0.0.0", 4], +]) + blocked.addSubnet(network, prefix, "ipv4"); +for (const [network, prefix] of [ + ["::", 128], + ["::1", 128], + ["fc00::", 7], + ["fe80::", 10], + ["fec0::", 10], + ["ff00::", 8], + ["2001:db8::", 32], +]) + blocked.addSubnet(network, prefix, "ipv6"); + +export function isPublicMediaUrl(value) { + try { + const url = new URL(value); + if (url.protocol !== "http:" && url.protocol !== "https:") return false; + const host = url.hostname.replace(/\.$/, ""); + if ( + host === "localhost" || + host.endsWith(".localhost") || + host.endsWith(".local") || + host.endsWith(".internal") + ) + return false; + const address = host.replace(/^\[|\]$/g, ""); + const family = isIP(address); + return family === 0 || !blocked.check(address, family === 4 ? "ipv4" : "ipv6"); + } catch { + return false; + } +} + +export async function fetchMedia(url, { method = "GET", signal, fetchImpl = fetch } = {}) { + let current = String(url); + for (let hop = 0; hop <= 5; hop++) { + if (!isPublicMediaUrl(current)) + throw new Error("Media download blocked: URL is not public HTTP(S)"); + const response = await fetchImpl(current, { method, signal, redirect: "manual" }); + if (!(response.status >= 300 && response.status < 400)) return response; + const location = response.headers.get("location"); + if (!location) return response; + await response.body?.cancel(); + current = new URL(location, current).href; + } + throw new Error("Media download exceeded redirect limit"); +} diff --git a/skills/media-use/scripts/lib/media-fetch.test.mjs b/skills/media-use/scripts/lib/media-fetch.test.mjs new file mode 100644 index 0000000000..72ddfb95a2 --- /dev/null +++ b/skills/media-use/scripts/lib/media-fetch.test.mjs @@ -0,0 +1,169 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fetchMedia, isPublicMediaUrl } from "./media-fetch.mjs"; +import { freezeUrl } from "./freeze.mjs"; +import { downloadTo } from "../../audio/scripts/lib/heygen.mjs"; +import { synthesizeHeygen } from "../../audio/scripts/lib/tts.mjs"; +import { faviconSearch } from "./logo-provider.mjs"; + +test("blocks private, reserved, alternate-encoded and mapped hosts", () => { + for (const host of [ + "localhost", + "localhost.", + "a.localhost", + "svc.internal", + "svc.local", + "127.1", + "2130706433", + "0x7f000001", + "10.0.0.1", + "169.254.169.254", + "100.100.100.200", + "192.168.1.1", + "172.31.0.1", + "224.0.0.1", + "[::]", + "[::1]", + "[::ffff:127.0.0.1]", + "[::ffff:a00:1]", + "[fe80::1]", + "[fd00::1]", + "[ff02::1]", + ]) { + assert.equal(isPublicMediaUrl(`https://${host}/asset`), false, host); + } + for (const url of [ + "https://cdn.example/asset.cube", + "http://example.com/clip.mp4", + "https://8.8.8.8/a", + "https://[::ffff:8.8.8.8]/a", + ]) + assert.equal(isPublicMediaUrl(url), true, url); + assert.equal(isPublicMediaUrl("file:///tmp/a.mp4"), false); +}); + +test("follows public relative redirects manually and forwards cancellation", async () => { + const calls = []; + const signal = new AbortController().signal; + const response = await fetchMedia("https://cdn.example/a", { + method: "HEAD", + signal, + fetchImpl: async (url, options) => { + calls.push(url); + assert.equal(options.redirect, "manual"); + assert.equal(options.signal, signal); + assert.equal(options.method, "HEAD"); + return calls.length === 1 + ? new Response(null, { status: 302, headers: { location: "/b" } }) + : new Response("media"); + }, + }); + assert.deepEqual(calls, ["https://cdn.example/a", "https://cdn.example/b"]); + assert.equal(await response.text(), "media"); +}); + +test("rejects private redirect targets before requesting them and cancels redirect bodies", async () => { + let calls = 0; + let cancelled = false; + await assert.rejects( + fetchMedia("https://cdn.example/a", { + fetchImpl: async () => { + calls++; + return new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { + status: 302, + headers: { location: "http://169.254.169.254/latest/meta-data" }, + }, + ); + }, + }), + /blocked/, + ); + assert.equal(calls, 1); + assert.equal(cancelled, true); +}); + +test("bounds redirect loops", async () => { + let calls = 0; + await assert.rejects( + fetchMedia("https://cdn.example/a", { + fetchImpl: async () => { + calls++; + return new Response(null, { status: 302, headers: { location: "/a" } }); + }, + }), + /redirect limit/, + ); + assert.equal(calls, 6); +}); + +test("freeze refuses a public-to-private redirect without writing response bytes", async (t) => { + const dir = mkdtempSync(join(tmpdir(), "hf-freeze-redirect-")); + t.mock.method(globalThis, "fetch", async (_url, options) => + options?.redirect === "manual" + ? new Response(null, { status: 302, headers: { location: "http://127.0.0.1/private.mp4" } }) + : new Response("private response"), + ); + try { + await assert.rejects(freezeUrl("https://cdn.example/a.mp4", join(dir, "out.mp4")), /blocked/); + assert.deepEqual(readdirSync(dir), []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +for (const entry of ["heygen audio", "tts mp3", "tts wav", "favicon"]) { + test(`${entry} blocks private redirects before saving or transcoding`, async (t) => { + const dir = mkdtempSync(join(tmpdir(), "hf-provider-redirect-")); + const requests = []; + const fetchImpl = t.mock.method(globalThis, "fetch", async (url, options) => { + requests.push(url); + return options?.redirect === "manual" + ? new Response(null, { + status: 302, + headers: { location: "http://169.254.169.254/private" }, + }) + : new Response(new Uint8Array(600)); + }); + try { + if (entry === "heygen audio") { + await assert.rejects( + downloadTo("https://cdn.example/audio", join(dir, "audio.mp3")), + /blocked/, + ); + } else if (entry === "favicon") { + assert.equal(await faviconSearch("GitHub logo"), null); + } else { + const result = await synthesizeHeygen( + { + text: "hi", + voiceId: "v1", + lang: "en", + speed: 1, + wavAbs: join(dir, entry === "tts wav" ? "audio.wav" : "audio.mp3"), + }, + { + heygenAuthHeaders: () => ({}), + heygenJSON: async () => ({ data: { audio_url: "https://cdn.example/audio" } }), + fetch: fetchImpl, + transcodeToWav: () => assert.fail("private bytes must not reach ffmpeg"), + }, + ); + assert.equal(result.ok, false); + assert.match(result.error, /blocked/); + } + assert.equal(requests.length, 1); + assert.deepEqual(readdirSync(dir), []); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}