feat(sdk): add MIME and PDF processor options

.github/workflows/ci.yml

@@ -14,7 +14,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - name: Use local scrapegraph-js package
-        run: sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json
+        run: |
+          sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json
       - run: bun install
       - run: bun run test
 
@@ -25,7 +26,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - name: Use local scrapegraph-js package
-        run: sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json
+        run: |
+          sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json
       - run: bun install
       - run: bun run build
       - run: bun run check

README.md

@@ -155,9 +155,15 @@ const res = await sgai.search({
   timeRange: "past_week",          // optional
   locationGeoCode: "us",           // optional
   fetchConfig: { /* ... */ },      // optional
+  allowedTypes: ["text/html", "application/pdf"], // optional MIME allowlist
 });
 ```
 
+By default `search` accepts every supported content type, including PDFs, and processes up to 25
+pages per PDF. You do not need to send `processors` or `maxPages` for this default. Use
+`allowedTypes` to restrict accepted MIME types. Only configure `processors` to override the cap;
+`{ type: "pdf" }` also defaults to 25, while `maxPages` accepts `1`–`500`, or `-1` for no page limit.
+
 ### crawl
 
 Crawl a website and its linked pages.

src/schemas.ts

@@ -10,6 +10,10 @@ export const htmlModeSchema = z.enum(["normal", "reader", "prune"]);
 export const fetchContentTypeSchema = z.enum([
 	"text/html",
 	"application/json",
+	"text/markdown",
+	"text/plain",
+	"text/csv",
+	"application/x-latex",
 	"application/pdf",
 	"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
 	"application/vnd.openxmlformats-officedocument.presentationml.presentation",
@@ -25,10 +29,27 @@ export const fetchContentTypeSchema = z.enum([
 	"application/epub+zip",
 	"application/rtf",
 	"application/vnd.oasis.opendocument.text",
-	"text/csv",
-	"text/plain",
-	"application/x-latex",
 ]);
+export const pdfProcessorSchema = z.object({
+	type: z.literal("pdf"),
+	maxPages: z.union([z.literal(-1), z.number().int().min(1).max(500)]).default(25),
+});
+export const allowedTypesSchema = z
+	.array(fetchContentTypeSchema)
+	.min(1)
+	.refine((types) => new Set(types).size === types.length, {
+		message: "duplicate allowed types not allowed",
+	});
+export const processorsSchema = z
+	.array(pdfProcessorSchema)
+	.min(1)
+	.refine(
+		(processors) =>
+			new Set(processors.map((processor) => processor.type)).size === processors.length,
+		{
+			message: "duplicate processor types not allowed",
+		},
+	);
 export const userPromptSchema = z.string().min(1).max(10_000);
 
 const PUBLIC_DOMAIN_RE =
@@ -214,6 +235,8 @@ export const scrapeFormatEntrySchema = z.discriminatedUnion("type", [
 export const scrapeRequestSchema = z.object({
 	url: urlSchema,
 	contentType: fetchContentTypeSchema.optional(),
+	allowedTypes: allowedTypesSchema.optional(),
+	processors: processorsSchema.optional(),
 	fetchConfig: fetchConfigSchema.optional(),
 	formats: z
 		.array(scrapeFormatEntrySchema)
@@ -233,6 +256,8 @@ export const extractRequestBaseSchema = z
 		prompt: userPromptSchema,
 		schema: z.record(z.string(), z.unknown()).optional(),
 		contentType: fetchContentTypeSchema.optional(),
+		allowedTypes: allowedTypesSchema.optional(),
+		processors: processorsSchema.optional(),
 		fetchConfig: fetchConfigSchema.optional(),
 	})
 	.refine((d) => d.url || d.html || d.markdown, {
@@ -249,6 +274,8 @@ export const searchRequestSchema = z
 		prompt: userPromptSchema.optional(),
 		schema: z.record(z.string(), z.unknown()).optional(),
 		locationGeoCode: z.string().max(10).optional(),
+		allowedTypes: allowedTypesSchema.optional(),
+		processors: processorsSchema.optional(),
 		timeRange: z
 			.enum(["past_hour", "past_24_hours", "past_week", "past_month", "past_year"])
 			.optional(),
@@ -517,7 +544,8 @@ export const crawlRequestSchema = z.object({
 		.describe(
 			'Glob-style URL patterns to exclude. Use "*/<slug>" for first-level paths and "**/<slug>/**" for nested paths.',
 		),
-	contentTypes: z.array(fetchContentTypeSchema).optional(),
+	allowedTypes: allowedTypesSchema.optional(),
+	processors: processorsSchema.optional(),
 	fetchConfig: fetchConfigSchema.optional(),
 });
 

src/types.ts

@@ -134,7 +134,7 @@ export type HealthResponse = z.infer<typeof healthResponseSchema>;
 
 // ─── scrape ──────────────────────────────────────────────────────────────────
 
-export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
+export type ScrapeRequest = z.input<typeof scrapeRequestSchema>;
 export type FetchConfig = z.infer<typeof fetchConfigSchema>;
 export type FetchMode = z.infer<typeof fetchModeSchema>;
 export type FetchContentType = z.infer<typeof fetchContentTypeSchema>;
@@ -202,7 +202,7 @@ export type ScrapeEvent =
 
 // ─── extract ─────────────────────────────────────────────────────────────────
 
-export type ExtractRequestBase = z.infer<typeof extractRequestBaseSchema>;
+export type ExtractRequestBase = z.input<typeof extractRequestBaseSchema>;
 export type LlmConfig = z.infer<typeof llmConfigSchema>;
 
 export type ExtractResponse = z.infer<typeof extractResponseSchema>;
@@ -217,7 +217,7 @@ export type ExtractEvent =
 
 // ─── search ──────────────────────────────────────────────────────────────────
 
-export type SearchRequest = z.infer<typeof searchRequestSchema>;
+export type SearchRequest = z.input<typeof searchRequestSchema>;
 
 export type SearchResult = z.infer<typeof searchResultSchema>;
 export type SearchMetadata = z.infer<typeof searchMetadataSchema>;
@@ -313,7 +313,7 @@ export type MonitorEvent =
 
 // ─── crawl ───────────────────────────────────────────────────────────────────
 
-export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
+export type CrawlRequest = z.input<typeof crawlRequestSchema>;
 export type CrawlStatus = z.infer<typeof crawlStatusSchema>;
 export type CrawlPageStatus = z.infer<typeof crawlPageStatusSchema>;
 

tests/schemas.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test";
+import {
+	crawlRequestSchema,
+	extractRequestBaseSchema,
+	scrapeRequestSchema,
+	searchRequestSchema,
+} from "../src/schemas.js";
+import type { SearchRequest } from "../src/types.js";
+
+const requestWithoutExplicitPageCap: SearchRequest = {
+	query: "example",
+	processors: [{ type: "pdf" }],
+};
+
+const requests = [
+	["scrape", scrapeRequestSchema, { url: "https://example.com" }],
+	["extract", extractRequestBaseSchema, { url: "https://example.com", prompt: "title" }],
+	["search", searchRequestSchema, { query: "example" }],
+	["crawl", crawlRequestSchema, { url: "https://example.com" }],
+] as const;
+
+describe("content type and PDF processor validation", () => {
+	test("request types and schemas allow the default 25-page PDF cap", () => {
+		expect(searchRequestSchema.parse(requestWithoutExplicitPageCap).processors).toEqual([
+			{ type: "pdf", maxPages: 25 },
+		]);
+		expect(searchRequestSchema.parse({ query: "example" }).processors).toBeUndefined();
+	});
+
+	for (const [name, schema, base] of requests) {
+		test(`${name} accepts the documented PDF configuration`, () => {
+			expect(
+				schema.safeParse({
+					...base,
+					allowedTypes: ["application/pdf"],
+					processors: [{ type: "pdf", maxPages: 10 }],
+				}).success,
+			).toBe(true);
+		});
+
+		test(`${name} rejects empty and duplicate arrays`, () => {
+			expect(schema.safeParse({ ...base, allowedTypes: [] }).success).toBe(false);
+			expect(
+				schema.safeParse({
+					...base,
+					allowedTypes: ["application/pdf", "application/pdf"],
+				}).success,
+			).toBe(false);
+			expect(schema.safeParse({ ...base, processors: [] }).success).toBe(false);
+			expect(
+				schema.safeParse({
+					...base,
+					processors: [
+						{ type: "pdf", maxPages: 1 },
+						{ type: "pdf", maxPages: 10 },
+					],
+				}).success,
+			).toBe(false);
+		});
+
+		test(`${name} enforces the documented PDF page limits`, () => {
+			for (const maxPages of [1, 500, -1]) {
+				expect(schema.safeParse({ ...base, processors: [{ type: "pdf", maxPages }] }).success).toBe(
+					true,
+				);
+			}
+			for (const maxPages of [0, -2, 501, 1.5]) {
+				expect(schema.safeParse({ ...base, processors: [{ type: "pdf", maxPages }] }).success).toBe(
+					false,
+				);
+			}
+		});
+	}
+});

tests/scrapegraphai.test.ts

@@ -710,6 +710,25 @@ describe("search", () => {
 		expect(res.status).toBe("success");
 		expectRequest(0, "POST", "/search", searchParams);
 	});
+
+	test("with PDF options", async () => {
+		const body = {
+			results: [],
+			metadata: { search: {}, pages: { requested: 1, scraped: 0 } },
+		};
+		fetchSpy = spyOn(globalThis, "fetch").mockResolvedValueOnce(json(body));
+		const searchParams = {
+			query: "papers",
+			numResults: 1,
+			allowedTypes: ["application/pdf"] as const,
+			processors: [{ type: "pdf" as const, maxPages: 10 }],
+		};
+
+		const res = await sdk.search(API_KEY, searchParams);
+
+		expect(res.status).toBe("success");
+		expectRequest(0, "POST", "/search", searchParams);
+	});
 });
 
 describe("getCredits", () => {
@@ -862,7 +881,7 @@ describe("crawl", () => {
 		expectRequest(0, "POST", "/crawl", patternParams);
 	});
 
-	test("start with fetchConfig and contentTypes", async () => {
+	test("start with fetchConfig, allowedTypes, and processors", async () => {
 		const body = {
 			id: "crawl-abc",
 			status: "running",
@@ -874,7 +893,8 @@ describe("crawl", () => {
 
 		const configParams = {
 			url: "https://example.com",
-			contentTypes: ["text/html" as const, "application/pdf" as const],
+			allowedTypes: ["text/html" as const, "application/pdf" as const],
+			processors: [{ type: "pdf" as const, maxPages: 10 }],
 			fetchConfig: {
 				mode: "js" as const,
 				stealth: true,