diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 039af49..d3b3132 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,8 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - name: Use local scrapegraph-js package - run: sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json + run: | + sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json - run: bun install - run: bun run test @@ -25,7 +26,8 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - name: Use local scrapegraph-js package - run: sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json + run: | + sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json - run: bun install - run: bun run build - run: bun run check diff --git a/README.md b/README.md index 37b1dad..dc99cfe 100644 --- a/README.md +++ b/README.md @@ -155,9 +155,15 @@ const res = await sgai.search({ timeRange: "past_week", // optional locationGeoCode: "us", // optional fetchConfig: { /* ... */ }, // optional + allowedTypes: ["text/html", "application/pdf"], // optional MIME allowlist }); ``` +By default `search` accepts every supported content type, including PDFs, and processes up to 25 +pages per PDF. You do not need to send `processors` or `maxPages` for this default. Use +`allowedTypes` to restrict accepted MIME types. Only configure `processors` to override the cap; +`{ type: "pdf" }` also defaults to 25, while `maxPages` accepts `1`–`500`, or `-1` for no page limit. + ### crawl Crawl a website and its linked pages. diff --git a/src/schemas.ts b/src/schemas.ts index 6a273c9..333f0ff 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -10,6 +10,10 @@ export const htmlModeSchema = z.enum(["normal", "reader", "prune"]); export const fetchContentTypeSchema = z.enum([ "text/html", "application/json", + "text/markdown", + "text/plain", + "text/csv", + "application/x-latex", "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.presentationml.presentation", @@ -25,10 +29,27 @@ export const fetchContentTypeSchema = z.enum([ "application/epub+zip", "application/rtf", "application/vnd.oasis.opendocument.text", - "text/csv", - "text/plain", - "application/x-latex", ]); +export const pdfProcessorSchema = z.object({ + type: z.literal("pdf"), + maxPages: z.union([z.literal(-1), z.number().int().min(1).max(500)]).default(25), +}); +export const allowedTypesSchema = z + .array(fetchContentTypeSchema) + .min(1) + .refine((types) => new Set(types).size === types.length, { + message: "duplicate allowed types not allowed", + }); +export const processorsSchema = z + .array(pdfProcessorSchema) + .min(1) + .refine( + (processors) => + new Set(processors.map((processor) => processor.type)).size === processors.length, + { + message: "duplicate processor types not allowed", + }, + ); export const userPromptSchema = z.string().min(1).max(10_000); const PUBLIC_DOMAIN_RE = @@ -214,6 +235,8 @@ export const scrapeFormatEntrySchema = z.discriminatedUnion("type", [ export const scrapeRequestSchema = z.object({ url: urlSchema, contentType: fetchContentTypeSchema.optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), fetchConfig: fetchConfigSchema.optional(), formats: z .array(scrapeFormatEntrySchema) @@ -233,6 +256,8 @@ export const extractRequestBaseSchema = z prompt: userPromptSchema, schema: z.record(z.string(), z.unknown()).optional(), contentType: fetchContentTypeSchema.optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), fetchConfig: fetchConfigSchema.optional(), }) .refine((d) => d.url || d.html || d.markdown, { @@ -249,6 +274,8 @@ export const searchRequestSchema = z prompt: userPromptSchema.optional(), schema: z.record(z.string(), z.unknown()).optional(), locationGeoCode: z.string().max(10).optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), timeRange: z .enum(["past_hour", "past_24_hours", "past_week", "past_month", "past_year"]) .optional(), @@ -517,7 +544,8 @@ export const crawlRequestSchema = z.object({ .describe( 'Glob-style URL patterns to exclude. Use "*/" for first-level paths and "**//**" for nested paths.', ), - contentTypes: z.array(fetchContentTypeSchema).optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), fetchConfig: fetchConfigSchema.optional(), }); diff --git a/src/types.ts b/src/types.ts index d2fd8ea..0fec5b1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -134,7 +134,7 @@ export type HealthResponse = z.infer; // ─── scrape ────────────────────────────────────────────────────────────────── -export type ScrapeRequest = z.infer; +export type ScrapeRequest = z.input; export type FetchConfig = z.infer; export type FetchMode = z.infer; export type FetchContentType = z.infer; @@ -202,7 +202,7 @@ export type ScrapeEvent = // ─── extract ───────────────────────────────────────────────────────────────── -export type ExtractRequestBase = z.infer; +export type ExtractRequestBase = z.input; export type LlmConfig = z.infer; export type ExtractResponse = z.infer; @@ -217,7 +217,7 @@ export type ExtractEvent = // ─── search ────────────────────────────────────────────────────────────────── -export type SearchRequest = z.infer; +export type SearchRequest = z.input; export type SearchResult = z.infer; export type SearchMetadata = z.infer; @@ -313,7 +313,7 @@ export type MonitorEvent = // ─── crawl ─────────────────────────────────────────────────────────────────── -export type CrawlRequest = z.infer; +export type CrawlRequest = z.input; export type CrawlStatus = z.infer; export type CrawlPageStatus = z.infer; diff --git a/tests/schemas.test.ts b/tests/schemas.test.ts new file mode 100644 index 0000000..a9fd249 --- /dev/null +++ b/tests/schemas.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test"; +import { + crawlRequestSchema, + extractRequestBaseSchema, + scrapeRequestSchema, + searchRequestSchema, +} from "../src/schemas.js"; +import type { SearchRequest } from "../src/types.js"; + +const requestWithoutExplicitPageCap: SearchRequest = { + query: "example", + processors: [{ type: "pdf" }], +}; + +const requests = [ + ["scrape", scrapeRequestSchema, { url: "https://example.com" }], + ["extract", extractRequestBaseSchema, { url: "https://example.com", prompt: "title" }], + ["search", searchRequestSchema, { query: "example" }], + ["crawl", crawlRequestSchema, { url: "https://example.com" }], +] as const; + +describe("content type and PDF processor validation", () => { + test("request types and schemas allow the default 25-page PDF cap", () => { + expect(searchRequestSchema.parse(requestWithoutExplicitPageCap).processors).toEqual([ + { type: "pdf", maxPages: 25 }, + ]); + expect(searchRequestSchema.parse({ query: "example" }).processors).toBeUndefined(); + }); + + for (const [name, schema, base] of requests) { + test(`${name} accepts the documented PDF configuration`, () => { + expect( + schema.safeParse({ + ...base, + allowedTypes: ["application/pdf"], + processors: [{ type: "pdf", maxPages: 10 }], + }).success, + ).toBe(true); + }); + + test(`${name} rejects empty and duplicate arrays`, () => { + expect(schema.safeParse({ ...base, allowedTypes: [] }).success).toBe(false); + expect( + schema.safeParse({ + ...base, + allowedTypes: ["application/pdf", "application/pdf"], + }).success, + ).toBe(false); + expect(schema.safeParse({ ...base, processors: [] }).success).toBe(false); + expect( + schema.safeParse({ + ...base, + processors: [ + { type: "pdf", maxPages: 1 }, + { type: "pdf", maxPages: 10 }, + ], + }).success, + ).toBe(false); + }); + + test(`${name} enforces the documented PDF page limits`, () => { + for (const maxPages of [1, 500, -1]) { + expect(schema.safeParse({ ...base, processors: [{ type: "pdf", maxPages }] }).success).toBe( + true, + ); + } + for (const maxPages of [0, -2, 501, 1.5]) { + expect(schema.safeParse({ ...base, processors: [{ type: "pdf", maxPages }] }).success).toBe( + false, + ); + } + }); + } +}); diff --git a/tests/scrapegraphai.test.ts b/tests/scrapegraphai.test.ts index edfe47a..31c6634 100644 --- a/tests/scrapegraphai.test.ts +++ b/tests/scrapegraphai.test.ts @@ -710,6 +710,25 @@ describe("search", () => { expect(res.status).toBe("success"); expectRequest(0, "POST", "/search", searchParams); }); + + test("with PDF options", async () => { + const body = { + results: [], + metadata: { search: {}, pages: { requested: 1, scraped: 0 } }, + }; + fetchSpy = spyOn(globalThis, "fetch").mockResolvedValueOnce(json(body)); + const searchParams = { + query: "papers", + numResults: 1, + allowedTypes: ["application/pdf"] as const, + processors: [{ type: "pdf" as const, maxPages: 10 }], + }; + + const res = await sdk.search(API_KEY, searchParams); + + expect(res.status).toBe("success"); + expectRequest(0, "POST", "/search", searchParams); + }); }); describe("getCredits", () => { @@ -862,7 +881,7 @@ describe("crawl", () => { expectRequest(0, "POST", "/crawl", patternParams); }); - test("start with fetchConfig and contentTypes", async () => { + test("start with fetchConfig, allowedTypes, and processors", async () => { const body = { id: "crawl-abc", status: "running", @@ -874,7 +893,8 @@ describe("crawl", () => { const configParams = { url: "https://example.com", - contentTypes: ["text/html" as const, "application/pdf" as const], + allowedTypes: ["text/html" as const, "application/pdf" as const], + processors: [{ type: "pdf" as const, maxPages: 10 }], fetchConfig: { mode: "js" as const, stealth: true,