From 8e7ef0bdb1c6339ed66169254ae12e6e3cad5b00 Mon Sep 17 00:00:00 2001 From: "jinhao.song" Date: Fri, 21 Aug 2026 22:10:30 +0000 Subject: [PATCH] feat(llm): add OrcaRouter as a named LLM provider Add OrcaRouter (https://www.orcarouter.ai) as a named provider for LLM-based extraction. OrcaRouter is an OpenAI-compatible gateway at https://api.orcarouter.ai/v1; model ids use the orcarouter/ prefix (e.g. orcarouter/auto, orcarouter/free). The pinned unclecode-litellm build has no native orcarouter/ provider prefix, so route orcarouter/ through the openai provider while keeping the full model id intact and pointing base_url at the gateway: - config.py: register orcarouter in PROVIDER_MODELS_PREFIXES (auto-resolves ORCAROUTER_API_KEY), add ORCAROUTER_BASE_URL and orcarouter_litellm_params() - utils.py: route orcarouter/ providers in perform_completion_with_backoff, aperform_completion_with_backoff, extract_blocks_batch, get_text_embeddings - async_configs.py: LLMConfig defaults base_url to the gateway for orcarouter/ - cli.py: route orcarouter/ in the -q streaming path - docs: document OrcaRouter usage in llm-strategies.md and README - tests: test_orcarouter_provider.py covers routing + auto key/base_url Verified locally: 9 new unit tests pass; chat completion and embeddings live-tested against the OrcaRouter gateway (HTTP 200). Co-Authored-By: Claude Signed-off-by: jinhao.song --- README.md | 1 + crawl4ai/async_configs.py | 7 ++- crawl4ai/cli.py | 10 ++-- crawl4ai/config.py | 33 ++++++++++++ crawl4ai/utils.py | 48 ++++++++++++++--- docs/md_v2/extraction/llm-strategies.md | 13 +++++ tests/test_orcarouter_provider.py | 72 +++++++++++++++++++++++++ 7 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 tests/test_orcarouter_provider.py diff --git a/README.md b/README.md index 838553762..a63e960bb 100644 --- a/README.md +++ b/README.md @@ -498,6 +498,7 @@ async def main(): extraction_strategy=LLMExtractionStrategy( # Here you can use any provider that Litellm library supports, for instance: ollama/qwen2 # provider="ollama/qwen2", api_token="no-token", + # OrcaRouter (OpenAI-compatible gateway): provider="orcarouter/auto", api_token=os.getenv('ORCAROUTER_API_KEY') llm_config = LLMConfig(provider="openai/gpt-4o", api_token=os.getenv('OPENAI_API_KEY')), schema=OpenAIModelFee.schema(), extraction_type="schema", diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 3efb0d380..bfe95716c 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -15,6 +15,7 @@ PAGE_TIMEOUT, IMAGE_SCORE_THRESHOLD, SOCIAL_MEDIA_DOMAINS, + ORCAROUTER_BASE_URL, ) from .user_agent_generator import UAGen, ValidUAGenerator # , OnlineUAGenerator @@ -2284,10 +2285,14 @@ def __init__( (prefix for prefix in prefixes if provider.startswith(prefix)), None, ) - self.api_token = PROVIDER_MODELS_PREFIXES.get(selected_prefix) + self.api_token = PROVIDER_MODELS_PREFIXES.get(selected_prefix) else: self.provider = DEFAULT_PROVIDER self.api_token = os.getenv(DEFAULT_PROVIDER_API_KEY) + # Named OrcaRouter provider: default the gateway base URL when the + # provider uses the `orcarouter/` prefix. + if provider.startswith("orcarouter/") and not base_url: + base_url = ORCAROUTER_BASE_URL self.base_url = base_url self.temperature = temperature self.max_tokens = max_tokens diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py index 02b67155e..d9da3032f 100644 --- a/crawl4ai/cli.py +++ b/crawl4ai/cli.py @@ -33,7 +33,7 @@ BestFirstCrawlingStrategy, ) from crawl4ai.browser_profiler import ShrinkLevel, _format_size -from crawl4ai.config import USER_SETTINGS +from crawl4ai.config import USER_SETTINGS, orcarouter_litellm_params from crawl4ai.cloud import cloud_cmd from litellm import completion from pathlib import Path @@ -65,7 +65,7 @@ def setup_llm_config() -> tuple[str, str]: if not provider: click.echo("\nNo default LLM provider configured.") - click.echo("Provider format: 'company/model' (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet')") + click.echo("Provider format: 'company/model' (e.g., 'openai/gpt-4o', 'anthropic/claude-3-sonnet', 'orcarouter/auto')") click.echo("See available providers at: https://docs.litellm.ai/docs/providers") provider = click.prompt("Enter provider") @@ -84,7 +84,7 @@ def setup_llm_config() -> tuple[str, str]: return provider, token async def stream_llm_response(url: str, markdown: str, query: str, provider: str, token: str): - response = completion( + completion_kwargs = dict( model=provider, api_key=token, messages=[ @@ -99,6 +99,10 @@ async def stream_llm_response(url: str, markdown: str, query: str, provider: str ], stream=True, ) + # Named OrcaRouter provider: route the OpenAI-compatible gateway while + # keeping the full `orcarouter/` id (LiteLLM has no native prefix). + completion_kwargs.update(orcarouter_litellm_params(provider, token, None)) + response = completion(**completion_kwargs) for chunk in response: if content := chunk["choices"][0]["delta"].get("content"): diff --git a/crawl4ai/config.py b/crawl4ai/config.py index 5d394136f..3186791ca 100644 --- a/crawl4ai/config.py +++ b/crawl4ai/config.py @@ -36,9 +36,42 @@ "anthropic": os.getenv("ANTHROPIC_API_KEY"), "gemini": os.getenv("GEMINI_API_KEY"), "deepseek": os.getenv("DEEPSEEK_API_KEY"), + "orcarouter": os.getenv("ORCAROUTER_API_KEY"), "bedrock": None, # Bedrock uses AWS credential chain (SigV4) or explicit api_token for bearer auth } +# OrcaRouter gateway defaults. OrcaRouter is an OpenAI-compatible gateway: +# https://api.orcarouter.ai/v1. Model ids use the `orcarouter/` prefix +# (e.g. `orcarouter/auto`, `orcarouter/free`). +ORCAROUTER_BASE_URL = "https://api.orcarouter.ai/v1" + + +def orcarouter_litellm_params(provider, api_token, base_url=None): + """Return LiteLLM kwargs that route an ``orcarouter/`` provider string. + + The pinned ``unclecode-litellm`` build has no native ``orcarouter/`` provider + prefix, so LiteLLM rejects ``orcarouter/auto`` with "LLM Provider NOT + provided". OrcaRouter is OpenAI-compatible, so we route through the + ``openai`` provider while keeping the full ``orcarouter/`` id intact + (OrcaRouter routes on that prefix) and pointing ``base_url`` at the gateway. + + Args: + provider (str): The provider string, e.g. "orcarouter/auto". + api_token (str): The OrcaRouter API token. + base_url (Optional[str]): Override for the gateway base URL. + + Returns: + dict: Extra kwargs to pass to ``litellm.completion``/``acompletion``. + Empty dict when ``provider`` is not an OrcaRouter model. + """ + if not provider or not provider.startswith("orcarouter/"): + return {} + return { + "custom_llm_provider": "openai", + "api_key": api_token, + "base_url": base_url or ORCAROUTER_BASE_URL, + } + # Chunk token threshold CHUNK_TOKEN_THRESHOLD = 2**11 # 2048 tokens OVERLAP_RATE = 0.1 diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 279c27708..f04ef031e 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -12,7 +12,15 @@ from array import array from .html2text import html2text, CustomHTML2Text # from .config import * -from .config import MIN_WORD_THRESHOLD, IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, IMAGE_SCORE_THRESHOLD, DEFAULT_PROVIDER, PROVIDER_MODELS +from .config import ( + MIN_WORD_THRESHOLD, + IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD, + IMAGE_SCORE_THRESHOLD, + DEFAULT_PROVIDER, + PROVIDER_MODELS, + ORCAROUTER_BASE_URL, + orcarouter_litellm_params, +) import httpx from socket import gaierror from pathlib import Path @@ -1786,6 +1794,11 @@ def perform_completion_with_backoff( if kwargs.get("extra_args"): extra_args.update(kwargs["extra_args"]) + # Named OrcaRouter provider: route the OpenAI-compatible gateway while + # keeping the full `orcarouter/` id (LiteLLM has no native prefix). + orca_params = orcarouter_litellm_params(provider, api_token, base_url) + extra_args.update(orca_params) + for attempt in range(max_attempts): try: response = completion( @@ -1879,6 +1892,11 @@ async def aperform_completion_with_backoff( if kwargs.get("extra_args"): extra_args.update(kwargs["extra_args"]) + # Named OrcaRouter provider: route the OpenAI-compatible gateway while + # keeping the full `orcarouter/` id (LiteLLM has no native prefix). + orca_params = orcarouter_litellm_params(provider, api_token, base_url) + extra_args.update(orca_params) + for attempt in range(max_attempts): try: response = await acompletion( @@ -1993,6 +2011,13 @@ def extract_blocks_batch(batch_data, provider="groq/llama3-70b-8192", api_token= api_token = os.getenv("GROQ_API_KEY", None) if not api_token else api_token from litellm import batch_completion + # Named OrcaRouter provider: route the OpenAI-compatible gateway while + # keeping the full `orcarouter/` id (LiteLLM has no native prefix), + # and default the token to ORCAROUTER_API_KEY. + if not api_token and provider and provider.startswith("orcarouter/"): + api_token = os.getenv("ORCAROUTER_API_KEY", None) + orca_params = orcarouter_litellm_params(provider, api_token, None) + messages = [] for url, _html in batch_data: @@ -2009,7 +2034,9 @@ def extract_blocks_batch(batch_data, provider="groq/llama3-70b-8192", api_token= messages.append([{"role": "user", "content": prompt_with_variables}]) - responses = batch_completion(model=provider, messages=messages, temperature=0.01) + responses = batch_completion( + model=provider, messages=messages, temperature=0.01, **orca_params + ) all_blocks = [] for response in responses: @@ -3535,19 +3562,26 @@ async def get_text_embeddings( # Get embedding model from config or use default embedding_model = llm_config.get('provider', 'text-embedding-3-small') api_base = llm_config.get('base_url', llm_config.get('api_base')) - + # Prepare kwargs kwargs = { 'model': embedding_model, 'input': texts, 'api_key': llm_config.get('api_token', llm_config.get('api_key')) } - + if api_base: kwargs['api_base'] = api_base - - # Handle OpenAI-compatible endpoints - if api_base and 'openai/' not in embedding_model: + + # Named OrcaRouter provider: OrcaRouter exposes OpenAI-compatible + # embedding models (e.g. `openai/text-embedding-3-small`) at + # https://api.orcarouter.ai/v1. Keep the `openai/` prefix so LiteLLM + # routes through the OpenAI provider, and default the base URL to the + # gateway when a user sets `orcarouter` as the embedding provider. + if embedding_model and embedding_model.startswith('orcarouter/'): + kwargs['model'] = f"openai/{embedding_model[len('orcarouter/'):]}" + kwargs['api_base'] = api_base or ORCAROUTER_BASE_URL + elif api_base and 'openai/' not in embedding_model: kwargs['model'] = f"openai/{embedding_model}" # Get embeddings diff --git a/docs/md_v2/extraction/llm-strategies.md b/docs/md_v2/extraction/llm-strategies.md index cba4d6e4f..0c5c2e831 100644 --- a/docs/md_v2/extraction/llm-strategies.md +++ b/docs/md_v2/extraction/llm-strategies.md @@ -34,6 +34,19 @@ Crawl4AI uses a “provider string” (e.g., `"openai/gpt-4o"`, `"ollama/llama2. This means you **aren’t locked** into a single LLM vendor. Switch or experiment easily. +### 2.1 OrcaRouter + +[OrcaRouter](https://www.orcarouter.ai) is an OpenAI-compatible gateway. Use the `orcarouter/` provider prefix and an `ORCAROUTER_API_KEY`: + +```python +llm_config = LLMConfig( + provider="orcarouter/auto", + api_token=os.getenv("ORCAROUTER_API_KEY"), +) +``` + +`base_url` defaults to `https://api.orcarouter.ai/v1`, so you can omit it. The full `orcarouter/` id is preserved when calling the gateway (e.g. `orcarouter/auto`, `orcarouter/free`), and the API token is auto-resolved from `ORCAROUTER_API_KEY` when you don't pass `api_token` explicitly. + --- ## 3. How LLM Extraction Works diff --git a/tests/test_orcarouter_provider.py b/tests/test_orcarouter_provider.py new file mode 100644 index 000000000..1a73cb84a --- /dev/null +++ b/tests/test_orcarouter_provider.py @@ -0,0 +1,72 @@ +"""Tests for the named OrcaRouter provider integration. + +OrcaRouter (https://www.orcarouter.ai) is an OpenAI-compatible gateway. The +pinned ``unclecode-litellm`` build has no native ``orcarouter/`` provider prefix, +so crawl4ai routes ``orcarouter/`` through the ``openai`` provider while +keeping the full model id intact and pointing ``base_url`` at the gateway. +""" + +import os +import sys + +# Env vars are read by crawl4ai.config.PROVIDER_MODELS_PREFIXES at import time +# (same as OPENAI_API_KEY / DEEPSEEK_API_KEY), so set them before importing. +os.environ.setdefault("ORCAROUTER_API_KEY", "sk-orca-env") + +# Add the parent directory to the Python path +parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(parent_dir) + +from crawl4ai.config import ORCAROUTER_BASE_URL, orcarouter_litellm_params +from crawl4ai import LLMConfig + + +class TestOrcarouterLitellmParams: + def test_routes_orcarouter_models(self): + params = orcarouter_litellm_params("orcarouter/auto", "sk-orca-test", None) + assert params["custom_llm_provider"] == "openai" + assert params["api_key"] == "sk-orca-test" + assert params["base_url"] == ORCAROUTER_BASE_URL + + def test_defaults_base_url_to_gateway(self): + params = orcarouter_litellm_params("orcarouter/free", "sk-orca-test", None) + assert params["base_url"] == "https://api.orcarouter.ai/v1" + + def test_honors_custom_base_url(self): + params = orcarouter_litellm_params( + "orcarouter/auto", "sk-orca-test", "https://example.com/v1" + ) + assert params["base_url"] == "https://example.com/v1" + + def test_ignores_non_orcarouter_providers(self): + assert orcarouter_litellm_params("openai/gpt-4o", "key", None) == {} + assert orcarouter_litellm_params("groq/llama3-70b-8192", "key", None) == {} + assert orcarouter_litellm_params(None, "key", None) == {} + + def test_keeps_full_model_id(self): + # The helper never rewrites the provider string; litellm receives + # `model="orcarouter/auto"` with custom_llm_provider="openai". + params = orcarouter_litellm_params("orcarouter/auto", "sk-orca-test", None) + assert "model" not in params + assert params["custom_llm_provider"] == "openai" + + +class TestLLMConfigOrcarouter: + def test_api_token_auto_resolved(self): + config = LLMConfig(provider="orcarouter/auto") + assert config.api_token == "sk-orca-env" + assert config.base_url == ORCAROUTER_BASE_URL + + def test_explicit_api_token_wins(self): + config = LLMConfig(provider="orcarouter/auto", api_token="sk-orca-explicit") + assert config.api_token == "sk-orca-explicit" + + def test_explicit_base_url_wins(self): + config = LLMConfig( + provider="orcarouter/auto", base_url="https://example.com/v1" + ) + assert config.base_url == "https://example.com/v1" + + def test_non_orcarouter_unchanged(self): + config = LLMConfig(provider="openai/gpt-4o") + assert config.base_url is None