From 9f63c23892280e09f81153dbd1361e3d415e109c Mon Sep 17 00:00:00 2001 From: TaylorHawkes <857858+TaylorHawkes@users.noreply.github.com> Date: Fri, 18 Sep 2026 08:12:05 -0600 Subject: [PATCH] feat: add Inference APIs as an LLM provider Adds Inference APIs (https://inferenceapis.com), an OpenAI-compatible host for open-weight models, following the same pattern as the Tensorix provider (#11184). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01NatZMWWFejKjLafCN9N6kU --- core/llm/autodetect.ts | 3 + core/llm/llms/InferenceAPIs.ts | 14 ++ core/llm/llms/index.ts | 2 + .../model-providers/more/inferenceapis.mdx | 126 ++++++++++++++++++ docs/docs.json | 1 + extensions/vscode/config_schema.json | 9 +- .../pages/AddNewModel/configs/providers.ts | 20 +++ packages/openai-adapters/src/index.ts | 2 + packages/openai-adapters/src/types.ts | 1 + 9 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 core/llm/llms/InferenceAPIs.ts create mode 100644 docs/customize/model-providers/more/inferenceapis.mdx diff --git a/core/llm/autodetect.ts b/core/llm/autodetect.ts index c8511554b8b..84f63081fd3 100644 --- a/core/llm/autodetect.ts +++ b/core/llm/autodetect.ts @@ -74,6 +74,7 @@ const PROVIDER_HANDLES_TEMPLATING: string[] = [ "nous", "zAI", "tensorix", + "inferenceapis", // TODO add these, change to inverted logic so only the ones that need templating are hardcoded // Asksage.ts // Azure.ts @@ -133,6 +134,7 @@ const PROVIDER_SUPPORTS_IMAGES: string[] = [ "watsonx", "zAI", "tensorix", + "inferenceapis", ]; const MODEL_SUPPORTS_IMAGES: RegExp[] = [ @@ -253,6 +255,7 @@ const PARALLEL_PROVIDERS: string[] = [ "scaleway", "minimax", "tensorix", + "inferenceapis", ]; function llmCanGenerateInParallel(provider: string, model: string): boolean { diff --git a/core/llm/llms/InferenceAPIs.ts b/core/llm/llms/InferenceAPIs.ts new file mode 100644 index 00000000000..22e433937c0 --- /dev/null +++ b/core/llm/llms/InferenceAPIs.ts @@ -0,0 +1,14 @@ +import OpenAI from "./OpenAI.js"; + +import type { LLMOptions } from "../../index.js"; + +class InferenceAPIs extends OpenAI { + static providerName = "inferenceapis"; + static defaultOptions: Partial = { + apiBase: "https://api.inferenceapis.com/v1/", + model: "zai-org/GLM-5.3", + useLegacyCompletionsEndpoint: false, + }; +} + +export default InferenceAPIs; diff --git a/core/llm/llms/index.ts b/core/llm/llms/index.ts index 4978f0617f2..bce38737ed3 100644 --- a/core/llm/llms/index.ts +++ b/core/llm/llms/index.ts @@ -30,6 +30,7 @@ import HuggingFaceInferenceAPI from "./HuggingFaceInferenceAPI"; import HuggingFaceTEIEmbeddingsProvider from "./HuggingFaceTEI"; import HuggingFaceTGI from "./HuggingFaceTGI"; import Inception from "./Inception"; +import InferenceAPIs from "./InferenceAPIs"; import Kindo from "./Kindo"; import LlamaCpp from "./LlamaCpp"; import Llamafile from "./Llamafile"; @@ -125,6 +126,7 @@ export const LLMClasses = [ xAI, SiliconFlow, Tensorix, + InferenceAPIs, Scaleway, Relace, Inception, diff --git a/docs/customize/model-providers/more/inferenceapis.mdx b/docs/customize/model-providers/more/inferenceapis.mdx new file mode 100644 index 00000000000..519c469dbf9 --- /dev/null +++ b/docs/customize/model-providers/more/inferenceapis.mdx @@ -0,0 +1,126 @@ +--- +title: "Inference APIs" +description: "Configure Inference APIs with Continue to access DeepSeek, GLM, Kimi, MiniMax, GPT-OSS, and Qwen3-VL models through a single OpenAI-compatible API" +--- + +[Inference APIs](https://inferenceapis.com) hosts open-weight models (DeepSeek V4, GLM 5.3, Kimi K2.7, MiniMax M3, GPT-OSS, Qwen3-VL) in the United States with zero data retention, billed per token with no subscription. + + + You can get an API key from + [inferenceapis.com/api-keys](https://inferenceapis.com/api-keys). + + +## Chat Model + +We recommend configuring **zai-org/GLM-5.3** as your chat model. + + + + ```yaml title="config.yaml" + name: My Config + version: 0.0.1 + schema: v1 + + models: + - name: GLM 5.3 + provider: inferenceapis + model: zai-org/GLM-5.3 + apiKey: + roles: + - chat + ``` + + + ```json title="config.json" + { + "models": [ + { + "title": "GLM 5.3", + "provider": "inferenceapis", + "model": "zai-org/GLM-5.3", + "apiKey": "" + } + ] + } + ``` + + + +## Autocomplete Model + +We recommend configuring **deepseek-ai/DeepSeek-V4-Flash** as your autocomplete model. + + + + ```yaml title="config.yaml" + name: My Config + version: 0.0.1 + schema: v1 + + models: + - name: DeepSeek V4 Flash + provider: inferenceapis + model: deepseek-ai/DeepSeek-V4-Flash + apiKey: + roles: + - autocomplete + ``` + + + ```json title="config.json" + { + "models": [ + { + "title": "DeepSeek V4 Flash", + "provider": "inferenceapis", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "apiKey": "" + } + ], + "tabAutocompleteModel": { + "title": "DeepSeek V4 Flash", + "provider": "inferenceapis", + "model": "deepseek-ai/DeepSeek-V4-Flash", + "apiKey": "" + } + } + ``` + + + +## Embeddings Model + +We recommend configuring **BAAI/bge-m3** as your embeddings model. + + + + ```yaml title="config.yaml" + name: My Config + version: 0.0.1 + schema: v1 + + models: + - name: BGE M3 + provider: inferenceapis + model: BAAI/bge-m3 + apiKey: + roles: + - embed + ``` + + + ```json title="config.json" + { + "embeddingsProvider": { + "provider": "inferenceapis", + "model": "BAAI/bge-m3", + "apiKey": "" + } + } + ``` + + + +Inference APIs also provides the Qwen/Qwen3-Embedding-8B embeddings model. [Click here](https://inferenceapis.com/models) to see the full list of available models. + +[View the source](https://github.com/continuedev/continue/blob/main/core/llm/llms/InferenceAPIs.ts) diff --git a/docs/docs.json b/docs/docs.json index b7a1d83f13a..2f9180dfb51 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -110,6 +110,7 @@ "customize/model-providers/more/deepseek", "customize/model-providers/more/deepinfra", "customize/model-providers/more/groq", + "customize/model-providers/more/inferenceapis", "customize/model-providers/more/llamacpp", "customize/model-providers/more/llamastack", "customize/model-providers/more/mimo", diff --git a/extensions/vscode/config_schema.json b/extensions/vscode/config_schema.json index fb3f4c61362..d1336a05dc9 100644 --- a/extensions/vscode/config_schema.json +++ b/extensions/vscode/config_schema.json @@ -229,6 +229,7 @@ "moonshot", "siliconflow", "tensorix", + "inferenceapis", "function-network", "scaleway", "relace", @@ -282,6 +283,7 @@ "### Moonshot\nTo get started with Moonshot AI, obtain your API key from [Moonshot AI](https://platform.moonshot.cn/). Moonshot AI provides high-quality large language models with competitive pricing.\n> [Reference](https://platform.moonshot.cn/docs/api)", "### SiliconFlow\nTo get started with SiliconFlow, obtain your API key from [SiliconCloud](https://cloud.siliconflow.cn/account/ak). SiliconCloud provides cost-effective GenAI services based on excellent open source basic models.\n> [Models](https://siliconflow.cn/zh-cn/models)", "### Tensorix\nTensorix is an OpenAI-compatible API gateway with access to DeepSeek, Llama, Qwen, GLM, and other models. Pay-as-you-go with no subscription required.\nTo get started, create an account and get an API key at [app.tensorix.ai](https://app.tensorix.ai).\n> [Models](https://tensorix.ai/models)", + "### Inference APIs\nInference APIs hosts open-weight models (DeepSeek V4, GLM 5.3, Kimi K2.7, MiniMax M3, GPT-OSS, Qwen3-VL) in the United States with zero data retention, billed per token with no subscription.\nTo get started, create an account and get an API key at [inferenceapis.com/api-keys](https://inferenceapis.com/api-keys).\n> [Models](https://inferenceapis.com/models)", "### Function Network offers private, affordable user-owned AI\nTo get started with Function Network, obtain your API key from [Function Network](https://www.function.network/join-waitlist). Function Network provides a variety of models for chat, completion, and embeddings.", "### Scaleway\n Generative APIs are serverless endpoints for the most popular AI models.\nHosted in European data centers and priced competitively per million tokens used, models served by Scaleway are ideal for users requiring low latency, full data privacy, and 100% compliance with EU AI Act. To get access to the Scaleway Generative APIs, read the [Quickstart guide](https://www.scaleway.com/en/docs/ai-data/generative-apis/quickstart/) and get a [valid API key](https://www.scaleway.com/en/docs/identity-and-access-management/iam/how-to/create-api-keys/).", "### Relace\n Relace provides a fast apply model. To get started, obtain an API key from [here](https://app.relace.ai/settings/api-keys).", @@ -2871,6 +2873,7 @@ "lmstudio", "siliconflow", "tensorix", + "inferenceapis", "function-network", "scaleway", "ovhcloud" @@ -2937,7 +2940,8 @@ "nvidia", "gemini", "siliconflow", - "tensorix" + "tensorix", + "inferenceapis" ] } }, @@ -3006,7 +3010,8 @@ "llm", "huggingface-tei", "siliconflow", - "tensorix" + "tensorix", + "inferenceapis" ] }, "params": { diff --git a/gui/src/pages/AddNewModel/configs/providers.ts b/gui/src/pages/AddNewModel/configs/providers.ts index 9e2aba08c5c..2099fc16d8b 100644 --- a/gui/src/pages/AddNewModel/configs/providers.ts +++ b/gui/src/pages/AddNewModel/configs/providers.ts @@ -1270,6 +1270,26 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo packages: [{ ...models.AUTODETECT }], apiKeyUrl: "https://app.tensorix.ai", }, + inferenceapis: { + title: "Inference APIs", + provider: "inferenceapis", + description: + "Inference APIs hosts open-weight models (DeepSeek V4, GLM 5.3, Kimi K2.7, MiniMax M3, GPT-OSS, Qwen3-VL) in the United States with zero data retention, billed per token with no subscription.", + longDescription: + "To get started with Inference APIs, create an account and get an API key at [inferenceapis.com/api-keys](https://inferenceapis.com/api-keys).", + tags: [ModelProviderTags.RequiresApiKey, ModelProviderTags.OpenSource], + collectInputFor: [ + { + inputType: "text", + key: "apiKey", + label: "API Key", + placeholder: "Enter your Inference APIs API key", + required: true, + }, + ], + packages: [{ ...models.AUTODETECT }], + apiKeyUrl: "https://inferenceapis.com/api-keys", + }, venice: { title: "Venice", provider: "venice", diff --git a/packages/openai-adapters/src/index.ts b/packages/openai-adapters/src/index.ts index 52fb2d33a0c..d88689aa29b 100644 --- a/packages/openai-adapters/src/index.ts +++ b/packages/openai-adapters/src/index.ts @@ -176,6 +176,8 @@ export function constructLlmApi(config: LLMConfig): BaseLlmApi | undefined { return openAICompatible("https://api.function.network/v1/", config); case "tensorix": return openAICompatible("https://api.tensorix.ai/v1/", config); + case "inferenceapis": + return openAICompatible("https://api.inferenceapis.com/v1/", config); case "openrouter": return new OpenRouterApi(config); case "clawrouter": diff --git a/packages/openai-adapters/src/types.ts b/packages/openai-adapters/src/types.ts index 14b9512f75b..f8d7f8f1b29 100644 --- a/packages/openai-adapters/src/types.ts +++ b/packages/openai-adapters/src/types.ts @@ -60,6 +60,7 @@ export const OpenAIConfigSchema = BasePlusConfig.extend({ z.literal("zAI"), z.literal("scaleway"), z.literal("tensorix"), + z.literal("inferenceapis"), z.literal("ncompass"), z.literal("relace"), z.literal("huggingface-inference-api"),