Skip to content

Commit d638855

Browse files
gary149gary149
committed
Add per-model tool calling support and overrides
Introduces a `supportsTools` flag for models to indicate tool/function calling capability, aggregates provider support, and adds per-model user overrides via settings. Updates UI to display tool support, enables forced tool calling, and ensures backend respects these settings during text generation and conversation flows.
1 parent 448efe5 commit d638855

File tree

15 files changed

+148
-58
lines changed

15 files changed

+148
-58
lines changed

‎src/lib/server/api/routes/groups/models.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ export type GETModelsResponse = Array<{
2121
preprompt?: string;
2222
multimodal: boolean;
2323
multimodalAcceptedMimetypes?: string[];
24+
supportsTools?: boolean;
2425
unlisted: boolean;
2526
hasInferenceAPI: boolean;
2627
// Mark router entry for UI decoration — always present
@@ -59,6 +60,7 @@ export const modelGroup = new Elysia().group("/models", (app) =>
5960
preprompt: model.preprompt,
6061
multimodal: model.multimodal,
6162
multimodalAcceptedMimetypes: model.multimodalAcceptedMimetypes,
63+
supportsTools: (model as unknown as { supportsTools?: boolean }).supportsTools ?? false,
6264
unlisted: model.unlisted,
6365
hasInferenceAPI: model.hasInferenceAPI,
6466
isRouter: model.isRouter,

‎src/lib/server/api/routes/groups/user.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ export const userGroup = new Elysia()
7171

7272
customPrompts: settings?.customPrompts ?? {},
7373
multimodalOverrides: settings?.multimodalOverrides ?? {},
74+
toolsOverrides: settings?.toolsOverrides ?? {},
7475
};
7576
})
7677
.post("/settings", async ({ locals, request }) => {
@@ -85,14 +86,13 @@ export const userGroup = new Elysia()
8586
activeModel: z.string().default(DEFAULT_SETTINGS.activeModel),
8687
customPrompts: z.record(z.string()).default({}),
8788
multimodalOverrides: z.record(z.boolean()).default({}),
89+
toolsOverrides: z.record(z.boolean()).default({}),
8890
disableStream: z.boolean().default(false),
8991
directPaste: z.boolean().default(false),
9092
hidePromptExamples: z.record(z.boolean()).default({}),
9193
})
9294
.parse(body) satisfies SettingsEditable;
9395

94-
// Tools removed: ignore tools updates
95-
9696
await collections.settings.updateOne(
9797
authCondition(locals),
9898
{

‎src/lib/server/models.ts‎

Lines changed: 36 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ const modelConfig = z.object({
5656
.optional(),
5757
multimodal: z.boolean().default(false),
5858
multimodalAcceptedMimetypes: z.array(z.string()).optional(),
59+
// Aggregated tool-calling capability across providers (HF router)
60+
supportsTools: z.boolean().default(false),
5961
unlisted: z.boolean().default(false),
6062
embeddingModel: z.never().optional(),
6163
/** Used to enable/disable system prompt usage */
@@ -234,6 +236,7 @@ const signatureForModel = (model: ProcessedModel) =>
234236
}) ?? null,
235237
multimodal: model.multimodal,
236238
multimodalAcceptedMimetypes: model.multimodalAcceptedMimetypes,
239+
supportsTools: (model as unknown as { supportsTools?: boolean }).supportsTools ?? false,
237240
isRouter: model.isRouter,
238241
hasInferenceAPI: model.hasInferenceAPI,
239242
});
@@ -329,36 +332,40 @@ const buildModels = async (): Promise<ProcessedModel[]> => {
329332
const parsed = listSchema.parse(json);
330333
logger.info({ count: parsed.data.length }, "[models] Parsed models count");
331334

332-
let modelsRaw = parsed.data.map((m) => {
333-
let logoUrl: string | undefined = undefined;
334-
if (isHFRouter && m.id.includes("/")) {
335-
const org = m.id.split("/")[0];
336-
logoUrl = `https://huggingface.co/api/organizations/${encodeURIComponent(org)}/avatar?redirect=true`;
337-
}
335+
let modelsRaw = parsed.data.map((m) => {
336+
let logoUrl: string | undefined = undefined;
337+
if (isHFRouter && m.id.includes("/")) {
338+
const org = m.id.split("/")[0];
339+
logoUrl = `https://huggingface.co/api/organizations/${encodeURIComponent(org)}/avatar?redirect=true`;
340+
}
338341

339-
const inputModalities = (m.architecture?.input_modalities ?? []).map((modality) =>
340-
modality.toLowerCase()
341-
);
342-
const supportsImageInput =
343-
inputModalities.includes("image") || inputModalities.includes("vision");
344-
return {
345-
id: m.id,
346-
name: m.id,
347-
displayName: m.id,
348-
description: m.description,
349-
logoUrl,
350-
providers: m.providers,
351-
multimodal: supportsImageInput,
352-
multimodalAcceptedMimetypes: supportsImageInput ? ["image/*"] : undefined,
353-
endpoints: [
354-
{
355-
type: "openai" as const,
356-
baseURL,
357-
// apiKey will be taken from OPENAI_API_KEY or HF_TOKEN automatically
358-
},
359-
],
360-
} as ModelConfig;
361-
}) as ModelConfig[];
342+
const inputModalities = (m.architecture?.input_modalities ?? []).map((modality) =>
343+
modality.toLowerCase()
344+
);
345+
const supportsImageInput =
346+
inputModalities.includes("image") || inputModalities.includes("vision");
347+
348+
// If any provider supports tools, consider the model as supporting tools
349+
const supportsTools = Boolean((m.providers ?? []).some((p) => p?.supports_tools === true));
350+
return {
351+
id: m.id,
352+
name: m.id,
353+
displayName: m.id,
354+
description: m.description,
355+
logoUrl,
356+
providers: m.providers,
357+
multimodal: supportsImageInput,
358+
multimodalAcceptedMimetypes: supportsImageInput ? ["image/*"] : undefined,
359+
supportsTools,
360+
endpoints: [
361+
{
362+
type: "openai" as const,
363+
baseURL,
364+
// apiKey will be taken from OPENAI_API_KEY or HF_TOKEN automatically
365+
},
366+
],
367+
} as ModelConfig;
368+
}) as ModelConfig[];
362369

363370
const overrides = getModelOverrides();
364371

‎src/lib/server/textGeneration/generate.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@ export async function* generate(
7272
if (hasRouteModel || hasProviderOnly) {
7373
yield {
7474
type: MessageUpdateType.RouterMetadata,
75-
route: output.routerMetadata.route,
76-
model: output.routerMetadata.model,
77-
provider: output.routerMetadata.provider,
75+
route: output.routerMetadata.route || "",
76+
model: output.routerMetadata.model || "",
77+
provider: (output.routerMetadata.provider as unknown as any) || undefined,
7878
};
7979
continue;
8080
}

‎src/lib/server/textGeneration/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ async function* textGenerationWithoutTitle(
5757
messages: processedMessages,
5858
assistant: ctx.assistant,
5959
forceMultimodal: ctx.forceMultimodal,
60+
forceTools: ctx.forceTools,
6061
locals: ctx.locals,
6162
preprompt,
6263
abortSignal: ctx.abortController.signal,

‎src/lib/server/textGeneration/mcp/runMcpFlow.ts‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import type { TextGenerationContext } from "../types";
2121

2222
export type RunMcpFlowContext = Pick<
2323
TextGenerationContext,
24-
"model" | "conv" | "assistant" | "forceMultimodal" | "locals"
24+
"model" | "conv" | "assistant" | "forceMultimodal" | "forceTools" | "locals"
2525
> & { messages: EndpointMessage[] };
2626

2727
export async function* runMcpFlow({
@@ -30,6 +30,7 @@ export async function* runMcpFlow({
3030
messages,
3131
assistant,
3232
forceMultimodal,
33+
forceTools,
3334
locals,
3435
preprompt,
3536
abortSignal,
@@ -80,6 +81,18 @@ export async function* runMcpFlow({
8081
return false;
8182
}
8283

84+
// Gate MCP flow based on model tool support (aggregated) with user override
85+
try {
86+
const supportsTools = Boolean((model as unknown as { supportsTools?: boolean }).supportsTools);
87+
const toolsEnabled = Boolean(forceTools) || supportsTools;
88+
if (!toolsEnabled) {
89+
logger.debug({ model: model.id }, "[mcp] tools disabled for model; skipping MCP flow");
90+
return false;
91+
}
92+
} catch {
93+
// If anything goes wrong reading the flag, proceed (previous behavior)
94+
}
95+
8396
const hasImageInput = messages.some((msg) =>
8497
(msg.files ?? []).some(
8598
(file) => typeof file?.mime === "string" && file.mime.startsWith("image/")
@@ -281,15 +294,15 @@ export async function* runMcpFlow({
281294
);
282295

283296
// If provider header was exposed, notify UI so it can render "via {provider}".
284-
if (providerHeader) {
285-
yield {
286-
type: MessageUpdateType.RouterMetadata,
287-
route: "",
288-
model: "",
289-
provider: providerHeader,
290-
};
291-
logger.debug({ provider: providerHeader }, "[mcp] provider metadata emitted");
292-
}
297+
if (providerHeader) {
298+
yield {
299+
type: MessageUpdateType.RouterMetadata,
300+
route: "",
301+
model: "",
302+
provider: providerHeader as unknown as any,
303+
};
304+
logger.debug({ provider: providerHeader }, "[mcp] provider metadata emitted");
305+
}
293306

294307
const toolCallState: Record<number, { id?: string; name?: string; arguments: string }> = {};
295308
let sawToolCall = false;

‎src/lib/server/textGeneration/types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ export interface TextGenerationContext {
1515
username?: string;
1616
/** Force-enable multimodal handling for endpoints that support it */
1717
forceMultimodal?: boolean;
18+
/** Force-enable tool calling even if model does not advertise support */
19+
forceTools?: boolean;
1820
locals: App.Locals | undefined;
1921
abortController: AbortController;
2022
}

‎src/lib/stores/settings.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ type SettingsStore = {
1212
activeModel: string;
1313
customPrompts: Record<string, string>;
1414
multimodalOverrides: Record<string, boolean>;
15+
toolsOverrides: Record<string, boolean>;
1516
recentlySaved: boolean;
1617
disableStream: boolean;
1718
directPaste: boolean;

‎src/lib/types/Settings.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ export interface Settings extends Timestamps {
2121
*/
2222
multimodalOverrides?: Record<string, boolean>;
2323

24+
/**
25+
* Per‑model overrides to enable tool calling (OpenAI tools/function calling)
26+
* even when not advertised by the provider list. Only `true` is meaningful.
27+
*/
28+
toolsOverrides?: Record<string, boolean>;
29+
2430
/**
2531
* Per-model toggle to hide Omni prompt suggestions shown near the composer.
2632
* When set to `true`, prompt examples for that model are suppressed.
@@ -38,6 +44,7 @@ export const DEFAULT_SETTINGS = {
3844
activeModel: defaultModel.id,
3945
customPrompts: {},
4046
multimodalOverrides: {},
47+
toolsOverrides: {},
4148
hidePromptExamples: {},
4249
disableStream: false,
4350
directPaste: false,

‎src/routes/api/models/+server.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export async function GET() {
1717
promptExamples: model.promptExamples ?? [],
1818
preprompt: model.preprompt ?? "",
1919
multimodal: model.multimodal ?? false,
20+
supportsTools: (model as unknown as { supportsTools?: boolean }).supportsTools ?? false,
2021
unlisted: model.unlisted ?? false,
2122
hasInferenceAPI: model.hasInferenceAPI ?? false,
2223
}));

0 commit comments

Comments
 (0)