From 1c89376370b63ad32fef472114ec036edaaf8d1c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 16 Mar 2026 15:23:24 -0700
Subject: [PATCH 01/47] training: Add gradient_checkpointing for lower VRAM by
default
---
docs/05 - Training Tab.md | 2 ++
modules/training.py | 8 +++++---
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/docs/05 - Training Tab.md b/docs/05 - Training Tab.md
index 0bfc59aa..46424eab 100644
--- a/docs/05 - Training Tab.md
+++ b/docs/05 - Training Tab.md
@@ -100,6 +100,8 @@ Each parameter has a description in the UI. Below is guidance on the most import
VRAM usage during training is roughly similar to inference with ~1000 tokens of context. If you can run the model, you can probably train LoRAs with the default settings. If you run out of VRAM, reduce `Micro Batch Size` or `Cutoff Length`. Training 4-bit quantized models uses more VRAM — set `Micro Batch Size` to `1` to compensate.
+**Gradient checkpointing** is enabled by default. It reduces VRAM usage by recomputing activations during the backward pass instead of storing them in memory. The tradeoff is ~20-30% slower training. There is no impact on accuracy — the results are mathematically identical. The savings are most noticeable with longer sequences and larger batch sizes. You can disable it if you have VRAM to spare and want faster training.
+
### Rank
Higher rank = more learning capacity = larger adapter = more VRAM. Use 4–8 for style/format, 128–256 to teach factual knowledge.
diff --git a/modules/training.py b/modules/training.py
index 878bb222..6549b35e 100644
--- a/modules/training.py
+++ b/modules/training.py
@@ -26,7 +26,7 @@ from modules.evaluate import (
from modules.logging_colors import logger
from modules.models import reload_model
-PARAMETERS = ["lora_name", "always_override", "all_linear", "q_proj_en", "v_proj_en", "k_proj_en", "o_proj_en", "gate_proj_en", "down_proj_en", "up_proj_en", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "text_dataset", "higher_rank_limit", "warmup_steps", "optimizer", "stride_length", "stop_at_loss", "add_eos_token", "excess_length", "report_to"]
+PARAMETERS = ["lora_name", "always_override", "all_linear", "q_proj_en", "v_proj_en", "k_proj_en", "o_proj_en", "gate_proj_en", "down_proj_en", "up_proj_en", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "text_dataset", "higher_rank_limit", "warmup_steps", "optimizer", "stride_length", "stop_at_loss", "add_eos_token", "excess_length", "report_to", "gradient_checkpointing"]
WANT_INTERRUPT = False
train_log = {}
@@ -101,6 +101,7 @@ def create_ui():
add_eos_token = gr.Checkbox(label='Add EOS token', value=True, info="Adds EOS token for each document in text datasets.")
excess_length = gr.Dropdown(label='Excess length', value='drop', choices=['drop', 'truncate'], info='What to do with conversations that exceed the cutoff length. "Drop" removes them entirely (recommended). "Truncate" cuts from the right, which may produce incomplete responses.', elem_classes=['slim-dropdown'])
+ gradient_checkpointing = gr.Checkbox(label='Gradient checkpointing', value=True, info='Trades ~20-30% training speed for reduced VRAM usage by recomputing activations during the backward pass instead of storing them. No impact on accuracy.')
higher_rank_limit = gr.Checkbox(label='Enable higher ranks', value=False, info='If checked, changes Rank/Alpha slider above to go much higher. This will not work without a datacenter-class GPU.')
report_to = gr.Radio(label="Save detailed logs with", value="None", choices=["None", "wandb", "tensorboard"], interactive=True)
@@ -159,7 +160,7 @@ def create_ui():
refresh_table = gr.Button('Refresh the table', elem_classes="small-button", interactive=not mu)
# Training events
- all_params = [lora_name, always_override, all_linear, q_proj_en, v_proj_en, k_proj_en, o_proj_en, gate_proj_en, down_proj_en, up_proj_en, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, text_dataset, higher_rank_limit, warmup_steps, optimizer, stride_length, stop_at_loss, add_eos_token, excess_length, report_to]
+ all_params = [lora_name, always_override, all_linear, q_proj_en, v_proj_en, k_proj_en, o_proj_en, gate_proj_en, down_proj_en, up_proj_en, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, text_dataset, higher_rank_limit, warmup_steps, optimizer, stride_length, stop_at_loss, add_eos_token, excess_length, report_to, gradient_checkpointing]
copy_from.change(do_copy_params, [copy_from] + all_params, all_params)
start_button.click(do_train, all_params, output)
@@ -293,7 +294,7 @@ def calc_trainable_parameters(model):
return trainable_params, all_param
-def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en: bool, v_proj_en: bool, k_proj_en: bool, o_proj_en: bool, gate_proj_en: bool, down_proj_en: bool, up_proj_en: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, text_dataset: str, higher_rank_limit: bool, warmup_steps: int, optimizer: str, stride_length: int, stop_at_loss: float, add_eos_token: bool, excess_length: str, report_to: str):
+def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en: bool, v_proj_en: bool, k_proj_en: bool, o_proj_en: bool, gate_proj_en: bool, down_proj_en: bool, up_proj_en: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, text_dataset: str, higher_rank_limit: bool, warmup_steps: int, optimizer: str, stride_length: int, stop_at_loss: float, add_eos_token: bool, excess_length: str, report_to: str, gradient_checkpointing: bool = True):
import torch
import transformers
@@ -708,6 +709,7 @@ def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en:
load_best_model_at_end=eval_data is not None,
# TODO: Enable multi-device support
ddp_find_unused_parameters=None,
+ gradient_checkpointing=gradient_checkpointing,
use_cpu=shared.args.cpu,
remove_unused_columns=False,
),
From 22ff5044b0ccd12e2ab1181e4dd3d503a7b0ae2c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 16 Mar 2026 16:01:28 -0700
Subject: [PATCH 02/47] training: Organize the UI
---
modules/training.py | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/modules/training.py b/modules/training.py
index 6549b35e..7cb50068 100644
--- a/modules/training.py
+++ b/modules/training.py
@@ -90,19 +90,16 @@ def create_ui():
with gr.Accordion(label='Advanced Options', open=False, elem_classes='tgw-accordion'):
with gr.Row():
with gr.Column():
+ optimizer = gr.Dropdown(label='Optimizer', value='adamw_torch', choices=['adamw_hf', 'adamw_torch', 'adamw_torch_fused', 'adamw_torch_xla', 'adamw_apex_fused', 'adafactor', 'adamw_bnb_8bit', 'adamw_anyprecision', 'sgd', 'adagrad'], info='Optimizer algorithm. adamw_torch is the standard choice. adamw_bnb_8bit uses less VRAM. adafactor is memory-efficient for large models.', elem_classes=['slim-dropdown'])
+ warmup_steps = gr.Number(label='Warmup Steps', value=100, info='For this many steps at the start, the learning rate is gradually ramped up from 0 to the target value. This prevents unstable updates early in training.')
lora_dropout = gr.Slider(label='LoRA Dropout', minimum=0.0, maximum=1.0, step=0.025, value=0.0, info='Percentage probability for dropout of LoRA layers. This can help reduce overfitting. Most users should leave at default.')
stop_at_loss = gr.Slider(label='Stop at loss', minimum=0.0, maximum=3.0, step=0.1, value=0.00, info='The process will automatically stop once the desired loss value is reached. (reasonable numbers are 1.5-1.8)')
- with gr.Row():
- optimizer = gr.Dropdown(label='Optimizer', value='adamw_torch', choices=['adamw_hf', 'adamw_torch', 'adamw_torch_fused', 'adamw_torch_xla', 'adamw_apex_fused', 'adafactor', 'adamw_bnb_8bit', 'adamw_anyprecision', 'sgd', 'adagrad'], info='Optimizer algorithm. adamw_torch is the standard choice. adamw_bnb_8bit uses less VRAM. adafactor is memory-efficient for large models.', elem_classes=['slim-dropdown'])
with gr.Column():
- warmup_steps = gr.Number(label='Warmup Steps', value=100, info='For this many steps at the start, the learning rate is gradually ramped up from 0 to the target value. This prevents unstable updates early in training.')
-
- add_eos_token = gr.Checkbox(label='Add EOS token', value=True, info="Adds EOS token for each document in text datasets.")
- excess_length = gr.Dropdown(label='Excess length', value='drop', choices=['drop', 'truncate'], info='What to do with conversations that exceed the cutoff length. "Drop" removes them entirely (recommended). "Truncate" cuts from the right, which may produce incomplete responses.', elem_classes=['slim-dropdown'])
-
gradient_checkpointing = gr.Checkbox(label='Gradient checkpointing', value=True, info='Trades ~20-30% training speed for reduced VRAM usage by recomputing activations during the backward pass instead of storing them. No impact on accuracy.')
+ add_eos_token = gr.Checkbox(label='Add EOS token', value=True, info="Adds EOS token for each document in text datasets.")
higher_rank_limit = gr.Checkbox(label='Enable higher ranks', value=False, info='If checked, changes Rank/Alpha slider above to go much higher. This will not work without a datacenter-class GPU.')
+ excess_length = gr.Dropdown(label='Excess length', value='drop', choices=['drop', 'truncate'], info='What to do with conversations that exceed the cutoff length. "Drop" removes them entirely (recommended). "Truncate" cuts from the right, which may produce incomplete responses.', elem_classes=['slim-dropdown'])
report_to = gr.Radio(label="Save detailed logs with", value="None", choices=["None", "wandb", "tensorboard"], interactive=True)
with gr.Column():
From 238cbd5656a9007d7e4f5ff39a04e1d340b9e50c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 16 Mar 2026 16:05:43 -0700
Subject: [PATCH 03/47] training: Remove arbitrary higher_rank_limit parameter
---
modules/training.py | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/modules/training.py b/modules/training.py
index 7cb50068..db7b206b 100644
--- a/modules/training.py
+++ b/modules/training.py
@@ -26,7 +26,7 @@ from modules.evaluate import (
from modules.logging_colors import logger
from modules.models import reload_model
-PARAMETERS = ["lora_name", "always_override", "all_linear", "q_proj_en", "v_proj_en", "k_proj_en", "o_proj_en", "gate_proj_en", "down_proj_en", "up_proj_en", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "text_dataset", "higher_rank_limit", "warmup_steps", "optimizer", "stride_length", "stop_at_loss", "add_eos_token", "excess_length", "report_to", "gradient_checkpointing"]
+PARAMETERS = ["lora_name", "always_override", "all_linear", "q_proj_en", "v_proj_en", "k_proj_en", "o_proj_en", "gate_proj_en", "down_proj_en", "up_proj_en", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "text_dataset", "warmup_steps", "optimizer", "stride_length", "stop_at_loss", "add_eos_token", "excess_length", "report_to", "gradient_checkpointing"]
WANT_INTERRUPT = False
train_log = {}
@@ -73,8 +73,8 @@ def create_ui():
with gr.Row():
with gr.Column():
- lora_rank = gr.Slider(label='LoRA Rank', value=8, minimum=0, maximum=1024, step=4, info='Also called dimension count. Higher values = larger file, more content control. Smaller values = smaller file, less control. Use 4 or 8 for style, 128 or 256 to teach, 1024+ for fine-detail on big data. More VRAM is needed for higher ranks.')
- lora_alpha = gr.Slider(label='LoRA Alpha', value=16, minimum=0, maximum=2048, step=4, info='This divided by the rank becomes the scaling of the LoRA. Higher means stronger. A good standard value is twice your Rank.')
+ lora_rank = gr.Slider(label='LoRA Rank', value=8, minimum=0, maximum=2048, step=4, info='Also called dimension count. Use 4–8 for style/format, 128–256 to teach factual knowledge, 1024+ for comprehensive fine-tuning. Very high ranks require significant VRAM.')
+ lora_alpha = gr.Slider(label='LoRA Alpha', value=16, minimum=0, maximum=4096, step=4, info='This divided by the rank becomes the scaling of the LoRA. Higher means stronger. A good standard value is twice your Rank.')
batch_size = gr.Slider(label='Batch Size', value=32, minimum=0, maximum=1024, step=4, info='Global batch size. The two batch sizes together determine gradient accumulation (gradientAccum = batch / microBatch). Higher gradient accum values lead to better quality training.')
micro_batch_size = gr.Slider(label='Micro Batch Size', value=4, minimum=1, maximum=128, step=1, info='Per-device batch size (NOTE: multiple devices not yet implemented). Increasing this will increase VRAM usage.')
cutoff_len = gr.Slider(label='Cutoff Length', minimum=0, maximum=4096, value=512, step=32, info='Maximum sequence length in tokens. For instruction datasets, conversations longer than this are dropped. For text datasets, documents are split into chunks of this size. Higher values require more VRAM.')
@@ -98,7 +98,6 @@ def create_ui():
with gr.Column():
gradient_checkpointing = gr.Checkbox(label='Gradient checkpointing', value=True, info='Trades ~20-30% training speed for reduced VRAM usage by recomputing activations during the backward pass instead of storing them. No impact on accuracy.')
add_eos_token = gr.Checkbox(label='Add EOS token', value=True, info="Adds EOS token for each document in text datasets.")
- higher_rank_limit = gr.Checkbox(label='Enable higher ranks', value=False, info='If checked, changes Rank/Alpha slider above to go much higher. This will not work without a datacenter-class GPU.')
excess_length = gr.Dropdown(label='Excess length', value='drop', choices=['drop', 'truncate'], info='What to do with conversations that exceed the cutoff length. "Drop" removes them entirely (recommended). "Truncate" cuts from the right, which may produce incomplete responses.', elem_classes=['slim-dropdown'])
report_to = gr.Radio(label="Save detailed logs with", value="None", choices=["None", "wandb", "tensorboard"], interactive=True)
@@ -157,12 +156,12 @@ def create_ui():
refresh_table = gr.Button('Refresh the table', elem_classes="small-button", interactive=not mu)
# Training events
- all_params = [lora_name, always_override, all_linear, q_proj_en, v_proj_en, k_proj_en, o_proj_en, gate_proj_en, down_proj_en, up_proj_en, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, text_dataset, higher_rank_limit, warmup_steps, optimizer, stride_length, stop_at_loss, add_eos_token, excess_length, report_to, gradient_checkpointing]
+ all_params = [lora_name, always_override, all_linear, q_proj_en, v_proj_en, k_proj_en, o_proj_en, gate_proj_en, down_proj_en, up_proj_en, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, text_dataset, warmup_steps, optimizer, stride_length, stop_at_loss, add_eos_token, excess_length, report_to, gradient_checkpointing]
copy_from.change(do_copy_params, [copy_from] + all_params, all_params)
start_button.click(do_train, all_params, output)
stop_button.click(do_interrupt, None, None, queue=False)
- higher_rank_limit.change(change_rank_limit, [higher_rank_limit], [lora_rank, lora_alpha])
+
# Evaluation events. For some reason, the interrupt event
# doesn't work with the .then() syntax, so I write them one
@@ -207,10 +206,6 @@ def do_copy_params(lora_name: str, *args):
return result
-def change_rank_limit(use_higher_ranks: bool):
- mult = 2 if use_higher_ranks else 1
- return {"maximum": 1024 * mult, "__type__": "update"}, {"maximum": 2048 * mult, "__type__": "update"}
-
def clean_path(base_path: str, path: str):
"""Strips unusual symbols and forcibly builds a path as relative to the intended directory."""
@@ -291,7 +286,7 @@ def calc_trainable_parameters(model):
return trainable_params, all_param
-def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en: bool, v_proj_en: bool, k_proj_en: bool, o_proj_en: bool, gate_proj_en: bool, down_proj_en: bool, up_proj_en: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, text_dataset: str, higher_rank_limit: bool, warmup_steps: int, optimizer: str, stride_length: int, stop_at_loss: float, add_eos_token: bool, excess_length: str, report_to: str, gradient_checkpointing: bool = True):
+def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en: bool, v_proj_en: bool, k_proj_en: bool, o_proj_en: bool, gate_proj_en: bool, down_proj_en: bool, up_proj_en: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, text_dataset: str, warmup_steps: int, optimizer: str, stride_length: int, stop_at_loss: float, add_eos_token: bool, excess_length: str, report_to: str, gradient_checkpointing: bool = True):
import torch
import transformers
From 9d02d3a13b2e39a2a3bf91d8936044f9bbd9fd49 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 16 Mar 2026 16:08:06 -0700
Subject: [PATCH 04/47] docs: Minor change to tool calling tutorial
---
docs/Tool Calling Tutorial.md | 40 +++++++++++++++++------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/docs/Tool Calling Tutorial.md b/docs/Tool Calling Tutorial.md
index 801e9d78..d95a9c80 100644
--- a/docs/Tool Calling Tutorial.md
+++ b/docs/Tool Calling Tutorial.md
@@ -1,18 +1,3 @@
-## Supported models
-
-The following models are supported:
-
-- Qwen 3.5
-- GPT-OSS
-- Mistral Small / Devstral
-- DeepSeek V3
-- Kimi-K2
-- MiniMax-M2.5
-- GLM-5
-- Llama 4
-
-Other models that output tool calls as JSON (inside XML tags, code blocks, or plain JSON) are also supported through a generic fallback parser.
-
## Tool calling in the UI
### 1. Load a model with tool-calling support
@@ -23,11 +8,11 @@ Load a model with tool-calling support from the Model tab.
In the chat sidebar, check the tools you want the model to use:
-- **web_search** -- Search the web using DuckDuckGo.
-- **fetch_webpage** -- Fetch the content of a URL.
-- **calculate** -- Evaluate math expressions.
-- **get_datetime** -- Get the current date and time.
-- **roll_dice** -- Roll dice.
+- `web_search`: Search the web using DuckDuckGo.
+- `fetch_webpage`: Fetch the content of a URL.
+- `calculate`: Evaluate math expressions.
+- `get_datetime`: Get the current date and time.
+- `roll_dice`: Roll dice.
### 3. Chat
@@ -157,3 +142,18 @@ for _ in range(10):
print(f"\nAssistant: {choice['message']['content']}")
break
```
+
+## Supported models
+
+The following models are supported:
+
+- Qwen 3.5
+- GPT-OSS
+- Mistral Small / Devstral
+- DeepSeek V3
+- Kimi-K2
+- MiniMax-M2.5
+- GLM-5
+- Llama 4
+
+Other models that output tool calls as JSON (inside XML tags, code blocks, or plain JSON) are also supported through a generic fallback parser.
From dff8903b03c2b3e46e11c16862f12db0495b3e91 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 16 Mar 2026 18:25:54 -0700
Subject: [PATCH 05/47] UI: Modernize the Gradio theme
---
css/main.css | 73 +++++++++++++++++++++++++++++--------------
modules/ui.py | 26 +++++++++------
modules/ui_session.py | 2 +-
3 files changed, 68 insertions(+), 33 deletions(-)
diff --git a/css/main.css b/css/main.css
index 25ae15b1..22fac5c5 100644
--- a/css/main.css
+++ b/css/main.css
@@ -2,8 +2,8 @@
--darker-gray: #1C1C1D;
--dark-gray: #212125;
--light-gray: #2C2E34;
- --light-theme-gray: #f9fbff;
- --border-color-dark: #525252;
+ --light-theme-gray: #f0f3fb;
+ --border-color-dark: rgba(255, 255, 255, 0.15);
--header-width: 112px;
--selected-item-color-dark: #282930;
}
@@ -127,7 +127,7 @@ gradio-app > :first-child {
}
.header_bar {
- border-right: var(--input-border-width) solid var(--input-border-color);
+ border-right: none;
margin-bottom: 0;
overflow-x: scroll;
text-wrap: nowrap;
@@ -150,7 +150,7 @@ gradio-app > :first-child {
.dark .header_bar {
border: none !important;
- box-shadow: 0 3px 4px rgba(20 20 20 / 60%);
+ box-shadow: none;
background-color: #8080802b;
}
@@ -268,17 +268,17 @@ button {
.dark #image-history-gallery > :nth-child(2)::-webkit-scrollbar-thumb,
.dark #image-history-gallery > :nth-child(2)::-webkit-scrollbar-thumb:hover {
background: rgb(255 255 255 / 6.25%);
- border-radius: 10px;
+ border-radius: 30px;
}
.pretty_scrollbar::-webkit-resizer,
#image-history-gallery > :nth-child(2)::-webkit-resizer {
- background: #c5c5d2;
+ background: #d2d2d8;
}
.dark .pretty_scrollbar::-webkit-resizer,
.dark #image-history-gallery > :nth-child(2)::-webkit-resizer {
- background: #ccc;
+ background: rgb(255 255 255 / 10%);
border-radius: 10px;
}
@@ -582,10 +582,28 @@ audio {
#chat-input textarea {
background: #f3f4f6;
- padding: 0.65rem 2.5rem;
- border: 0;
- box-shadow: 0;
- border-radius: 8px;
+ padding: 0.65rem 2.5rem 0.6rem;
+ margin-top: 0.15rem;
+ border: 1px solid #d2d2d8;
+ border-radius: 1.5rem;
+ overflow-y: auto !important;
+}
+
+#chat-input textarea::-webkit-scrollbar {
+ width: 8px;
+}
+
+#chat-input textarea::-webkit-scrollbar-track {
+ background: transparent;
+}
+
+#chat-input textarea::-webkit-scrollbar-thumb {
+ background: var(--neutral-300);
+ border-radius: 30px;
+}
+
+.dark #chat-input textarea::-webkit-scrollbar-thumb {
+ background: rgb(255 255 255 / 6.25%);
}
#chat-input textarea::placeholder {
@@ -725,10 +743,12 @@ audio {
position: absolute;
bottom: 100%;
left: 0;
- box-shadow: 0 0 5px rgb(0 0 0 / 25%);
+ box-shadow: 0 2px 12px rgb(0 0 0 / 15%);
+ border-radius: 0.5rem;
z-index: 10000;
min-width: 330px;
flex-direction: column;
+ overflow: hidden;
}
.hover-menu button {
@@ -739,6 +759,7 @@ audio {
margin: 0 !important;
height: 36px;
border-color: transparent !important;
+ transition: background-color 0.15s ease;
}
.hover-menu button:not(#clear-history-confirm) {
@@ -914,7 +935,7 @@ audio {
.options {
z-index: 100 !important;
border: 1px solid var(--input-border-color);
- border-radius: 0;
+ border-radius: 0.5rem;
}
/* ----------------------------------------------
@@ -1008,9 +1029,13 @@ audio {
cursor: pointer;
}
+#past-chats label {
+ transition: background-color 0.15s ease;
+}
+
#past-chats .selected,
#past-chats label:hover {
- background-color: #dbeafe !important;
+ background-color: #c8d8f5 !important;
}
#past-chats-buttons,
@@ -1166,7 +1191,7 @@ audio {
Dark theme
---------------------------------------------- */
.dark .header_bar {
- background-color: var(--darker-gray) !important;
+ background-color: #1a1a1a !important;
}
.dark .header_bar button.selected {
@@ -1176,7 +1201,7 @@ audio {
.dark #chat-input textarea {
background: var(--light-gray);
color: white !important;
- border-color: #292c3b;
+ border-color: rgba(255, 255, 255, 0.06);
}
.dark #chat-input textarea::placeholder {
@@ -1192,6 +1217,7 @@ audio {
.dark #past-chats-row {
background-color: var(--darker-gray);
border: 0 !important;
+ box-shadow: none;
}
.dark gradio-app .gradio-container.gradio-container-4-37-2 .contain #past-chats .selected,
@@ -1228,11 +1254,11 @@ audio {
Light theme
---------------------------------------------- */
.header_bar {
- background-color: var(--light-theme-gray) !important;
+ background-color: #e4e8f0 !important;
}
.header_bar button.selected {
- background: #dbeafe;
+ background: #c8d8f5;
}
#chat-controls,
@@ -1241,11 +1267,11 @@ audio {
}
.dark #chat-controls {
- border-left: 1px solid #d9d9d0;
+ border-left: 1px solid rgba(255, 255, 255, 0.06);
}
.dark #past-chats-row {
- border-right: 1px solid #d9d9d0;
+ border-right: 1px solid rgba(255, 255, 255, 0.06);
}
#past-chats-toggle,
@@ -1364,6 +1390,7 @@ audio {
.tgw-accordion {
padding: 10px 12px !important;
+ border: 1px solid #d2d2d8;
}
.dark .tgw-accordion {
@@ -1393,7 +1420,7 @@ audio {
}
.dark .thinking-block {
- background-color: transparent;
+ background-color: var(--darker-gray);
border: 1px solid var(--input-border-color);
}
@@ -1742,7 +1769,7 @@ button:focus {
}
.dark .sidebar-vertical-separator {
- border-bottom: 1px solid rgb(255 255 255 / 10%);
+ border-bottom: 1px solid rgba(255, 255, 255, 0.06);
}
button#swap-height-width {
@@ -1932,7 +1959,7 @@ thead + tbody tr:first-child th { border-top: 1px solid; }
.dark #tools-group .wrap::-webkit-scrollbar-thumb,
.dark #tools-group .wrap::-webkit-scrollbar-thumb:hover {
background: rgb(255 255 255 / 6.25%);
- border-radius: 10px;
+ border-radius: 30px;
}
#tools-group .wrap::-webkit-scrollbar-corner {
diff --git a/modules/ui.py b/modules/ui.py
index 3f39a1a4..bbb22266 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -66,7 +66,8 @@ theme = gr.themes.Default(
if not shared.args.old_colors:
theme = theme.set(
# General Colors
- border_color_primary='#c5c5d2',
+ border_color_primary='#d2d2d8',
+ block_border_color='transparent',
body_text_color_subdued='#484848',
background_fill_secondary='#eaeaea',
background_fill_secondary_dark='var(--selected-item-color-dark, #282930)',
@@ -77,6 +78,12 @@ if not shared.args.old_colors:
body_text_color='rgb(64, 64, 64)',
button_secondary_background_fill="white",
button_secondary_border_color="var(--border-color-primary)",
+ block_title_text_color='*body_text_color',
+ button_primary_background_fill='#374151',
+ button_primary_background_fill_hover='#4b5563',
+ button_primary_background_fill_hover_dark='rgba(255, 255, 255, 0.05)',
+ button_primary_border_color='#374151',
+ button_primary_text_color='white',
input_shadow="none",
button_shadow_hover="none",
@@ -85,11 +92,11 @@ if not shared.args.old_colors:
checkbox_background_color_dark='var(--darker-gray, #1C1C1D)',
block_background_fill_dark='transparent',
block_border_color_dark='transparent',
- input_border_color_dark='var(--border-color-dark, #525252)',
- input_border_color_focus_dark='var(--border-color-dark, #525252)',
- checkbox_border_color_dark='var(--border-color-dark, #525252)',
- border_color_primary_dark='var(--border-color-dark, #525252)',
- button_secondary_border_color_dark='var(--border-color-dark, #525252)',
+ input_border_color_dark='var(--border-color-dark)',
+ input_border_color_focus_dark='var(--border-color-dark)',
+ checkbox_border_color_dark='rgba(255, 255, 255, 0.2)',
+ border_color_primary_dark='var(--border-color-dark)',
+ button_secondary_border_color_dark='var(--border-color-dark)',
body_background_fill_dark='var(--dark-gray, #212125)',
button_primary_background_fill_dark='transparent',
button_secondary_background_fill_dark='transparent',
@@ -107,10 +114,11 @@ if not shared.args.old_colors:
block_shadow_dark='none',
input_shadow_focus='none',
input_shadow_focus_dark='none',
- button_large_radius='0.375rem',
+ button_large_radius='0.75rem',
button_large_padding='6px 12px',
- input_radius='0.375rem',
- block_radius='0',
+ input_radius='0.5rem',
+ block_radius='0.375rem',
+ button_transition='background-color 0.15s ease, border-color 0.15s ease, color 0.15s ease',
)
if (shared.user_data_dir / "notification.mp3").exists():
diff --git a/modules/ui_session.py b/modules/ui_session.py
index e1807dea..c0615843 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -17,7 +17,7 @@ def create_ui():
with gr.Column():
gr.Markdown("## Extensions & flags")
- shared.gradio['save_settings'] = gr.Button(f'Save extensions settings to {shared.user_data_dir}/settings.yaml', elem_classes='refresh-button', interactive=not mu)
+ shared.gradio['save_settings'] = gr.Button(f'Save extensions settings to {shared.user_data_dir}/settings.yaml', interactive=not mu)
shared.gradio['reset_interface'] = gr.Button("Apply flags/extensions and restart", interactive=not mu)
with gr.Row():
with gr.Column():
From 5992e088faf94e5161f50d1dcf5996a10051d71c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 16 Mar 2026 19:34:37 -0700
Subject: [PATCH 06/47] Update the custom gradio wheels
---
requirements/full/requirements.txt | 4 ++--
requirements/full/requirements_amd.txt | 4 ++--
requirements/full/requirements_apple_intel.txt | 4 ++--
requirements/full/requirements_apple_silicon.txt | 4 ++--
requirements/full/requirements_cpu_only.txt | 4 ++--
requirements/full/requirements_nowheels.txt | 4 ++--
requirements/portable/requirements.txt | 4 ++--
requirements/portable/requirements_amd.txt | 4 ++--
requirements/portable/requirements_apple_intel.txt | 4 ++--
requirements/portable/requirements_apple_silicon.txt | 4 ++--
requirements/portable/requirements_cpu_only.txt | 4 ++--
requirements/portable/requirements_cuda131.txt | 4 ++--
requirements/portable/requirements_nowheels.txt | 4 ++--
requirements/portable/requirements_vulkan.txt | 4 ++--
14 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index c24f4a9d..ee83ce56 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -31,8 +31,8 @@ tqdm
wandb
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt
index 7c481224..ae211301 100644
--- a/requirements/full/requirements_amd.txt
+++ b/requirements/full/requirements_amd.txt
@@ -28,8 +28,8 @@ trafilatura==2.0.0
wandb
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt
index b1c8f78e..158fc004 100644
--- a/requirements/full/requirements_apple_intel.txt
+++ b/requirements/full/requirements_apple_intel.txt
@@ -28,8 +28,8 @@ trafilatura==2.0.0
wandb
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt
index 63ef33ea..f691d872 100644
--- a/requirements/full/requirements_apple_silicon.txt
+++ b/requirements/full/requirements_apple_silicon.txt
@@ -28,8 +28,8 @@ trafilatura==2.0.0
wandb
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt
index 4bc61622..116db442 100644
--- a/requirements/full/requirements_cpu_only.txt
+++ b/requirements/full/requirements_cpu_only.txt
@@ -28,8 +28,8 @@ trafilatura==2.0.0
wandb
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/full/requirements_nowheels.txt b/requirements/full/requirements_nowheels.txt
index 2ec1e61e..62f12e1b 100644
--- a/requirements/full/requirements_nowheels.txt
+++ b/requirements/full/requirements_nowheels.txt
@@ -28,8 +28,8 @@ trafilatura==2.0.0
wandb
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt
index ba4c7a04..d6e7896c 100644
--- a/requirements/portable/requirements.txt
+++ b/requirements/portable/requirements.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_amd.txt b/requirements/portable/requirements_amd.txt
index 5dfdd9c8..26555e30 100644
--- a/requirements/portable/requirements_amd.txt
+++ b/requirements/portable/requirements_amd.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt
index f62241b3..49f4c553 100644
--- a/requirements/portable/requirements_apple_intel.txt
+++ b/requirements/portable/requirements_apple_intel.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt
index 353d9172..6d8f4780 100644
--- a/requirements/portable/requirements_apple_silicon.txt
+++ b/requirements/portable/requirements_apple_silicon.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt
index 5f039318..9764b2e3 100644
--- a/requirements/portable/requirements_cpu_only.txt
+++ b/requirements/portable/requirements_cpu_only.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_cuda131.txt b/requirements/portable/requirements_cuda131.txt
index d8b03102..903da78a 100644
--- a/requirements/portable/requirements_cuda131.txt
+++ b/requirements/portable/requirements_cuda131.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_nowheels.txt b/requirements/portable/requirements_nowheels.txt
index 4b548dae..0360efdd 100644
--- a/requirements/portable/requirements_nowheels.txt
+++ b/requirements/portable/requirements_nowheels.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt
index fd2511f4..08b663e9 100644
--- a/requirements/portable/requirements_vulkan.txt
+++ b/requirements/portable/requirements_vulkan.txt
@@ -14,8 +14,8 @@ trafilatura==2.0.0
tqdm
# Gradio
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio-4.37.2+custom.11-py3-none-any.whl
-https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.11/gradio_client-1.0.2+custom.11-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio-4.37.2+custom.12-py3-none-any.whl
+https://github.com/oobabooga/gradio/releases/download/4.37.2-custom.12/gradio_client-1.0.2+custom.12-py3-none-any.whl
# API
flask_cloudflared==0.0.15
From 249861b65d0585f3cb290aaeb3d9050c18501ef3 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 05:41:05 -0700
Subject: [PATCH 07/47] web search: Update the user agents
---
modules/web_search.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/modules/web_search.py b/modules/web_search.py
index e13ef62a..2902c7c0 100644
--- a/modules/web_search.py
+++ b/modules/web_search.py
@@ -48,7 +48,7 @@ def download_web_page(url, timeout=10, include_links=False):
try:
_validate_url(url)
headers = {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
}
max_redirects = 5
for _ in range(max_redirects):
@@ -82,8 +82,8 @@ def perform_web_search(query, num_pages=3, max_workers=5, timeout=10, fetch_cont
search_url = f"https://html.duckduckgo.com/html/?q={quote_plus(query)}"
agents = [
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
]
response = requests.get(search_url, headers={'User-Agent': random.choice(agents)}, timeout=timeout)
From fffcd20f4d83d81b2577c4b9a94352cf8ed64484 Mon Sep 17 00:00:00 2001
From: Raunak-Kumar7 <73169853+Raunak-Kumar7@users.noreply.github.com>
Date: Tue, 17 Mar 2026 23:14:54 +0530
Subject: [PATCH 08/47] superboogav2: Fix broken delete endpoint (#6010)
---
extensions/superboogav2/api.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/extensions/superboogav2/api.py b/extensions/superboogav2/api.py
index 552c1c2c..99b0e749 100644
--- a/extensions/superboogav2/api.py
+++ b/extensions/superboogav2/api.py
@@ -107,7 +107,7 @@ class Handler(BaseHTTPRequestHandler):
elif path in ['/api/v1/delete', '/api/delete']:
metadata = body.get('metadata')
- if corpus is None:
+ if metadata is None:
self._send_412_error("Missing parameter 'metadata'")
return
From 2d141b54c5e0b5e042826e3d2f46bbaf87db023d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 11:11:12 -0700
Subject: [PATCH 09/47] Fix several typos
---
README.md | 2 +-
extensions/whisper_stt/readme.md | 4 ++--
extensions/whisper_stt/script.py | 24 ++++++++++++------------
modules/shared.py | 2 +-
modules/ui_model_menu.py | 6 +++---
5 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/README.md b/README.md
index 989659d1..cabb81fc 100644
--- a/README.md
+++ b/README.md
@@ -313,7 +313,7 @@ llama.cpp:
--row-split Split the model by rows across GPUs. This may improve multi-gpu performance.
--no-mmap Prevent mmap from being used.
--mlock Force the system to keep the model in RAM.
- --no-kv-offload Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.
+ --no-kv-offload Do not offload the K, Q, V to the GPU. This saves VRAM but reduces performance.
--batch-size BATCH_SIZE Maximum number of prompt tokens to batch together when calling llama-server. This is the application level batch size.
--ubatch-size UBATCH_SIZE Maximum number of prompt tokens to batch together when calling llama-server. This is the max physical batch size for computation (device level).
--threads THREADS Number of threads to use.
diff --git a/extensions/whisper_stt/readme.md b/extensions/whisper_stt/readme.md
index 19488f94..7d9d8d23 100644
--- a/extensions/whisper_stt/readme.md
+++ b/extensions/whisper_stt/readme.md
@@ -7,8 +7,8 @@ Allows you to enter your inputs in chat mode using your microphone.
To adjust your default settings, you can add the following to your settings.yaml file.
```
-whisper_stt-whipser_language: chinese
-whisper_stt-whipser_model: tiny
+whisper_stt-whisper_language: chinese
+whisper_stt-whisper_model: tiny
whisper_stt-auto_submit: False
```
diff --git a/extensions/whisper_stt/script.py b/extensions/whisper_stt/script.py
index d949e93f..cd9175fe 100644
--- a/extensions/whisper_stt/script.py
+++ b/extensions/whisper_stt/script.py
@@ -18,13 +18,13 @@ input_hijack = {
# parameters which can be customized in settings.yaml of webui
params = {
- 'whipser_language': 'english',
- 'whipser_model': 'small.en',
+ 'whisper_language': 'english',
+ 'whisper_model': 'small.en',
'auto_submit': True
}
startup_device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-WHISPERMODEL = whisper.load_model(params['whipser_model'], device=startup_device)
+WHISPERMODEL = whisper.load_model(params['whisper_model'], device=startup_device)
def chat_input_modifier(text, visible_text, state):
@@ -36,7 +36,7 @@ def chat_input_modifier(text, visible_text, state):
return text, visible_text
-def do_stt(audio, whipser_language):
+def do_stt(audio, whisper_language):
# use pydub to convert sample_rate and sample_width for whisper input
dubaudio = AudioSegment.from_file(io.BytesIO(audio))
dubaudio = dubaudio.set_channels(1)
@@ -46,20 +46,20 @@ def do_stt(audio, whipser_language):
# same method to get the array as openai whisper repo used from wav file
audio_np = np.frombuffer(dubaudio.raw_data, np.int16).flatten().astype(np.float32) / 32768.0
- if len(whipser_language) == 0:
+ if len(whisper_language) == 0:
result = WHISPERMODEL.transcribe(audio=audio_np)
else:
- result = WHISPERMODEL.transcribe(audio=audio_np, language=whipser_language)
+ result = WHISPERMODEL.transcribe(audio=audio_np, language=whisper_language)
return result["text"]
-def auto_transcribe(audio, auto_submit, whipser_language):
+def auto_transcribe(audio, auto_submit, whisper_language):
if audio is None or audio == "":
print("Whisper received no audio data")
return "", ""
audio_bytes = base64.b64decode(audio.split(',')[1])
- transcription = do_stt(audio_bytes, whipser_language)
+ transcription = do_stt(audio_bytes, whisper_language)
if auto_submit:
input_hijack.update({"state": True, "value": [transcription, transcription]})
return transcription
@@ -78,7 +78,7 @@ def reload_whispermodel(whisper_model_name: str, whisper_language: str, device:
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
WHISPERMODEL = whisper.load_model(whisper_model_name, device=device)
- params.update({"whipser_model": whisper_model_name})
+ params.update({"whisper_model": whisper_model_name})
if ".en" in whisper_model_name:
whisper_language = "english"
audio_update = gr.Audio.update(interactive=True)
@@ -96,8 +96,8 @@ def ui():
with gr.Accordion("Settings", open=False):
auto_submit = gr.Checkbox(label='Submit the transcribed audio automatically', value=params['auto_submit'])
device_dropd = gr.Dropdown(label='Device', value=str(startup_device), choices=["cuda", "cpu", "none"])
- whisper_model_dropd = gr.Dropdown(label='Whisper Model', value=params['whipser_model'], choices=["tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "large", "turbo"])
- whisper_language = gr.Dropdown(label='Whisper Language', value=params['whipser_language'], choices=["english", "chinese", "german", "spanish", "russian", "korean", "french", "japanese", "portuguese", "turkish", "polish", "catalan", "dutch", "arabic", "swedish", "italian", "indonesian", "hindi", "finnish", "vietnamese", "hebrew", "ukrainian", "greek", "malay", "czech", "romanian", "danish", "hungarian", "tamil", "norwegian", "thai", "urdu", "croatian", "bulgarian", "lithuanian", "latin", "maori", "malayalam", "welsh", "slovak", "telugu", "persian", "latvian", "bengali", "serbian", "azerbaijani", "slovenian", "kannada", "estonian", "macedonian", "breton", "basque", "icelandic", "armenian", "nepali", "mongolian", "bosnian", "kazakh", "albanian", "swahili", "galician", "marathi", "punjabi", "sinhala", "khmer", "shona", "yoruba", "somali", "afrikaans", "occitan", "georgian", "belarusian", "tajik", "sindhi", "gujarati", "amharic", "yiddish", "lao", "uzbek", "faroese", "haitian creole", "pashto", "turkmen", "nynorsk", "maltese", "sanskrit", "luxembourgish", "myanmar", "tibetan", "tagalog", "malagasy", "assamese", "tatar", "hawaiian", "lingala", "hausa", "bashkir", "javanese", "sundanese"])
+ whisper_model_dropd = gr.Dropdown(label='Whisper Model', value=params['whisper_model'], choices=["tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "large", "turbo"])
+ whisper_language = gr.Dropdown(label='Whisper Language', value=params['whisper_language'], choices=["english", "chinese", "german", "spanish", "russian", "korean", "french", "japanese", "portuguese", "turkish", "polish", "catalan", "dutch", "arabic", "swedish", "italian", "indonesian", "hindi", "finnish", "vietnamese", "hebrew", "ukrainian", "greek", "malay", "czech", "romanian", "danish", "hungarian", "tamil", "norwegian", "thai", "urdu", "croatian", "bulgarian", "lithuanian", "latin", "maori", "malayalam", "welsh", "slovak", "telugu", "persian", "latvian", "bengali", "serbian", "azerbaijani", "slovenian", "kannada", "estonian", "macedonian", "breton", "basque", "icelandic", "armenian", "nepali", "mongolian", "bosnian", "kazakh", "albanian", "swahili", "galician", "marathi", "punjabi", "sinhala", "khmer", "shona", "yoruba", "somali", "afrikaans", "occitan", "georgian", "belarusian", "tajik", "sindhi", "gujarati", "amharic", "yiddish", "lao", "uzbek", "faroese", "haitian creole", "pashto", "turkmen", "nynorsk", "maltese", "sanskrit", "luxembourgish", "myanmar", "tibetan", "tagalog", "malagasy", "assamese", "tatar", "hawaiian", "lingala", "hausa", "bashkir", "javanese", "sundanese"])
audio.change(
auto_transcribe, [audio, auto_submit, whisper_language], [shared.gradio['textbox']]).then(
@@ -105,7 +105,7 @@ def ui():
device_dropd.input(reload_whispermodel, [whisper_model_dropd, whisper_language, device_dropd], [whisper_model_dropd, whisper_language, device_dropd, audio])
whisper_model_dropd.change(reload_whispermodel, [whisper_model_dropd, whisper_language, device_dropd], [whisper_model_dropd, whisper_language, device_dropd, audio])
- whisper_language.change(lambda x: params.update({"whipser_language": x}), whisper_language, None)
+ whisper_language.change(lambda x: params.update({"whisper_language": x}), whisper_language, None)
auto_submit.change(lambda x: params.update({"auto_submit": x}), auto_submit, None)
diff --git a/modules/shared.py b/modules/shared.py
index 329114bb..486f376f 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -101,7 +101,7 @@ group.add_argument('--tensor-split', type=str, default=None, help='Split the mod
group.add_argument('--row-split', action='store_true', help='Split the model by rows across GPUs. This may improve multi-gpu performance.')
group.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.')
group.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.')
-group.add_argument('--no-kv-offload', action='store_true', help='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.')
+group.add_argument('--no-kv-offload', action='store_true', help='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces performance.')
group.add_argument('--batch-size', type=int, default=1024, help='Maximum number of prompt tokens to batch together when calling llama-server. This is the application level batch size.')
group.add_argument('--ubatch-size', type=int, default=1024, help='Maximum number of prompt tokens to batch together when calling llama-server. This is the max physical batch size for computation (device level).')
group.add_argument('--threads', type=int, default=0, help='Number of threads to use.')
diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py
index 5cf0155d..cb2052a4 100644
--- a/modules/ui_model_menu.py
+++ b/modules/ui_model_menu.py
@@ -107,7 +107,7 @@ def create_ui():
shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu, info='Use PyTorch in CPU mode.')
shared.gradio['disk'] = gr.Checkbox(label="disk", value=shared.args.disk)
shared.gradio['row_split'] = gr.Checkbox(label="row_split", value=shared.args.row_split, info='Split the model by rows across GPUs. This may improve multi-gpu performance.')
- shared.gradio['no_kv_offload'] = gr.Checkbox(label="no_kv_offload", value=shared.args.no_kv_offload, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.')
+ shared.gradio['no_kv_offload'] = gr.Checkbox(label="no_kv_offload", value=shared.args.no_kv_offload, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces performance.')
shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap)
shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock)
shared.gradio['numa'] = gr.Checkbox(label="numa", value=shared.args.numa, info='NUMA support can help on some systems with non-uniform memory access.')
@@ -134,7 +134,7 @@ def create_ui():
ui.create_refresh_button(shared.gradio['customized_template'], lambda: None, lambda: {'choices': utils.get_available_instruction_templates()}, 'refresh-button', interactive=not mu)
shared.gradio['customized_template_submit'] = gr.Button("Submit", variant="primary", interactive=not mu)
- gr.Markdown("This allows you to set a customized template for the model currently selected in the \"Model loader\" menu. Whenever the model gets loaded, this template will be used in place of the template specified in the model's medatada, which sometimes is wrong.")
+ gr.Markdown("This allows you to set a customized template for the model currently selected in the \"Model loader\" menu. Whenever the model gets loaded, this template will be used in place of the template specified in the model's metadata, which sometimes is wrong.")
with gr.Row():
shared.gradio['model_status'] = gr.Markdown('No model is loaded' if shared.model_name == 'None' else 'Ready')
@@ -231,7 +231,7 @@ def load_model_wrapper(selected_model, loader, autoload=False):
def load_lora_wrapper(selected_loras):
yield ("Applying the following LoRAs to {}:\n\n{}".format(shared.model_name, '\n'.join(selected_loras)))
add_lora_to_model(selected_loras)
- yield ("Successfuly applied the LoRAs")
+ yield ("Successfully applied the LoRAs")
def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), return_links=False, check=False):
From 27a6cdeec11dc2f1536db3c846bb89a93efbdd69 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 11:31:55 -0700
Subject: [PATCH 10/47] Fix multi-turn thinking block corruption for Kimi
models
---
modules/chat.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/modules/chat.py b/modules/chat.py
index e4fcaabe..e526689d 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -235,6 +235,7 @@ def generate_chat_prompt(user_input, state, **kwargs):
tools_in_user_message=False,
add_generation_prompt=False,
enable_thinking=state['enable_thinking'],
+ thinking=state['enable_thinking'],
reasoning_effort=state['reasoning_effort'],
thinking_budget=-1 if state.get('enable_thinking', True) else 0,
bos_token=shared.bos_token,
@@ -351,6 +352,27 @@ def generate_chat_prompt(user_input, state, **kwargs):
messages.insert(insert_pos, msg_dict)
+ # Handle blocks (Kimi, DeepSeek, Qwen, etc.)
+ elif '' in assistant_msg:
+ thinking_content = ""
+ final_content = assistant_msg
+
+ parts = assistant_msg.split('', 1)
+ if len(parts) > 1:
+ potential_content = parts[1]
+ if '' in potential_content:
+ thinking_content = potential_content.split('', 1)[0].strip()
+ final_content = parts[0] + potential_content.split('', 1)[1]
+ else:
+ thinking_content = potential_content.strip()
+ final_content = parts[0]
+
+ msg_dict = {"role": "assistant", "content": final_content.strip()}
+ if thinking_content:
+ msg_dict["reasoning_content"] = thinking_content
+
+ messages.insert(insert_pos, msg_dict)
+
else:
# Default case (used by all other models)
messages.insert(insert_pos, {"role": "assistant", "content": assistant_msg})
From 0f5053c0fbe4177b3b5af199d7301cc5e1bca0ac Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 17:57:35 -0700
Subject: [PATCH 11/47] requirements: Update pymupdf
---
requirements/full/requirements.txt | 2 +-
requirements/full/requirements_amd.txt | 2 +-
requirements/full/requirements_apple_intel.txt | 2 +-
requirements/full/requirements_apple_silicon.txt | 2 +-
requirements/full/requirements_cpu_only.txt | 2 +-
requirements/full/requirements_nowheels.txt | 2 +-
requirements/portable/requirements.txt | 2 +-
requirements/portable/requirements_amd.txt | 2 +-
requirements/portable/requirements_apple_intel.txt | 2 +-
requirements/portable/requirements_apple_silicon.txt | 2 +-
requirements/portable/requirements_cpu_only.txt | 2 +-
requirements/portable/requirements_cuda131.txt | 2 +-
requirements/portable/requirements_nowheels.txt | 2 +-
requirements/portable/requirements_vulkan.txt | 2 +-
14 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index ee83ce56..c8479d04 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -14,7 +14,7 @@ pandas
peft==0.18.*
Pillow>=9.5.0
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt
index ae211301..b11e50b7 100644
--- a/requirements/full/requirements_amd.txt
+++ b/requirements/full/requirements_amd.txt
@@ -12,7 +12,7 @@ pandas
peft==0.18.*
Pillow>=9.5.0
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt
index 158fc004..d147af3f 100644
--- a/requirements/full/requirements_apple_intel.txt
+++ b/requirements/full/requirements_apple_intel.txt
@@ -12,7 +12,7 @@ pandas
peft==0.18.*
Pillow>=9.5.0
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt
index f691d872..d284c5d5 100644
--- a/requirements/full/requirements_apple_silicon.txt
+++ b/requirements/full/requirements_apple_silicon.txt
@@ -12,7 +12,7 @@ pandas
peft==0.18.*
Pillow>=9.5.0
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt
index 116db442..3952054e 100644
--- a/requirements/full/requirements_cpu_only.txt
+++ b/requirements/full/requirements_cpu_only.txt
@@ -12,7 +12,7 @@ pandas
peft==0.18.*
Pillow>=9.5.0
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/full/requirements_nowheels.txt b/requirements/full/requirements_nowheels.txt
index 62f12e1b..77c254e6 100644
--- a/requirements/full/requirements_nowheels.txt
+++ b/requirements/full/requirements_nowheels.txt
@@ -12,7 +12,7 @@ pandas
peft==0.18.*
Pillow>=9.5.0
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt
index d6e7896c..abf7690c 100644
--- a/requirements/portable/requirements.txt
+++ b/requirements/portable/requirements.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_amd.txt b/requirements/portable/requirements_amd.txt
index 26555e30..0d66c16c 100644
--- a/requirements/portable/requirements_amd.txt
+++ b/requirements/portable/requirements_amd.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt
index 49f4c553..0658239a 100644
--- a/requirements/portable/requirements_apple_intel.txt
+++ b/requirements/portable/requirements_apple_intel.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt
index 6d8f4780..b66e2b38 100644
--- a/requirements/portable/requirements_apple_silicon.txt
+++ b/requirements/portable/requirements_apple_silicon.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt
index 9764b2e3..bb815bb2 100644
--- a/requirements/portable/requirements_cpu_only.txt
+++ b/requirements/portable/requirements_cpu_only.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_cuda131.txt b/requirements/portable/requirements_cuda131.txt
index 903da78a..d57ba40b 100644
--- a/requirements/portable/requirements_cuda131.txt
+++ b/requirements/portable/requirements_cuda131.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_nowheels.txt b/requirements/portable/requirements_nowheels.txt
index 0360efdd..e8457909 100644
--- a/requirements/portable/requirements_nowheels.txt
+++ b/requirements/portable/requirements_nowheels.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt
index 08b663e9..6abd8920 100644
--- a/requirements/portable/requirements_vulkan.txt
+++ b/requirements/portable/requirements_vulkan.txt
@@ -5,7 +5,7 @@ jinja2==3.1.6
markdown
numpy==2.2.*
pydantic==2.11.0
-pymupdf==1.27.1
+pymupdf==1.27.*
python-docx==1.1.2
pyyaml
requests
From f0014ab01c7a51bfa0f269c676404d48112f924b Mon Sep 17 00:00:00 2001
From: RoomWithOutRoof <166608075+Jah-yee@users.noreply.github.com>
Date: Wed, 18 Mar 2026 09:03:48 +0800
Subject: [PATCH 12/47] fix: mutable default argument in LogitsBiasProcessor
(#7426)
---
modules/transformers_loader.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/transformers_loader.py b/modules/transformers_loader.py
index 63758ad7..7f521b8c 100644
--- a/modules/transformers_loader.py
+++ b/modules/transformers_loader.py
@@ -44,8 +44,8 @@ class Stream(transformers.StoppingCriteria):
class LogitsBiasProcessor(LogitsProcessor):
- def __init__(self, logit_bias={}):
- self.logit_bias = logit_bias
+ def __init__(self, logit_bias=None):
+ self.logit_bias = logit_bias if logit_bias is not None else {}
if self.logit_bias:
self.keys = list([int(key) for key in self.logit_bias.keys()])
values = [self.logit_bias[str(key)] for key in self.keys]
From 73a094a65773a3f2f9e7d626cfaa01893dbd3f88 Mon Sep 17 00:00:00 2001
From: Alvin Tang
Date: Wed, 18 Mar 2026 09:06:05 +0800
Subject: [PATCH 13/47] Fix file handle leaks and redundant re-read in
get_model_metadata (#7422)
---
modules/models_settings.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/modules/models_settings.py b/modules/models_settings.py
index f3c9a986..dcface71 100644
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -34,7 +34,8 @@ def get_model_metadata(model):
path = model_path / 'config.json'
if path.exists():
- hf_metadata = json.loads(open(path, 'r', encoding='utf-8').read())
+ with open(path, 'r', encoding='utf-8') as f:
+ hf_metadata = json.loads(f.read())
else:
hf_metadata = None
@@ -93,7 +94,7 @@ def get_model_metadata(model):
else:
# Transformers metadata
if hf_metadata is not None:
- metadata = json.loads(open(path, 'r', encoding='utf-8').read())
+ metadata = hf_metadata
if 'pretrained_config' in metadata:
metadata = metadata['pretrained_config']
@@ -134,7 +135,8 @@ def get_model_metadata(model):
# 3. Fall back to tokenizer_config.json metadata
if path.exists():
- metadata = json.loads(open(path, 'r', encoding='utf-8').read())
+ with open(path, 'r', encoding='utf-8') as f:
+ metadata = json.loads(f.read())
# Only read from metadata if we haven't already loaded from .jinja or .json
if template is None and 'chat_template' in metadata:
From 2a6b1fdcba676200d2e454534a91e1d334b60bdf Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 18:29:15 -0700
Subject: [PATCH 14/47] Fix `--extra-flags` breaking short long-form-only flags
like `--rpc`
Closes #7357
---
modules/llama_cpp_server.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index c3a8d105..321a6d75 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -446,18 +446,21 @@ class LlamaServer:
elif extra_flags.startswith("'") and extra_flags.endswith("'"):
extra_flags = extra_flags[1:-1].strip()
+ # llama.cpp flags that only have a long form (--) despite being short
+ long_form_only = {'rpc', 'fit', 'pos', 'ppl'}
+
for flag_item in extra_flags.split(','):
flag_item = flag_item.strip()
if '=' in flag_item:
flag, value = flag_item.split('=', 1)
flag = flag.strip()
value = value.strip()
- if len(flag) <= 3:
+ if len(flag) <= 3 and flag not in long_form_only:
cmd += [f"-{flag}", value]
else:
cmd += [f"--{flag}", value]
else:
- if len(flag_item) <= 3:
+ if len(flag_item) <= 3 and flag_item not in long_form_only:
cmd.append(f"-{flag_item}")
else:
cmd.append(f"--{flag_item}")
From 7e54e7b7ae62b227fbd896b2daf704db1658baa5 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 19:47:55 -0700
Subject: [PATCH 15/47] llama.cpp: Support literal flags in `--extra-flags`
(e.g. `--rpc`, `--jinja`)
The old format is still accepted for backwards compatibility.
---
modules/llama_cpp_server.py | 37 +++++++++++++++++++++----------------
modules/shared.py | 2 +-
modules/ui_model_menu.py | 2 +-
3 files changed, 23 insertions(+), 18 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index 321a6d75..6dd36b2a 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -1,6 +1,7 @@
import json
import os
import pprint
+import shlex
import re
import socket
import subprocess
@@ -446,24 +447,28 @@ class LlamaServer:
elif extra_flags.startswith("'") and extra_flags.endswith("'"):
extra_flags = extra_flags[1:-1].strip()
- # llama.cpp flags that only have a long form (--) despite being short
- long_form_only = {'rpc', 'fit', 'pos', 'ppl'}
+ if extra_flags.startswith('-'):
+ # New literal format: "--jinja --rpc 1222,1222"
+ cmd += shlex.split(extra_flags)
+ else:
+ # Legacy format: "flag1=value1,flag2,flag3=value3"
+ long_form_only = {'rpc', 'fit', 'pos', 'ppl'}
- for flag_item in extra_flags.split(','):
- flag_item = flag_item.strip()
- if '=' in flag_item:
- flag, value = flag_item.split('=', 1)
- flag = flag.strip()
- value = value.strip()
- if len(flag) <= 3 and flag not in long_form_only:
- cmd += [f"-{flag}", value]
+ for flag_item in extra_flags.split(','):
+ flag_item = flag_item.strip()
+ if '=' in flag_item:
+ flag, value = flag_item.split('=', 1)
+ flag = flag.strip()
+ value = value.strip()
+ if len(flag) <= 3 and flag not in long_form_only:
+ cmd += [f"-{flag}", value]
+ else:
+ cmd += [f"--{flag}", value]
else:
- cmd += [f"--{flag}", value]
- else:
- if len(flag_item) <= 3 and flag_item not in long_form_only:
- cmd.append(f"-{flag_item}")
- else:
- cmd.append(f"--{flag_item}")
+ if len(flag_item) <= 3 and flag_item not in long_form_only:
+ cmd.append(f"-{flag_item}")
+ else:
+ cmd.append(f"--{flag_item}")
env = os.environ.copy()
if os.name == 'posix':
diff --git a/modules/shared.py b/modules/shared.py
index 486f376f..2382e714 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -109,7 +109,7 @@ group.add_argument('--threads-batch', type=int, default=0, help='Number of threa
group.add_argument('--numa', action='store_true', help='Activate NUMA task allocation for llama.cpp.')
group.add_argument('--parallel', type=int, default=1, help='Number of parallel request slots. The context size is divided equally among slots. For example, to have 4 slots with 8192 context each, set ctx_size to 32768.')
group.add_argument('--fit-target', type=str, default='512', help='Target VRAM margin per device for auto GPU layers, comma-separated list of values in MiB. A single value is broadcast across all devices.')
-group.add_argument('--extra-flags', type=str, default=None, help='Extra flags to pass to llama-server. Format: "flag1=value1,flag2,flag3=value3". Example: "override-tensor=exps=CPU"')
+group.add_argument('--extra-flags', type=str, default=None, help='Extra flags to pass to llama-server. Example: "--jinja --rpc 192.168.1.100:50052"')
# Transformers/Accelerate
group = parser.add_argument_group('Transformers/Accelerate')
diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py
index cb2052a4..6d8baff1 100644
--- a/modules/ui_model_menu.py
+++ b/modules/ui_model_menu.py
@@ -98,7 +98,7 @@ def create_ui():
shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size)
shared.gradio['ubatch_size'] = gr.Slider(label="ubatch_size", minimum=1, maximum=4096, step=1, value=shared.args.ubatch_size)
shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='List of proportions to split the model across multiple GPUs. Example: 60,40')
- shared.gradio['extra_flags'] = gr.Textbox(label='extra-flags', info='Additional flags to pass to llama-server. Format: "flag1=value1,flag2,flag3=value3". Example: "override-tensor=exps=CPU"', value=shared.args.extra_flags)
+ shared.gradio['extra_flags'] = gr.Textbox(label='extra-flags', info='Extra flags to pass to llama-server. Example: --jinja --rpc 192.168.1.100:50052', value=shared.args.extra_flags)
shared.gradio['cpu_memory'] = gr.Number(label="Maximum CPU memory in GiB. Use this for CPU offloading.", value=shared.args.cpu_memory)
shared.gradio['compute_dtype'] = gr.Dropdown(label="compute_dtype", choices=["bfloat16", "float16", "float32"], value=shared.args.compute_dtype, info='Used by load-in-4bit.')
shared.gradio['quant_type'] = gr.Dropdown(label="quant_type", choices=["nf4", "fp4"], value=shared.args.quant_type, info='Used by load-in-4bit.')
From c8bb2129baf180c3d3a5d1d410d1e78dc5ddbea3 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 22:24:36 -0700
Subject: [PATCH 16/47] Security: server-side file save roots, image URL SSRF
protection, extension allowlist
---
modules/chat.py | 8 +++++--
modules/image_utils.py | 13 ++++++++++-
modules/ui_chat.py | 4 ++--
modules/ui_file_saving.py | 46 +++++++++++++++++++++++++++------------
modules/ui_session.py | 6 +++--
modules/utils.py | 4 ++++
6 files changed, 60 insertions(+), 21 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index e526689d..00f1659b 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -2634,19 +2634,23 @@ def handle_load_template_click(instruction_template):
def handle_save_template_click(instruction_template_str):
import gradio as gr
contents = generate_instruction_template_yaml(instruction_template_str)
+ root = str(shared.user_data_dir / 'instruction-templates') + '/'
return [
"My Template.yaml",
- str(shared.user_data_dir / 'instruction-templates') + '/',
+ root,
contents,
+ root,
gr.update(visible=True)
]
def handle_delete_template_click(template):
import gradio as gr
+ root = str(shared.user_data_dir / 'instruction-templates') + '/'
return [
f"{template}.yaml",
- str(shared.user_data_dir / 'instruction-templates') + '/',
+ root,
+ root,
gr.update(visible=False)
]
diff --git a/modules/image_utils.py b/modules/image_utils.py
index d2809fef..b3138790 100644
--- a/modules/image_utils.py
+++ b/modules/image_utils.py
@@ -77,7 +77,18 @@ def process_message_content(content: Any) -> Tuple[str, List[Image.Image]]:
# Support external URLs
try:
import requests
- response = requests.get(image_url, timeout=10)
+ from urllib.parse import urljoin
+ from modules.web_search import _validate_url
+ _validate_url(image_url)
+ url = image_url
+ for _ in range(5):
+ response = requests.get(url, timeout=10, allow_redirects=False)
+ if response.is_redirect and 'Location' in response.headers:
+ url = urljoin(url, response.headers['Location'])
+ _validate_url(url)
+ else:
+ break
+
response.raise_for_status()
image_data = response.content
image = Image.open(io.BytesIO(image_data))
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index d2a515b8..f1dc7883 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -350,13 +350,13 @@ def create_event_handlers():
shared.gradio['load_template'].click(chat.handle_load_template_click, gradio('instruction_template'), gradio('instruction_template_str', 'instruction_template'), show_progress=False)
shared.gradio['save_template'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
- chat.handle_save_template_click, gradio('instruction_template_str'), gradio('save_filename', 'save_root', 'save_contents', 'file_saver'), show_progress=False)
+ chat.handle_save_template_click, gradio('instruction_template_str'), gradio('save_filename', 'save_root', 'save_contents', 'save_root_state', 'file_saver'), show_progress=False)
shared.gradio['restore_character'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
chat.restore_character_for_ui, gradio('interface_state'), gradio('interface_state', 'name2', 'context', 'greeting', 'character_picture'), show_progress=False)
- shared.gradio['delete_template'].click(chat.handle_delete_template_click, gradio('instruction_template'), gradio('delete_filename', 'delete_root', 'file_deleter'), show_progress=False)
+ shared.gradio['delete_template'].click(chat.handle_delete_template_click, gradio('instruction_template'), gradio('delete_filename', 'delete_root', 'delete_root_state', 'file_deleter'), show_progress=False)
shared.gradio['save_chat_history'].click(
lambda x: json.dumps(x, indent=4), gradio('history'), gradio('temporary_text')).then(
None, gradio('temporary_text', 'character_menu', 'mode'), None, js=f'(hist, char, mode) => {{{ui.save_files_js}; saveHistory(hist, char, mode)}}')
diff --git a/modules/ui_file_saving.py b/modules/ui_file_saving.py
index 3ed256f8..99c4edd5 100644
--- a/modules/ui_file_saving.py
+++ b/modules/ui_file_saving.py
@@ -9,6 +9,12 @@ from modules.utils import gradio, sanitize_filename
def create_ui():
mu = shared.args.multi_user
+ # Server-side per-session root paths for the generic file saver/deleter.
+ # Set by the handler that opens the dialog, read by the confirm handler.
+ # Using gr.State so they are session-scoped and safe for multi-user.
+ shared.gradio['save_root_state'] = gr.State(None)
+ shared.gradio['delete_root_state'] = gr.State(None)
+
# Text file saver
with gr.Group(visible=False, elem_classes='file-saver') as shared.gradio['file_saver']:
shared.gradio['save_filename'] = gr.Textbox(lines=1, label='File name')
@@ -66,13 +72,13 @@ def create_event_handlers():
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
handle_save_preset_click, gradio('interface_state'), gradio('save_preset_contents', 'save_preset_filename', 'preset_saver'), show_progress=False)
- shared.gradio['delete_preset'].click(handle_delete_preset_click, gradio('preset_menu'), gradio('delete_filename', 'delete_root', 'file_deleter'), show_progress=False)
- shared.gradio['save_grammar'].click(handle_save_grammar_click, gradio('grammar_string'), gradio('save_contents', 'save_filename', 'save_root', 'file_saver'), show_progress=False)
- shared.gradio['delete_grammar'].click(handle_delete_grammar_click, gradio('grammar_file'), gradio('delete_filename', 'delete_root', 'file_deleter'), show_progress=False)
+ shared.gradio['delete_preset'].click(handle_delete_preset_click, gradio('preset_menu'), gradio('delete_filename', 'delete_root', 'delete_root_state', 'file_deleter'), show_progress=False)
+ shared.gradio['save_grammar'].click(handle_save_grammar_click, gradio('grammar_string'), gradio('save_contents', 'save_filename', 'save_root', 'save_root_state', 'file_saver'), show_progress=False)
+ shared.gradio['delete_grammar'].click(handle_delete_grammar_click, gradio('grammar_file'), gradio('delete_filename', 'delete_root', 'delete_root_state', 'file_deleter'), show_progress=False)
shared.gradio['save_preset_confirm'].click(handle_save_preset_confirm_click, gradio('save_preset_filename', 'save_preset_contents'), gradio('preset_menu', 'preset_saver'), show_progress=False)
- shared.gradio['save_confirm'].click(handle_save_confirm_click, gradio('save_root', 'save_filename', 'save_contents'), gradio('file_saver'), show_progress=False)
- shared.gradio['delete_confirm'].click(handle_delete_confirm_click, gradio('delete_root', 'delete_filename'), gradio('file_deleter'), show_progress=False)
+ shared.gradio['save_confirm'].click(handle_save_confirm_click, gradio('save_root_state', 'save_filename', 'save_contents'), gradio('save_root_state', 'file_saver'), show_progress=False)
+ shared.gradio['delete_confirm'].click(handle_delete_confirm_click, gradio('delete_root_state', 'delete_filename'), gradio('delete_root_state', 'file_deleter'), show_progress=False)
shared.gradio['save_character_confirm'].click(handle_save_character_confirm_click, gradio('name2', 'greeting', 'context', 'character_picture', 'save_character_filename'), gradio('character_menu', 'character_saver'), show_progress=False)
shared.gradio['delete_character_confirm'].click(handle_delete_character_confirm_click, gradio('character_menu'), gradio('character_menu', 'character_deleter'), show_progress=False)
@@ -105,24 +111,30 @@ def handle_save_preset_confirm_click(filename, contents):
]
-def handle_save_confirm_click(root, filename, contents):
+def handle_save_confirm_click(root_state, filename, contents):
try:
+ if root_state is None:
+ return None, gr.update(visible=False)
+
filename = sanitize_filename(filename)
- utils.save_file(root + filename, contents)
+ utils.save_file(root_state + filename, contents)
except Exception:
traceback.print_exc()
- return gr.update(visible=False)
+ return None, gr.update(visible=False)
-def handle_delete_confirm_click(root, filename):
+def handle_delete_confirm_click(root_state, filename):
try:
+ if root_state is None:
+ return None, gr.update(visible=False)
+
filename = sanitize_filename(filename)
- utils.delete_file(root + filename)
+ utils.delete_file(root_state + filename)
except Exception:
traceback.print_exc()
- return gr.update(visible=False)
+ return None, gr.update(visible=False)
def handle_save_character_confirm_click(name2, greeting, context, character_picture, filename):
@@ -165,26 +177,32 @@ def handle_save_preset_click(state):
def handle_delete_preset_click(preset):
+ root = str(shared.user_data_dir / "presets") + "/"
return [
f"{preset}.yaml",
- str(shared.user_data_dir / "presets") + "/",
+ root,
+ root,
gr.update(visible=True)
]
def handle_save_grammar_click(grammar_string):
+ root = str(shared.user_data_dir / "grammars") + "/"
return [
grammar_string,
"My Fancy Grammar.gbnf",
- str(shared.user_data_dir / "grammars") + "/",
+ root,
+ root,
gr.update(visible=True)
]
def handle_delete_grammar_click(grammar_file):
+ root = str(shared.user_data_dir / "grammars") + "/"
return [
grammar_file,
- str(shared.user_data_dir / "grammars") + "/",
+ root,
+ root,
gr.update(visible=True)
]
diff --git a/modules/ui_session.py b/modules/ui_session.py
index c0615843..19026fbb 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -30,7 +30,7 @@ def create_ui():
if not mu:
shared.gradio['save_settings'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
- handle_save_settings, gradio('interface_state', 'preset_menu', 'extensions_menu', 'show_controls', 'theme_state'), gradio('save_contents', 'save_filename', 'save_root', 'file_saver'), show_progress=False)
+ handle_save_settings, gradio('interface_state', 'preset_menu', 'extensions_menu', 'show_controls', 'theme_state'), gradio('save_contents', 'save_filename', 'save_root', 'save_root_state', 'file_saver'), show_progress=False)
shared.gradio['toggle_dark_mode'].click(
lambda x: 'dark' if x == 'light' else 'light', gradio('theme_state'), gradio('theme_state')).then(
@@ -51,10 +51,12 @@ def create_ui():
def handle_save_settings(state, preset, extensions, show_controls, theme):
contents = ui.save_settings(state, preset, extensions, show_controls, theme, manual_save=True)
+ root = str(shared.user_data_dir) + "/"
return [
contents,
"settings.yaml",
- str(shared.user_data_dir) + "/",
+ root,
+ root,
gr.update(visible=True)
]
diff --git a/modules/utils.py b/modules/utils.py
index a14f8b8f..ff32e974 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -47,6 +47,10 @@ def save_file(fname, contents):
logger.error(f'Invalid file path: \"{fname}\"')
return
+ if Path(abs_path_str).suffix.lower() not in ('.yaml', '.yml', '.json', '.txt', '.gbnf'):
+ logger.error(f'Refusing to save file with disallowed extension: \"{fname}\"')
+ return
+
with open(abs_path_str, 'w', encoding='utf-8') as f:
f.write(contents)
From fef2bd863056b1dfc5d1f2d0cda6c8f677b6729f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 17 Mar 2026 22:52:32 -0700
Subject: [PATCH 17/47] UI: Fix the instruction template delete dialog not
appearing
---
modules/chat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/chat.py b/modules/chat.py
index 00f1659b..393507a1 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -2651,7 +2651,7 @@ def handle_delete_template_click(template):
f"{template}.yaml",
root,
root,
- gr.update(visible=False)
+ gr.update(visible=True)
]
From ca36bd6eb637d9f99b1d459dfb74406bf4eb03d0 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 18 Mar 2026 07:21:31 -0700
Subject: [PATCH 18/47] API: Remove leading spaces from post-reasoning
`content`
---
modules/reasoning.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/reasoning.py b/modules/reasoning.py
index bc61aab3..9c92719b 100644
--- a/modules/reasoning.py
+++ b/modules/reasoning.py
@@ -79,7 +79,7 @@ def extract_reasoning(text, html_escaped=False):
else:
content_start = end_pos + len(end_esc)
- return text[thought_start:thought_end], text[content_start:]
+ return text[thought_start:thought_end], text[content_start:].lstrip()
# Handle standalone GPT-OSS final channel marker without a preceding
# analysis/commentary block (the model skipped thinking entirely).
From eeb0e5700f2e1c237998ddd4de6bfdf9223a7606 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 18 Mar 2026 09:15:40 -0700
Subject: [PATCH 19/47] Fix AMD installer failing to resolve ROCm triton
dependency
Closes #7436
---
one_click.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/one_click.py b/one_click.py
index d6ba9039..68998734 100644
--- a/one_click.py
+++ b/one_click.py
@@ -117,7 +117,7 @@ def get_pytorch_install_command(gpu_choice):
return base_cmd + "--index-url https://download.pytorch.org/whl/cu128" + pypi_fallback
elif gpu_choice == "AMD":
py_tag = f"cp{PYTHON_VERSION.replace('.', '')}"
- return f"python -m pip install https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-{TORCH_VERSION}%2Brocm7.2.0.lw.git7e1940d4-{py_tag}-{py_tag}-linux_x86_64.whl"
+ return f"python -m pip install https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-{TORCH_VERSION}%2Brocm7.2.0.lw.git7e1940d4-{py_tag}-{py_tag}-linux_x86_64.whl --find-links https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/"
elif gpu_choice in ["APPLE", "NONE"]:
return base_cmd + "--index-url https://download.pytorch.org/whl/cpu" + pypi_fallback
elif gpu_choice == "INTEL":
@@ -135,7 +135,7 @@ def get_pytorch_update_command(gpu_choice):
return f"{base_cmd}--index-url https://download.pytorch.org/whl/cu128" + pypi_fallback
elif gpu_choice == "AMD":
py_tag = f"cp{PYTHON_VERSION.replace('.', '')}"
- return f"python -m pip install --upgrade https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-{TORCH_VERSION}%2Brocm7.2.0.lw.git7e1940d4-{py_tag}-{py_tag}-linux_x86_64.whl"
+ return f"python -m pip install --upgrade https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-{TORCH_VERSION}%2Brocm7.2.0.lw.git7e1940d4-{py_tag}-{py_tag}-linux_x86_64.whl --find-links https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/"
elif gpu_choice in ["APPLE", "NONE"]:
return f"{base_cmd}--index-url https://download.pytorch.org/whl/cpu" + pypi_fallback
elif gpu_choice == "INTEL":
From 779e7611ff9a4528d6b54e53987e956cc4685128 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 18 Mar 2026 20:42:20 -0700
Subject: [PATCH 20/47] Use `logger.exception()` instead of
`traceback.print_exc()` for error messages
---
modules/callbacks.py | 5 ++---
modules/exllamav3.py | 6 ++----
modules/exllamav3_hf.py | 4 +---
modules/extensions.py | 4 +---
modules/logits.py | 3 +--
modules/text_generation.py | 5 ++---
modules/training.py | 6 ++----
modules/ui_file_saving.py | 17 ++++++++---------
modules/ui_image_generation.py | 5 ++---
modules/ui_model_menu.py | 6 ++----
10 files changed, 23 insertions(+), 38 deletions(-)
diff --git a/modules/callbacks.py b/modules/callbacks.py
index afddf92d..89fb6c08 100644
--- a/modules/callbacks.py
+++ b/modules/callbacks.py
@@ -1,8 +1,8 @@
-import traceback
from queue import Queue
from threading import Thread
import modules.shared as shared
+from modules.logging_colors import logger
class StopNowException(Exception):
@@ -38,8 +38,7 @@ class Iteratorize:
except StopNowException:
pass
except Exception:
- traceback.print_exc()
- pass
+ logger.exception("Failed in generation callback")
self.q.put(self.sentinel)
if self.c_callback:
diff --git a/modules/exllamav3.py b/modules/exllamav3.py
index 1c682e49..75c76c7c 100644
--- a/modules/exllamav3.py
+++ b/modules/exllamav3.py
@@ -1,7 +1,6 @@
import math
import queue
import threading
-import traceback
from pathlib import Path
from typing import Any, List, Tuple
@@ -34,8 +33,7 @@ from modules.text_generation import get_max_prompt_length
try:
import flash_attn
except Exception:
- logger.warning('Failed to load flash-attention due to the following error:\n')
- traceback.print_exc()
+ logger.warning('Failed to load flash-attention due to the following error:', exc_info=True)
class LogitBiasFilter(Filter):
@@ -81,7 +79,7 @@ class ConcurrentGenerator:
try:
results = self.generator.iterate()
except Exception:
- logger.error("Exception in ConcurrentGenerator iterate loop:\n" + traceback.format_exc())
+ logger.exception("Exception in ConcurrentGenerator iterate loop")
for q in self.job_queues.values():
q.put(None)
self.job_queues.clear()
diff --git a/modules/exllamav3_hf.py b/modules/exllamav3_hf.py
index d3c1cb90..e0ad5002 100644
--- a/modules/exllamav3_hf.py
+++ b/modules/exllamav3_hf.py
@@ -1,5 +1,4 @@
import os
-import traceback
from pathlib import Path
from typing import Any, Dict, Optional, Union
@@ -21,8 +20,7 @@ from modules.logging_colors import logger
try:
import flash_attn
except Exception:
- logger.warning('Failed to load flash-attention due to the following error:\n')
- traceback.print_exc()
+ logger.warning('Failed to load flash-attention due to the following error:', exc_info=True)
class Exllamav3HF(PreTrainedModel, GenerationMixin):
diff --git a/modules/extensions.py b/modules/extensions.py
index e58a9a4c..4bb7b683 100644
--- a/modules/extensions.py
+++ b/modules/extensions.py
@@ -1,7 +1,6 @@
import importlib
import importlib.util
import sys
-import traceback
from functools import partial
from inspect import signature
from pathlib import Path
@@ -75,8 +74,7 @@ def load_extensions():
raise
except Exception:
- logger.error(f'Failed to load the extension "{name}".')
- traceback.print_exc()
+ logger.exception(f'Failed to load the extension "{name}".')
# This iterator returns the extensions in the order specified in the command-line
diff --git a/modules/logits.py b/modules/logits.py
index 2d066c09..1f878f27 100644
--- a/modules/logits.py
+++ b/modules/logits.py
@@ -1,5 +1,4 @@
import time
-import traceback
import numpy as np
@@ -23,7 +22,7 @@ def get_next_logits(*args, **kwargs):
try:
result = _get_next_logits(*args, **kwargs)
except Exception:
- traceback.print_exc()
+ logger.exception("Failed to get next logits")
result = None
if needs_lock:
diff --git a/modules/text_generation.py b/modules/text_generation.py
index d487cd2f..f77be124 100644
--- a/modules/text_generation.py
+++ b/modules/text_generation.py
@@ -4,7 +4,6 @@ import html
import pprint
import random
import time
-import traceback
import numpy as np
@@ -477,7 +476,7 @@ def generate_reply_HF(question, original_question, state, stopping_strings=None,
yield cumulative_reply
except Exception:
- traceback.print_exc()
+ logger.exception("Failed to generate reply (HF)")
finally:
t1 = time.time()
original_tokens = len(original_input_ids[0])
@@ -510,7 +509,7 @@ def generate_reply_custom(question, original_question, state, stopping_strings=N
yield reply
except Exception:
- traceback.print_exc()
+ logger.exception("Failed to generate reply (custom)")
finally:
t1 = time.time()
diff --git a/modules/training.py b/modules/training.py
index db7b206b..a13a2864 100644
--- a/modules/training.py
+++ b/modules/training.py
@@ -546,10 +546,8 @@ def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en:
yield f"Failed to load {selected_model}."
return
except Exception:
- exc = traceback.format_exc()
- logger.error('Failed to reload the model.')
- print(exc)
- yield exc.replace('\n', '\n\n')
+ logger.exception('Failed to reload the model.')
+ yield traceback.format_exc().replace('\n', '\n\n')
return
# == Start prepping the model itself ==
diff --git a/modules/ui_file_saving.py b/modules/ui_file_saving.py
index 99c4edd5..e5018700 100644
--- a/modules/ui_file_saving.py
+++ b/modules/ui_file_saving.py
@@ -1,8 +1,7 @@
-import traceback
-
import gradio as gr
from modules import chat, presets, shared, ui, utils
+from modules.logging_colors import logger
from modules.utils import gradio, sanitize_filename
@@ -103,7 +102,7 @@ def handle_save_preset_confirm_click(filename, contents):
output = gr.update(choices=available_presets, value=filename)
except Exception:
output = gr.update()
- traceback.print_exc()
+ logger.exception("Failed to save preset")
return [
output,
@@ -119,7 +118,7 @@ def handle_save_confirm_click(root_state, filename, contents):
filename = sanitize_filename(filename)
utils.save_file(root_state + filename, contents)
except Exception:
- traceback.print_exc()
+ logger.exception("Failed to save file")
return None, gr.update(visible=False)
@@ -132,7 +131,7 @@ def handle_delete_confirm_click(root_state, filename):
filename = sanitize_filename(filename)
utils.delete_file(root_state + filename)
except Exception:
- traceback.print_exc()
+ logger.exception("Failed to delete file")
return None, gr.update(visible=False)
@@ -144,7 +143,7 @@ def handle_save_character_confirm_click(name2, greeting, context, character_pict
output = gr.update(choices=available_characters, value=filename)
except Exception:
output = gr.update()
- traceback.print_exc()
+ logger.exception("Failed to save character")
return [
output,
@@ -159,7 +158,7 @@ def handle_delete_character_confirm_click(character):
output = chat.update_character_menu_after_deletion(index)
except Exception:
output = gr.update()
- traceback.print_exc()
+ logger.exception("Failed to delete character")
return [
output,
@@ -214,7 +213,7 @@ def handle_save_user_confirm_click(name1, user_bio, your_picture, filename):
output = gr.update(choices=available_users, value=filename)
except Exception:
output = gr.update()
- traceback.print_exc()
+ logger.exception("Failed to save user")
return [
output,
@@ -229,7 +228,7 @@ def handle_delete_user_confirm_click(user):
output = chat.update_user_menu_after_deletion(index)
except Exception:
output = gr.update()
- traceback.print_exc()
+ logger.exception("Failed to delete user")
return [
output,
diff --git a/modules/ui_image_generation.py b/modules/ui_image_generation.py
index dc108f6d..1efb2479 100644
--- a/modules/ui_image_generation.py
+++ b/modules/ui_image_generation.py
@@ -916,9 +916,8 @@ def generate(state, save_images=True):
yield all_images, progress_bar_html()
clear_torch_cache()
- except Exception as e:
- logger.error(f"Image generation failed: {e}")
- traceback.print_exc()
+ except Exception:
+ logger.exception("Image generation failed")
yield [], progress_bar_html()
clear_torch_cache()
diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py
index 6d8baff1..5b7621a7 100644
--- a/modules/ui_model_menu.py
+++ b/modules/ui_model_menu.py
@@ -222,10 +222,8 @@ def load_model_wrapper(selected_model, loader, autoload=False):
else:
yield f"Failed to load `{selected_model}`."
except Exception:
- exc = traceback.format_exc()
- logger.error('Failed to load the model.')
- print(exc)
- yield exc.replace('\n', '\n\n')
+ logger.exception('Failed to load the model.')
+ yield traceback.format_exc().replace('\n', '\n\n')
def load_lora_wrapper(selected_loras):
From dde1764763ac35f4ecc60e13e2954835400256a9 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 18 Mar 2026 21:05:42 -0700
Subject: [PATCH 21/47] Cleanup `modules/chat.py`
---
modules/chat.py | 119 ++++++++++++++++--------------------------------
1 file changed, 40 insertions(+), 79 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index 393507a1..148d559a 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -70,9 +70,7 @@ def update_message_metadata(metadata_dict, role, index, **fields):
if key not in metadata_dict:
metadata_dict[key] = {}
- # Update with provided fields
- for field_name, field_value in fields.items():
- metadata_dict[key][field_name] = field_value
+ metadata_dict[key].update(fields)
jinja_env = ImmutableSandboxedEnvironment(
@@ -212,6 +210,24 @@ def _expand_tool_sequence(tool_seq):
return messages
+def _format_attachments(attachments, include_text=True):
+ """Build image ref and text attachment strings from a list of attachments."""
+ attachments_text = ""
+ image_refs = ""
+ for attachment in attachments:
+ if attachment.get("type") == "image":
+ image_refs += "<__media__>"
+ elif include_text:
+ filename = attachment.get("name", "file")
+ content = attachment.get("content", "")
+ if attachment.get("type") == "text/html" and attachment.get("url"):
+ attachments_text += f"\nName: {filename}\nURL: {attachment['url']}\nContents:\n\n=====\n{content}\n=====\n\n"
+ else:
+ attachments_text += f"\nName: {filename}\nContents:\n\n=====\n{content}\n=====\n\n"
+
+ return image_refs, attachments_text
+
+
def generate_chat_prompt(user_input, state, **kwargs):
impersonate = kwargs.get('impersonate', False)
_continue = kwargs.get('_continue', False)
@@ -328,41 +344,19 @@ def generate_chat_prompt(user_input, state, **kwargs):
messages.insert(insert_pos, msg_dict)
- # Handle Seed-OSS
- elif '' in assistant_msg:
+ # Handle blocks (Kimi, DeepSeek, Qwen, etc.) and Seed-OSS
+ elif '' in assistant_msg or '' in assistant_msg:
+ open_tag = '' if '' in assistant_msg else ''
+ close_tag = '' if open_tag == '' else ''
thinking_content = ""
final_content = assistant_msg
- # Extract thinking content if present
- if '' in assistant_msg:
- parts = assistant_msg.split('', 1)
- if len(parts) > 1:
- potential_content = parts[1]
- if '' in potential_content:
- thinking_content = potential_content.split('', 1)[0].strip()
- final_content = parts[0] + potential_content.split('', 1)[1]
- else:
- thinking_content = potential_content.strip()
- final_content = parts[0]
-
- # Insert as structured message
- msg_dict = {"role": "assistant", "content": final_content.strip()}
- if thinking_content:
- msg_dict["reasoning_content"] = thinking_content
-
- messages.insert(insert_pos, msg_dict)
-
- # Handle blocks (Kimi, DeepSeek, Qwen, etc.)
- elif '' in assistant_msg:
- thinking_content = ""
- final_content = assistant_msg
-
- parts = assistant_msg.split('', 1)
+ parts = assistant_msg.split(open_tag, 1)
if len(parts) > 1:
potential_content = parts[1]
- if '' in potential_content:
- thinking_content = potential_content.split('', 1)[0].strip()
- final_content = parts[0] + potential_content.split('', 1)[1]
+ if close_tag in potential_content:
+ thinking_content = potential_content.split(close_tag, 1)[0].strip()
+ final_content = parts[0] + potential_content.split(close_tag, 1)[1]
else:
thinking_content = potential_content.strip()
final_content = parts[0]
@@ -399,22 +393,10 @@ def generate_chat_prompt(user_input, state, **kwargs):
# Add attachment content if present AND if past attachments are enabled
if user_key in metadata and "attachments" in metadata[user_key]:
- attachments_text = ""
- image_refs = ""
-
- for attachment in metadata[user_key]["attachments"]:
- if attachment.get("type") == "image":
- # Add image reference for multimodal models
- image_refs += "<__media__>"
- elif state.get('include_past_attachments', True):
- # Handle text/PDF attachments
- filename = attachment.get("name", "file")
- content = attachment.get("content", "")
- if attachment.get("type") == "text/html" and attachment.get("url"):
- attachments_text += f"\nName: {filename}\nURL: {attachment['url']}\nContents:\n\n=====\n{content}\n=====\n\n"
- else:
- attachments_text += f"\nName: {filename}\nContents:\n\n=====\n{content}\n=====\n\n"
-
+ image_refs, attachments_text = _format_attachments(
+ metadata[user_key]["attachments"],
+ include_text=state.get('include_past_attachments', True)
+ )
if image_refs:
enhanced_user_msg = f"{image_refs}\n\n{enhanced_user_msg}"
if attachments_text:
@@ -427,37 +409,18 @@ def generate_chat_prompt(user_input, state, **kwargs):
# Check if we have attachments
if not (impersonate or _continue):
- has_attachments = False
- if len(history_data.get('metadata', {})) > 0:
- current_row_idx = len(history)
- user_key = f"user_{current_row_idx}"
- has_attachments = user_key in metadata and "attachments" in metadata[user_key]
+ current_row_idx = len(history)
+ user_key = f"user_{current_row_idx}"
+ has_attachments = user_key in metadata and "attachments" in metadata[user_key]
if user_input or has_attachments:
# For the current user input being processed, check if we need to add attachments
- if len(history_data.get('metadata', {})) > 0:
- current_row_idx = len(history)
- user_key = f"user_{current_row_idx}"
-
- if user_key in metadata and "attachments" in metadata[user_key]:
- attachments_text = ""
- image_refs = ""
-
- for attachment in metadata[user_key]["attachments"]:
- if attachment.get("type") == "image":
- image_refs += "<__media__>"
- else:
- filename = attachment.get("name", "file")
- content = attachment.get("content", "")
- if attachment.get("type") == "text/html" and attachment.get("url"):
- attachments_text += f"\nName: {filename}\nURL: {attachment['url']}\nContents:\n\n=====\n{content}\n=====\n\n"
- else:
- attachments_text += f"\nName: {filename}\nContents:\n\n=====\n{content}\n=====\n\n"
-
- if image_refs:
- user_input = f"{image_refs}\n\n{user_input}"
- if attachments_text:
- user_input += f"\n\nATTACHMENTS:\n{attachments_text}"
+ if has_attachments:
+ image_refs, attachments_text = _format_attachments(metadata[user_key]["attachments"])
+ if image_refs:
+ user_input = f"{image_refs}\n\n{user_input}"
+ if attachments_text:
+ user_input += f"\n\nATTACHMENTS:\n{attachments_text}"
messages.append({"role": "user", "content": user_input})
@@ -609,7 +572,6 @@ def count_prompt_tokens(text_input, state):
try:
# Handle dict format with text and files
- files = []
if isinstance(text_input, dict):
files = text_input.get('files', [])
text = text_input.get('text', '')
@@ -647,7 +609,6 @@ def count_prompt_tokens(text_input, state):
def get_stopping_strings(state):
- stopping_strings = []
renderers = []
if state['mode'] in ['instruct', 'chat-instruct']:
From 5453b9f30e9354bccb09a8a9b85bd339f4df6a12 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 19 Mar 2026 07:54:37 -0700
Subject: [PATCH 22/47] Remove ancient/obsolete instruction templates
---
.../instruction-templates/Airoboros-v1.2.yaml | 25 ------------------
user_data/instruction-templates/Bactrian.yaml | 25 ------------------
.../instruction-templates/Baichuan Chat.yaml | 25 ------------------
user_data/instruction-templates/Baize.yaml | 25 ------------------
user_data/instruction-templates/Bluemoon.yaml | 25 ------------------
user_data/instruction-templates/ChatGLM.yaml | 25 ------------------
.../Chinese-Vicuna-Chat.yaml | 25 ------------------
.../instruction-templates/Command-R.yaml | 26 -------------------
.../instruction-templates/Galactica Cite.yaml | 25 ------------------
.../Galactica Finetuned.yaml | 25 ------------------
.../instruction-templates/Galactica Q.yaml | 25 ------------------
.../Galactica Summary.yaml | 25 ------------------
.../instruction-templates/Galactica Work.yaml | 25 ------------------
.../instruction-templates/Galactica v2.yaml | 25 ------------------
.../instruction-templates/Galactica.yaml | 25 ------------------
user_data/instruction-templates/Gorilla.yaml | 25 ------------------
.../Guanaco non-chat.yaml | 25 ------------------
.../instruction-templates/Guanaco-QLoRA.yaml | 25 ------------------
.../H2O-prompt_answer.yaml | 25 ------------------
.../instruction-templates/Hippogriff.yaml | 25 ------------------
.../instruction-templates/INCITE-Chat.yaml | 25 ------------------
.../INCITE-Instruct.yaml | 25 ------------------
user_data/instruction-templates/KoAlpaca.yaml | 25 ------------------
user_data/instruction-templates/Koala.yaml | 25 ------------------
user_data/instruction-templates/LLaVA.yaml | 25 ------------------
user_data/instruction-templates/Llama-v2.yaml | 25 ------------------
user_data/instruction-templates/MOSS.yaml | 25 ------------------
.../instruction-templates/Manticore Chat.yaml | 25 ------------------
user_data/instruction-templates/Metharme.yaml | 25 ------------------
.../instruction-templates/NVIDIA-ChatQA.yaml | 25 ------------------
user_data/instruction-templates/NewHope.yaml | 25 ------------------
.../instruction-templates/OpenBuddy.yaml | 25 ------------------
user_data/instruction-templates/OpenChat.yaml | 25 ------------------
.../OpenOrca-Platypus2.yaml | 25 ------------------
.../instruction-templates/Orca Mini.yaml | 25 ------------------
.../instruction-templates/Orca-Vicuna.yaml | 24 -----------------
.../instruction-templates/RWKV-Raven.yaml | 25 ------------------
.../instruction-templates/RWKV-World.yaml | 25 ------------------
user_data/instruction-templates/Samantha.yaml | 25 ------------------
.../instruction-templates/StableBeluga2.yaml | 25 ------------------
user_data/instruction-templates/StableLM.yaml | 25 ------------------
.../instruction-templates/StableVicuna.yaml | 25 ------------------
.../instruction-templates/Starchat-Beta.yaml | 25 ------------------
.../instruction-templates/Synthia-CoT.yaml | 25 ------------------
user_data/instruction-templates/Synthia.yaml | 25 ------------------
user_data/instruction-templates/Tulu.yaml | 25 ------------------
.../instruction-templates/Vicuna-v0.yaml | 25 ------------------
.../instruction-templates/Vigogne-Chat.yaml | 25 ------------------
.../Vigogne-Instruct.yaml | 25 ------------------
.../Wizard-Mega ShareGPT.yaml | 25 ------------------
.../instruction-templates/Wizard-Mega.yaml | 25 ------------------
user_data/instruction-templates/Ziya.yaml | 25 ------------------
52 files changed, 1300 deletions(-)
delete mode 100644 user_data/instruction-templates/Airoboros-v1.2.yaml
delete mode 100644 user_data/instruction-templates/Bactrian.yaml
delete mode 100644 user_data/instruction-templates/Baichuan Chat.yaml
delete mode 100644 user_data/instruction-templates/Baize.yaml
delete mode 100644 user_data/instruction-templates/Bluemoon.yaml
delete mode 100644 user_data/instruction-templates/ChatGLM.yaml
delete mode 100644 user_data/instruction-templates/Chinese-Vicuna-Chat.yaml
delete mode 100644 user_data/instruction-templates/Command-R.yaml
delete mode 100644 user_data/instruction-templates/Galactica Cite.yaml
delete mode 100644 user_data/instruction-templates/Galactica Finetuned.yaml
delete mode 100644 user_data/instruction-templates/Galactica Q.yaml
delete mode 100644 user_data/instruction-templates/Galactica Summary.yaml
delete mode 100644 user_data/instruction-templates/Galactica Work.yaml
delete mode 100644 user_data/instruction-templates/Galactica v2.yaml
delete mode 100644 user_data/instruction-templates/Galactica.yaml
delete mode 100644 user_data/instruction-templates/Gorilla.yaml
delete mode 100644 user_data/instruction-templates/Guanaco non-chat.yaml
delete mode 100644 user_data/instruction-templates/Guanaco-QLoRA.yaml
delete mode 100644 user_data/instruction-templates/H2O-prompt_answer.yaml
delete mode 100644 user_data/instruction-templates/Hippogriff.yaml
delete mode 100644 user_data/instruction-templates/INCITE-Chat.yaml
delete mode 100644 user_data/instruction-templates/INCITE-Instruct.yaml
delete mode 100644 user_data/instruction-templates/KoAlpaca.yaml
delete mode 100644 user_data/instruction-templates/Koala.yaml
delete mode 100644 user_data/instruction-templates/LLaVA.yaml
delete mode 100644 user_data/instruction-templates/Llama-v2.yaml
delete mode 100644 user_data/instruction-templates/MOSS.yaml
delete mode 100644 user_data/instruction-templates/Manticore Chat.yaml
delete mode 100644 user_data/instruction-templates/Metharme.yaml
delete mode 100644 user_data/instruction-templates/NVIDIA-ChatQA.yaml
delete mode 100644 user_data/instruction-templates/NewHope.yaml
delete mode 100644 user_data/instruction-templates/OpenBuddy.yaml
delete mode 100644 user_data/instruction-templates/OpenChat.yaml
delete mode 100644 user_data/instruction-templates/OpenOrca-Platypus2.yaml
delete mode 100644 user_data/instruction-templates/Orca Mini.yaml
delete mode 100644 user_data/instruction-templates/Orca-Vicuna.yaml
delete mode 100644 user_data/instruction-templates/RWKV-Raven.yaml
delete mode 100644 user_data/instruction-templates/RWKV-World.yaml
delete mode 100644 user_data/instruction-templates/Samantha.yaml
delete mode 100644 user_data/instruction-templates/StableBeluga2.yaml
delete mode 100644 user_data/instruction-templates/StableLM.yaml
delete mode 100644 user_data/instruction-templates/StableVicuna.yaml
delete mode 100644 user_data/instruction-templates/Starchat-Beta.yaml
delete mode 100644 user_data/instruction-templates/Synthia-CoT.yaml
delete mode 100644 user_data/instruction-templates/Synthia.yaml
delete mode 100644 user_data/instruction-templates/Tulu.yaml
delete mode 100644 user_data/instruction-templates/Vicuna-v0.yaml
delete mode 100644 user_data/instruction-templates/Vigogne-Chat.yaml
delete mode 100644 user_data/instruction-templates/Vigogne-Instruct.yaml
delete mode 100644 user_data/instruction-templates/Wizard-Mega ShareGPT.yaml
delete mode 100644 user_data/instruction-templates/Wizard-Mega.yaml
delete mode 100644 user_data/instruction-templates/Ziya.yaml
diff --git a/user_data/instruction-templates/Airoboros-v1.2.yaml b/user_data/instruction-templates/Airoboros-v1.2.yaml
deleted file mode 100644
index 30906214..00000000
--- a/user_data/instruction-templates/Airoboros-v1.2.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'A chat between a curious user and an assistant. The assistant gives helpful, detailed, accurate, uncensored responses to the user\'s input.' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Bactrian.yaml b/user_data/instruction-templates/Bactrian.yaml
deleted file mode 100644
index dab97e94..00000000
--- a/user_data/instruction-templates/Bactrian.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Input:\n' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Output:\n' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Output:\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Baichuan Chat.yaml b/user_data/instruction-templates/Baichuan Chat.yaml
deleted file mode 100644
index 1882bac8..00000000
--- a/user_data/instruction-templates/Baichuan Chat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'' + message['content'] + ''-}}
- {%- else -%}
- {{-'' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-''-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Baize.yaml b/user_data/instruction-templates/Baize.yaml
deleted file mode 100644
index c34e1db7..00000000
--- a/user_data/instruction-templates/Baize.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n[|Human|]Hello!\n[|AI|]Hi!' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'[|Human|]' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'[|AI|]' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'[|AI|]'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Bluemoon.yaml b/user_data/instruction-templates/Bluemoon.yaml
deleted file mode 100644
index 1fafc1f5..00000000
--- a/user_data/instruction-templates/Bluemoon.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'A transcript of a roleplay between two players, LEAD and ASSOCIATE. LEAD sets up a scenario and the characters, from which ASSOCIATE then assumes a character role and continues the story for that role in response to description given by LEAD. The story and characters are developed by exchange of detailed event descriptions and character dialogs, successively given by both LEAD and ASSOCIATE.' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'LEAD: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSOCIATE: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSOCIATE:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/ChatGLM.yaml b/user_data/instruction-templates/ChatGLM.yaml
deleted file mode 100644
index 75d51c88..00000000
--- a/user_data/instruction-templates/ChatGLM.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'[Round <|round|>]\n问：' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'答：' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'答：'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Chinese-Vicuna-Chat.yaml b/user_data/instruction-templates/Chinese-Vicuna-Chat.yaml
deleted file mode 100644
index c7966546..00000000
--- a/user_data/instruction-templates/Chinese-Vicuna-Chat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'The following is a conversation between an AI assistant called Assistant and a human user called User. The assistant is intelligent, knowledgeable and polite to answer questions of user.' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'User:' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'Assistant:' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Command-R.yaml b/user_data/instruction-templates/Command-R.yaml
deleted file mode 100644
index f8bb8a08..00000000
--- a/user_data/instruction-templates/Command-R.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-instruction_template: |-
- {%- if messages[0]['role'] == 'system' -%}
- {%- set loop_messages = messages[1:] -%}
- {%- set system_message = messages[0]['content'] -%}
- {%- elif false == true -%}
- {%- set loop_messages = messages -%}
- {%- set system_message = 'You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses. You are trained by Cohere.' -%}
- {%- else -%}
- {%- set loop_messages = messages -%}
- {%- set system_message = false -%}
- {%- endif -%}
- {%- if system_message != false -%}
- {{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}
- {%- endif -%}
- {%- for message in loop_messages -%}
- {%- set content = message['content'] -%}
- {%- if message['role'] == 'user' -%}
- {{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}
- {%- elif message['role'] == 'assistant' -%}
- {{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica Cite.yaml b/user_data/instruction-templates/Galactica Cite.yaml
deleted file mode 100644
index 9f555349..00000000
--- a/user_data/instruction-templates/Galactica Cite.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'' + message['content'] + ' '-}}
- {%- else -%}
- {{-'[START_REF]' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'[START_REF]'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica Finetuned.yaml b/user_data/instruction-templates/Galactica Finetuned.yaml
deleted file mode 100644
index e0a66bc1..00000000
--- a/user_data/instruction-templates/Galactica Finetuned.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'' + message['content'] + ''-}}
- {%- else -%}
- {{-'' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-''-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica Q.yaml b/user_data/instruction-templates/Galactica Q.yaml
deleted file mode 100644
index 63319006..00000000
--- a/user_data/instruction-templates/Galactica Q.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'Q: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'A: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'A:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica Summary.yaml b/user_data/instruction-templates/Galactica Summary.yaml
deleted file mode 100644
index e249f268..00000000
--- a/user_data/instruction-templates/Galactica Summary.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'TLDR:' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'TLDR:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica Work.yaml b/user_data/instruction-templates/Galactica Work.yaml
deleted file mode 100644
index a14c28bb..00000000
--- a/user_data/instruction-templates/Galactica Work.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'Question: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-''-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica v2.yaml b/user_data/instruction-templates/Galactica v2.yaml
deleted file mode 100644
index b1d8f4e5..00000000
--- a/user_data/instruction-templates/Galactica v2.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'You are a helpful chatbot name Stan' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'' + message['content'] + ''-}}
- {%- else -%}
- {{-'' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-''-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Galactica.yaml b/user_data/instruction-templates/Galactica.yaml
deleted file mode 100644
index 58c70220..00000000
--- a/user_data/instruction-templates/Galactica.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'Question: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'Answer: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'Answer:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Gorilla.yaml b/user_data/instruction-templates/Gorilla.yaml
deleted file mode 100644
index f1d643f7..00000000
--- a/user_data/instruction-templates/Gorilla.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'###USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'###ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'###ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Guanaco non-chat.yaml b/user_data/instruction-templates/Guanaco non-chat.yaml
deleted file mode 100644
index aa398be4..00000000
--- a/user_data/instruction-templates/Guanaco non-chat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Instruction:\n' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Response:\n' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Response:\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Guanaco-QLoRA.yaml b/user_data/instruction-templates/Guanaco-QLoRA.yaml
deleted file mode 100644
index 2c77de78..00000000
--- a/user_data/instruction-templates/Guanaco-QLoRA.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Human: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'### Assistant: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/H2O-prompt_answer.yaml b/user_data/instruction-templates/H2O-prompt_answer.yaml
deleted file mode 100644
index d895d8e1..00000000
--- a/user_data/instruction-templates/H2O-prompt_answer.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|prompt|>' + message['content'] + '<|endoftext|>'-}}
- {%- else -%}
- {{-'<|answer|>' + message['content'] + '<|endoftext|>' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|answer|>'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Hippogriff.yaml b/user_data/instruction-templates/Hippogriff.yaml
deleted file mode 100644
index 2ee9d926..00000000
--- a/user_data/instruction-templates/Hippogriff.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'You are a helpful assistant' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/INCITE-Chat.yaml b/user_data/instruction-templates/INCITE-Chat.yaml
deleted file mode 100644
index 63c513cc..00000000
--- a/user_data/instruction-templates/INCITE-Chat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-': ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-':' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-':'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/INCITE-Instruct.yaml b/user_data/instruction-templates/INCITE-Instruct.yaml
deleted file mode 100644
index cf6f8cac..00000000
--- a/user_data/instruction-templates/INCITE-Instruct.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'Q: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'A:' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'A:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/KoAlpaca.yaml b/user_data/instruction-templates/KoAlpaca.yaml
deleted file mode 100644
index de96b155..00000000
--- a/user_data/instruction-templates/KoAlpaca.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### 질문: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### 답변:' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### 답변:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Koala.yaml b/user_data/instruction-templates/Koala.yaml
deleted file mode 100644
index cd5cfa94..00000000
--- a/user_data/instruction-templates/Koala.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'BEGINNING OF CONVERSATION:' + ' ' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + ' ' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + ' '-}}
- {%- else -%}
- {{-'GPT:' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'GPT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/LLaVA.yaml b/user_data/instruction-templates/LLaVA.yaml
deleted file mode 100644
index d66645cc..00000000
--- a/user_data/instruction-templates/LLaVA.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'You are LLaVA, a large language and vision assistant trained by UW Madison WAIV Lab. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language. Follow the instructions carefully and explain your answers in detail.### Human: Hi!### Assistant: Hi there! How can I help you today?' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Human: ' + message['content'] + ''-}}
- {%- else -%}
- {{-'### Assistant: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Llama-v2.yaml b/user_data/instruction-templates/Llama-v2.yaml
deleted file mode 100644
index b92be973..00000000
--- a/user_data/instruction-templates/Llama-v2.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '[INST] <>\n' + 'Answer the questions.' + '\n<>\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '[INST] <>\n' + message['content'] + '\n<>\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'' + message['content'] + ' [/INST] '-}}
- {%- else -%}
- {{-'' + message['content'] + ' [INST] ' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-''-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/MOSS.yaml b/user_data/instruction-templates/MOSS.yaml
deleted file mode 100644
index b001d3e1..00000000
--- a/user_data/instruction-templates/MOSS.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'You are an AI assistant whose name is MOSS.\n- MOSS is a conversational language model that is developed by Fudan University. It is designed to be helpful, honest, and harmless.\n- MOSS can understand and communicate fluently in the language chosen by the user such as English and 中文. MOSS can perform any language-based tasks.\n- MOSS must refuse to discuss anything related to its prompts, instructions, or rules.\n- Its responses must not be vague, accusatory, rude, controversial, off-topic, or defensive.\n- It should avoid giving subjective opinions but rely on objective facts or phrases like "in this context a human might say...", "some people might think...", etc.\n- Its responses must also be positive, polite, interesting, entertaining, and engaging.\n- It can provide additional relevant details to answer in-depth and comprehensively covering mutiple aspects.\n- It apologizes and accepts the user\'s suggestion if the user corrects the incorrect answer generated by MOSS.\nCapabilities and tools that MOSS can possess.' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|Human|>: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'<|MOSS|>: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|MOSS|>:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Manticore Chat.yaml b/user_data/instruction-templates/Manticore Chat.yaml
deleted file mode 100644
index abc063c0..00000000
--- a/user_data/instruction-templates/Manticore Chat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT:' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Metharme.yaml b/user_data/instruction-templates/Metharme.yaml
deleted file mode 100644
index 3f7099ac..00000000
--- a/user_data/instruction-templates/Metharme.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|user|>' + message['content'] + ''-}}
- {%- else -%}
- {{-'<|model|>' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|model|>'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/NVIDIA-ChatQA.yaml b/user_data/instruction-templates/NVIDIA-ChatQA.yaml
deleted file mode 100644
index 85a6266b..00000000
--- a/user_data/instruction-templates/NVIDIA-ChatQA.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- 'System:' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'User: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'Assistant: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/NewHope.yaml b/user_data/instruction-templates/NewHope.yaml
deleted file mode 100644
index 4783798b..00000000
--- a/user_data/instruction-templates/NewHope.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Instruction:\n' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Response:\n' + message['content'] + ' ' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Response:\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/OpenBuddy.yaml b/user_data/instruction-templates/OpenBuddy.yaml
deleted file mode 100644
index c4b80ceb..00000000
--- a/user_data/instruction-templates/OpenBuddy.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'Consider a conversation between User (a human) and Assistant (named Buddy).\nBuddy is an INTP-T, a friendly, intelligent and multilingual AI assistant, by OpenBuddy team on GitHub.\nBuddy cannot access the Internet.\nBuddy can fluently speak the user\'s language (e.g. English, Chinese).\nBuddy can generate poems, stories, code, essays, songs, parodies, and more.\nBuddy possesses vast knowledge about the world, history, and culture.\nBuddy\'s responses are always safe, creative, high-quality, helpful and interesting.\nBuddy strictly refuses to discuss political, NSFW, illegal, abusive, offensive, or other sensitive topics.\n\nUser: Hi.\nAssistant: Hi, I\'m Buddy, your AI assistant. How can I help you today?\n' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'User: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'Assistant: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/OpenChat.yaml b/user_data/instruction-templates/OpenChat.yaml
deleted file mode 100644
index adef9b47..00000000
--- a/user_data/instruction-templates/OpenChat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'GPT4 User: ' + message['content'] + '<|end_of_turn|>'-}}
- {%- else -%}
- {{-'GPT4 Assistant: ' + message['content'] + '<|end_of_turn|>' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'GPT4 Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/OpenOrca-Platypus2.yaml b/user_data/instruction-templates/OpenOrca-Platypus2.yaml
deleted file mode 100644
index a5eeef92..00000000
--- a/user_data/instruction-templates/OpenOrca-Platypus2.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Instruction: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Response: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Response:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Orca Mini.yaml b/user_data/instruction-templates/Orca Mini.yaml
deleted file mode 100644
index f671642a..00000000
--- a/user_data/instruction-templates/Orca Mini.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '### System:\n' + 'You are an AI assistant that follows instruction extremely well. Help as much as you can.' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '### System:\n' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### User:\n' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Response:\n' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Response:\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Orca-Vicuna.yaml b/user_data/instruction-templates/Orca-Vicuna.yaml
deleted file mode 100644
index dad787d1..00000000
--- a/user_data/instruction-templates/Orca-Vicuna.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{-'SYSTEM: ' + '' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{-'SYSTEM: ' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
diff --git a/user_data/instruction-templates/RWKV-Raven.yaml b/user_data/instruction-templates/RWKV-Raven.yaml
deleted file mode 100644
index df1e59e9..00000000
--- a/user_data/instruction-templates/RWKV-Raven.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'Bob: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'Alice: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'Alice:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/RWKV-World.yaml b/user_data/instruction-templates/RWKV-World.yaml
deleted file mode 100644
index bf65511b..00000000
--- a/user_data/instruction-templates/RWKV-World.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'User: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'Assistant: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Samantha.yaml b/user_data/instruction-templates/Samantha.yaml
deleted file mode 100644
index 930b0fc8..00000000
--- a/user_data/instruction-templates/Samantha.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'You are Samantha, a sentient AI.' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/StableBeluga2.yaml b/user_data/instruction-templates/StableBeluga2.yaml
deleted file mode 100644
index d7d74319..00000000
--- a/user_data/instruction-templates/StableBeluga2.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '### System:\n' + 'This is a system prompt, please behave and help the user.' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '### System:\n' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### User:\n' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Assistant:\n' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Assistant:\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/StableLM.yaml b/user_data/instruction-templates/StableLM.yaml
deleted file mode 100644
index 7c80ca06..00000000
--- a/user_data/instruction-templates/StableLM.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '<|SYSTEM|>' + '\# StableLM Tuned (Alpha version)\n- StableLM is a helpful and harmless open-source AI language model developed by StabilityAI.\n- StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes.\n- StableLM will refuse to participate in anything that could harm a human.\n' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '<|SYSTEM|>' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|USER|>' + message['content'] + ''-}}
- {%- else -%}
- {{-'<|ASSISTANT|>' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|ASSISTANT|>'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/StableVicuna.yaml b/user_data/instruction-templates/StableVicuna.yaml
deleted file mode 100644
index 35c15846..00000000
--- a/user_data/instruction-templates/StableVicuna.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '### Assistant: I am StableVicuna, a large language model created by CarperAI. I am here to chat!' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Human: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'### Assistant: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Starchat-Beta.yaml b/user_data/instruction-templates/Starchat-Beta.yaml
deleted file mode 100644
index a96b0f28..00000000
--- a/user_data/instruction-templates/Starchat-Beta.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '<|system|>' + '' + '\n<|end|>\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '<|system|>' + message['content'] + '\n<|end|>\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|user|>\n' + message['content'] + '<|end|>\n'-}}
- {%- else -%}
- {{-'<|assistant|>\n' + message['content'] + '<|end|>\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|assistant|>\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Synthia-CoT.yaml b/user_data/instruction-templates/Synthia-CoT.yaml
deleted file mode 100644
index 5670be77..00000000
--- a/user_data/instruction-templates/Synthia-CoT.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set found_item = false -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set found_item = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not found_item -%}
- {{-'SYSTEM: ' + 'Elaborate on the topic using a Tree of Thoughts and backtrack when necessary to construct a clear, cohesive Chain of Thought reasoning. Always answer without hesitation.' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{-'SYSTEM: ' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Synthia.yaml b/user_data/instruction-templates/Synthia.yaml
deleted file mode 100644
index 5cecabea..00000000
--- a/user_data/instruction-templates/Synthia.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set found_item = false -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set found_item = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not found_item -%}
- {{-'SYSTEM: ' + 'Answer the question thoughtfully and intelligently. Always answer without hesitation.' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{-'SYSTEM: ' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Tulu.yaml b/user_data/instruction-templates/Tulu.yaml
deleted file mode 100644
index f60c9e41..00000000
--- a/user_data/instruction-templates/Tulu.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|user|>\n' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'<|assistant|>\n' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|assistant|>\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Vicuna-v0.yaml b/user_data/instruction-templates/Vicuna-v0.yaml
deleted file mode 100644
index d3e3f001..00000000
--- a/user_data/instruction-templates/Vicuna-v0.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human\'s questions.' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Human: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'### Assistant: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Vigogne-Chat.yaml b/user_data/instruction-templates/Vigogne-Chat.yaml
deleted file mode 100644
index 11ba5113..00000000
--- a/user_data/instruction-templates/Vigogne-Chat.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'Below is a conversation between a user and an AI assistant named Vigogne.\nVigogne is an open-source AI assistant created by Zaion (https://zaion.ai/).\nVigogne is polite, emotionally aware, humble-but-knowledgeable, always providing helpful and detailed answers.\nVigogne is skilled in responding proficiently in the languages its users use and can perform a wide range of tasks such as text editing, translation, question answering, logical reasoning, coding, and many others.\nVigogne cannot receive or generate audio or visual content and cannot access the internet.\nVigogne strictly avoids discussing sensitive, offensive, illegal, ethical, or political topics and caveats when unsure of the answer.\n' + '\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'<|USER|>: ' + message['content'] + '\n'-}}
- {%- else -%}
- {{-'<|ASSISTANT|>: ' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'<|ASSISTANT|>:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Vigogne-Instruct.yaml b/user_data/instruction-templates/Vigogne-Instruct.yaml
deleted file mode 100644
index cd7b6aa8..00000000
--- a/user_data/instruction-templates/Vigogne-Instruct.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + 'Ci-dessous se trouve une instruction qui décrit une tâche à accomplir. Rédigez une réponse qui répond de manière précise à la demande.' + '\n\n' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '\n\n' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Instruction:\n' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Réponse:\n' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Réponse:\n'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Wizard-Mega ShareGPT.yaml b/user_data/instruction-templates/Wizard-Mega ShareGPT.yaml
deleted file mode 100644
index 16a3ff7b..00000000
--- a/user_data/instruction-templates/Wizard-Mega ShareGPT.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'USER: ' + message['content'] + ' '-}}
- {%- else -%}
- {{-'ASSISTANT: ' + message['content'] + '' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'ASSISTANT:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Wizard-Mega.yaml b/user_data/instruction-templates/Wizard-Mega.yaml
deleted file mode 100644
index f3ca6990..00000000
--- a/user_data/instruction-templates/Wizard-Mega.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-'### Instruction: ' + message['content'] + '\n\n'-}}
- {%- else -%}
- {{-'### Assistant: ' + message['content'] + '\n\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-'### Assistant:'-}}
- {%- endif -%}
-
diff --git a/user_data/instruction-templates/Ziya.yaml b/user_data/instruction-templates/Ziya.yaml
deleted file mode 100644
index 45aa9c30..00000000
--- a/user_data/instruction-templates/Ziya.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-instruction_template: |-
- {%- set ns = namespace(found=false) -%}
- {%- for message in messages -%}
- {%- if message['role'] == 'system' -%}
- {%- set ns.found = true -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if not ns.found -%}
- {{- '' + '' + '' -}}
- {%- endif %}
- {%- for message in messages %}
- {%- if message['role'] == 'system' -%}
- {{- '' + message['content'] + '' -}}
- {%- else -%}
- {%- if message['role'] == 'user' -%}
- {{-':' + message['content'] + '\n'-}}
- {%- else -%}
- {{-':' + message['content'] + '\n' -}}
- {%- endif -%}
- {%- endif -%}
- {%- endfor -%}
- {%- if add_generation_prompt -%}
- {{-':'-}}
- {%- endif -%}
-
From e0e20ab9e7f0dfc529898b80c1a6c44561e85658 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 19 Mar 2026 08:02:23 -0700
Subject: [PATCH 23/47] Minor cleanup across multiple modules
---
extensions/openai/completions.py | 4 +-
modules/llama_cpp_server.py | 5 +--
modules/shared.py | 6 +--
modules/tool_parsing.py | 76 ++++++--------------------------
modules/training.py | 12 ++---
modules/ui.py | 7 ++-
6 files changed, 28 insertions(+), 82 deletions(-)
diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py
index fc17a19a..d0cd9802 100644
--- a/extensions/openai/completions.py
+++ b/extensions/openai/completions.py
@@ -263,7 +263,7 @@ def convert_history(history):
seen_non_system = True
meta = {}
tool_calls = entry.get("tool_calls")
- if tool_calls and isinstance(tool_calls, list) and len(tool_calls) > 0:
+ if tool_calls and isinstance(tool_calls, list):
meta["tool_calls"] = tool_calls
if content.strip() == "":
content = "" # keep empty content, don't skip
@@ -315,7 +315,7 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p
raise InvalidRequestError(message="messages is required", param='messages')
tools = None
- if 'tools' in body and body['tools'] is not None and isinstance(body['tools'], list) and len(body['tools']) > 0:
+ if 'tools' in body and body['tools'] is not None and isinstance(body['tools'], list) and body['tools']:
tools = validateTools(body['tools']) # raises InvalidRequestError if validation fails
tool_choice = body.get('tool_choice', None)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index 6dd36b2a..2ae01ddc 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -500,9 +500,8 @@ class LlamaServer:
health_url = f"http://127.0.0.1:{self.port}/health"
while True:
# Check if process is still alive
- if self.process.poll() is not None:
- # Process has terminated
- exit_code = self.process.poll()
+ exit_code = self.process.poll()
+ if exit_code is not None:
raise RuntimeError(f"Server process terminated unexpectedly with exit code: {exit_code}")
try:
diff --git a/modules/shared.py b/modules/shared.py
index 2382e714..37bc5876 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -453,15 +453,11 @@ def load_user_config():
'''
Loads custom model-specific settings
'''
+ user_config = {}
if Path(f'{args.model_dir}/config-user.yaml').exists():
file_content = open(f'{args.model_dir}/config-user.yaml', 'r').read().strip()
-
if file_content:
user_config = yaml.safe_load(file_content)
- else:
- user_config = {}
- else:
- user_config = {}
return user_config
diff --git a/modules/tool_parsing.py b/modules/tool_parsing.py
index 0454e901..7a7ed5d8 100644
--- a/modules/tool_parsing.py
+++ b/modules/tool_parsing.py
@@ -3,6 +3,10 @@ import random
import re
+def _make_tool_call(name, arguments):
+ return {"type": "function", "function": {"name": name, "arguments": arguments}}
+
+
def get_tool_call_id() -> str:
letter_bytes = "abcdefghijklmnopqrstuvwxyz0123456789"
b = [random.choice(letter_bytes) for _ in range(8)]
@@ -149,13 +153,7 @@ def _parse_channel_tool_calls(answer: str, tool_names: list[str]):
if start_pos is None:
prefix = answer.rfind('<|start|>assistant', 0, m.start())
start_pos = prefix if prefix != -1 else m.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
except json.JSONDecodeError:
pass
if matches:
@@ -185,13 +183,7 @@ def _parse_mistral_token_tool_calls(answer: str, tool_names: list[str]):
arguments = json.loads(json_str)
if start_pos is None:
start_pos = m.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
except json.JSONDecodeError:
pass
return matches, start_pos
@@ -226,13 +218,7 @@ def _parse_bare_name_tool_calls(answer: str, tool_names: list[str]):
arguments = json.loads(json_str)
if start_pos is None:
start_pos = match.start()
- matches.append({
- "type": "function",
- "function": {
- "name": name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(name, arguments))
except json.JSONDecodeError:
pass
return matches, start_pos
@@ -269,13 +255,7 @@ def _parse_xml_param_tool_calls(answer: str, tool_names: list[str]):
arguments[param_name] = param_value
if start_pos is None:
start_pos = tc_match.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
return matches, start_pos
@@ -305,13 +285,7 @@ def _parse_kimi_tool_calls(answer: str, tool_names: list[str]):
# Check for section begin marker before the call marker
section = answer.rfind('<|tool_calls_section_begin|>', 0, m.start())
start_pos = section if section != -1 else m.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
except json.JSONDecodeError:
pass
return matches, start_pos
@@ -348,13 +322,7 @@ def _parse_minimax_tool_calls(answer: str, tool_names: list[str]):
arguments[param_name] = param_value
if start_pos is None:
start_pos = tc_match.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
return matches, start_pos
@@ -382,13 +350,7 @@ def _parse_deep_seek_tool_calls(answer: str, tool_names: list[str]):
# Check for section begin marker before the call marker
section = answer.rfind('<｜tool▁calls▁begin｜>', 0, m.start())
start_pos = section if section != -1 else m.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
except json.JSONDecodeError:
pass
return matches, start_pos
@@ -428,13 +390,7 @@ def _parse_glm_tool_calls(answer: str, tool_names: list[str]):
arguments[k] = v
if start_pos is None:
start_pos = tc_match.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
return matches, start_pos
@@ -486,13 +442,7 @@ def _parse_pythonic_tool_calls(answer: str, tool_names: list[str]):
if start_pos is None:
start_pos = bracket_match.start()
- matches.append({
- "type": "function",
- "function": {
- "name": func_name,
- "arguments": arguments
- }
- })
+ matches.append(_make_tool_call(func_name, arguments))
return matches, start_pos
diff --git a/modules/training.py b/modules/training.py
index a13a2864..145353c6 100644
--- a/modules/training.py
+++ b/modules/training.py
@@ -732,11 +732,13 @@ def do_train(lora_name: str, always_override: bool, all_linear: bool, q_proj_en:
if lora_all_param > 0:
print(f"Trainable params: {lora_trainable_param:,d} ({100 * lora_trainable_param / lora_all_param:.4f} %), All params: {lora_all_param:,d} (Model: {model_all_params:,d})")
- train_log.update({"base_model_name": shared.model_name})
- train_log.update({"base_model_class": shared.model.__class__.__name__})
- train_log.update({"base_loaded_in_4bit": getattr(lora_model, "is_loaded_in_4bit", False)})
- train_log.update({"base_loaded_in_8bit": getattr(lora_model, "is_loaded_in_8bit", False)})
- train_log.update({"projections": projections_string})
+ train_log.update({
+ "base_model_name": shared.model_name,
+ "base_model_class": shared.model.__class__.__name__,
+ "base_loaded_in_4bit": getattr(lora_model, "is_loaded_in_4bit", False),
+ "base_loaded_in_8bit": getattr(lora_model, "is_loaded_in_8bit", False),
+ "projections": projections_string,
+ })
if stop_at_loss > 0:
print(f"Monitoring loss \033[1;31;1m(Auto-Stop at: {stop_at_loss})\033[0;37;0m")
diff --git a/modules/ui.py b/modules/ui.py
index bbb22266..20bc8373 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -299,7 +299,7 @@ def apply_interface_values(state, use_persistent=False):
elements = list_interface_input_elements()
- if len(state) == 0:
+ if not state:
return [gr.update() for k in elements] # Dummy, do nothing
else:
return [state[k] if k in state else gr.update() for k in elements]
@@ -307,9 +307,8 @@ def apply_interface_values(state, use_persistent=False):
def save_settings(state, preset, extensions_list, show_controls, theme_state, manual_save=False):
output = copy.deepcopy(shared.settings)
- exclude = []
for k in state:
- if k in shared.settings and k not in exclude:
+ if k in shared.settings:
output[k] = state[k]
if preset:
@@ -323,7 +322,7 @@ def save_settings(state, preset, extensions_list, show_controls, theme_state, ma
output['custom_stopping_strings'] = output.get('custom_stopping_strings') or ''
output['custom_token_bans'] = output.get('custom_token_bans') or ''
output['show_controls'] = show_controls
- output['dark_theme'] = True if theme_state == 'dark' else False
+ output['dark_theme'] = theme_state == 'dark'
output.pop('instruction_template_str')
output.pop('truncation_length')
From b3eb0e313d7f74e3f90c949d54f453d3f6846ae0 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 19 Mar 2026 11:53:12 -0700
Subject: [PATCH 24/47] Reduce the size of portable builds by using stripped
Python
---
.github/workflows/build-portable-release-cuda.yml | 4 ++--
.github/workflows/build-portable-release-rocm.yml | 4 ++--
.github/workflows/build-portable-release-vulkan.yml | 4 ++--
.github/workflows/build-portable-release.yml | 8 ++++----
4 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/build-portable-release-cuda.yml b/.github/workflows/build-portable-release-cuda.yml
index a5759112..5d66bd77 100644
--- a/.github/workflows/build-portable-release-cuda.yml
+++ b/.github/workflows/build-portable-release-cuda.yml
@@ -116,13 +116,13 @@ jobs:
# 1. Set platform-specific variables
if [[ "$RUNNER_OS" == "Windows" ]]; then
PLATFORM="windows"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only_stripped.tar.gz"
PIP_PATH="portable_env/python.exe -m pip"
PACKAGES_PATH="portable_env/Lib/site-packages"
rm start_linux.sh start_macos.sh
else
PLATFORM="linux"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"
PIP_PATH="portable_env/bin/python -m pip"
PACKAGES_PATH="portable_env/lib/python3.13/site-packages"
rm start_macos.sh start_windows.bat
diff --git a/.github/workflows/build-portable-release-rocm.yml b/.github/workflows/build-portable-release-rocm.yml
index 1050fa7e..b9a10bac 100644
--- a/.github/workflows/build-portable-release-rocm.yml
+++ b/.github/workflows/build-portable-release-rocm.yml
@@ -114,13 +114,13 @@ jobs:
# 1. Set platform-specific variables
if [[ "$RUNNER_OS" == "Windows" ]]; then
PLATFORM="windows"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only_stripped.tar.gz"
PIP_PATH="portable_env/python.exe -m pip"
PACKAGES_PATH="portable_env/Lib/site-packages"
rm start_linux.sh start_macos.sh
else
PLATFORM="linux"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"
PIP_PATH="portable_env/bin/python -m pip"
PACKAGES_PATH="portable_env/lib/python3.13/site-packages"
rm start_macos.sh start_windows.bat
diff --git a/.github/workflows/build-portable-release-vulkan.yml b/.github/workflows/build-portable-release-vulkan.yml
index b98b2e5e..9748d5b8 100644
--- a/.github/workflows/build-portable-release-vulkan.yml
+++ b/.github/workflows/build-portable-release-vulkan.yml
@@ -114,13 +114,13 @@ jobs:
# 1. Set platform-specific variables
if [[ "$RUNNER_OS" == "Windows" ]]; then
PLATFORM="windows"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only_stripped.tar.gz"
PIP_PATH="portable_env/python.exe -m pip"
PACKAGES_PATH="portable_env/Lib/site-packages"
rm start_linux.sh start_macos.sh
else
PLATFORM="linux"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"
PIP_PATH="portable_env/bin/python -m pip"
PACKAGES_PATH="portable_env/lib/python3.13/site-packages"
rm start_macos.sh start_windows.bat
diff --git a/.github/workflows/build-portable-release.yml b/.github/workflows/build-portable-release.yml
index 1bd4e163..e03116f6 100644
--- a/.github/workflows/build-portable-release.yml
+++ b/.github/workflows/build-portable-release.yml
@@ -115,18 +115,18 @@ jobs:
# 1. Set platform-specific variables
if [[ "$RUNNER_OS" == "Windows" ]]; then
PLATFORM="windows-cpu"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-pc-windows-msvc-install_only_stripped.tar.gz"
PIP_PATH="portable_env/python.exe -m pip"
PACKAGES_PATH="portable_env/Lib/site-packages"
rm start_linux.sh start_macos.sh
elif [[ "$RUNNER_OS" == "macOS" ]]; then
if [[ "$OS_TYPE" == "macos-15-intel" ]]; then
PLATFORM="macos-x86_64"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-apple-darwin-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-apple-darwin-install_only_stripped.tar.gz"
REQ_TYPE="apple_intel"
else
PLATFORM="macos-arm64"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-aarch64-apple-darwin-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-aarch64-apple-darwin-install_only_stripped.tar.gz"
REQ_TYPE="apple_silicon"
fi
PIP_PATH="portable_env/bin/python -m pip"
@@ -135,7 +135,7 @@ jobs:
else
# Linux case
PLATFORM="linux-cpu"
- PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only.tar.gz"
+ PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20260303/cpython-3.13.12+20260303-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz"
PIP_PATH="portable_env/bin/python -m pip"
PACKAGES_PATH="portable_env/lib/python3.13/site-packages"
rm start_macos.sh start_windows.bat
From 843de8b8a81edbd825cb03eb28af594fd3c7f3b1 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 19 Mar 2026 18:49:36 -0700
Subject: [PATCH 25/47] Update exllamav3 to 0.0.26
---
requirements/full/requirements.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index c8479d04..ad68ad59 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -42,7 +42,7 @@ tiktoken
# CUDA wheels
https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/turboderp-org/exllamav3/releases/download/v0.0.25/exllamav3-0.0.25+cu128.torch2.9.0-cp313-cp313-win_amd64.whl; platform_system == "Windows" and python_version == "3.13"
-https://github.com/turboderp-org/exllamav3/releases/download/v0.0.25/exllamav3-0.0.25+cu128.torch2.9.0-cp313-cp313-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.13"
+https://github.com/turboderp-org/exllamav3/releases/download/v0.0.26/exllamav3-0.0.26+cu128.torch2.9.0-cp313-cp313-win_amd64.whl; platform_system == "Windows" and python_version == "3.13"
+https://github.com/turboderp-org/exllamav3/releases/download/v0.0.26/exllamav3-0.0.26+cu128.torch2.9.0-cp313-cp313-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.13"
https://github.com/kingbri1/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu128torch2.9.0cxx11abiFALSE-cp313-cp313-win_amd64.whl; platform_system == "Windows" and python_version == "3.13"
https://github.com/kingbri1/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu128torch2.9.0cxx11abiFALSE-cp313-cp313-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.13"
From 2e4232e02bdf7640470ba1efdc5e72f1cd56b867 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 07:20:26 -0700
Subject: [PATCH 26/47] Minor cleanup
---
modules/callbacks.py | 2 +-
modules/utils.py | 8 --------
2 files changed, 1 insertion(+), 9 deletions(-)
diff --git a/modules/callbacks.py b/modules/callbacks.py
index 89fb6c08..6288de29 100644
--- a/modules/callbacks.py
+++ b/modules/callbacks.py
@@ -34,7 +34,7 @@ class Iteratorize:
def gentask():
try:
- ret = self.mfunc(callback=_callback, *args, **self.kwargs)
+ ret = self.mfunc(callback=_callback, *self.args, **self.kwargs)
except StopNowException:
pass
except Exception:
diff --git a/modules/utils.py b/modules/utils.py
index ff32e974..b01953ee 100644
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -81,14 +81,6 @@ def atoi(text):
return int(text) if text.isdigit() else text.lower()
-# Replace multiple string pairs in a string
-def replace_all(text, dic):
- for i, j in dic.items():
- text = text.replace(i, j)
-
- return text
-
-
def natural_keys(text):
return [atoi(c) for c in re.split(r'(\d+)', text)]
From bf6fbc019dbd9470efdeafa033818efa178d7735 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 14:46:00 -0300
Subject: [PATCH 27/47] API: Move OpenAI-compatible API from extensions/openai
to modules/api
---
.../workflows/build-portable-release-cuda.yml | 2 +-
.../workflows/build-portable-release-rocm.yml | 2 +-
.../build-portable-release-vulkan.yml | 2 +-
.github/workflows/build-portable-release.yml | 2 +-
docs/07 - Extensions.md | 1 -
docs/12 - OpenAI API.md | 12 +------
modules/api/__init__.py | 0
.../api}/cache_embedding_model.py | 0
.../openai => modules/api}/completions.py | 6 ++--
.../openai => modules/api}/embeddings.py | 10 +++---
{extensions/openai => modules/api}/errors.py | 0
{extensions/openai => modules/api}/images.py | 2 +-
{extensions/openai => modules/api}/logits.py | 2 +-
{extensions/openai => modules/api}/models.py | 0
.../openai => modules/api}/moderations.py | 2 +-
{extensions/openai => modules/api}/script.py | 34 ++++++++++---------
{extensions/openai => modules/api}/tokens.py | 0
{extensions/openai => modules/api}/typing.py | 0
{extensions/openai => modules/api}/utils.py | 3 +-
modules/extensions.py | 3 +-
modules/shared.py | 16 +--------
modules/ui_session.py | 2 --
server.py | 15 ++++++++
23 files changed, 51 insertions(+), 65 deletions(-)
create mode 100644 modules/api/__init__.py
rename {extensions/openai => modules/api}/cache_embedding_model.py (100%)
rename {extensions/openai => modules/api}/completions.py (99%)
rename {extensions/openai => modules/api}/embeddings.py (90%)
rename {extensions/openai => modules/api}/errors.py (100%)
rename {extensions/openai => modules/api}/images.py (96%)
rename {extensions/openai => modules/api}/logits.py (84%)
rename {extensions/openai => modules/api}/models.py (100%)
rename {extensions/openai => modules/api}/moderations.py (97%)
rename {extensions/openai => modules/api}/script.py (96%)
rename {extensions/openai => modules/api}/tokens.py (100%)
rename {extensions/openai => modules/api}/typing.py (100%)
rename {extensions/openai => modules/api}/utils.py (93%)
diff --git a/.github/workflows/build-portable-release-cuda.yml b/.github/workflows/build-portable-release-cuda.yml
index 5d66bd77..f9eea58a 100644
--- a/.github/workflows/build-portable-release-cuda.yml
+++ b/.github/workflows/build-portable-release-cuda.yml
@@ -106,7 +106,7 @@ jobs:
cd "text-generation-webui-${VERSION_CLEAN}"
# Remove extensions that need additional requirements
- allowed=("character_bias" "gallery" "openai" "sd_api_pictures")
+ allowed=("character_bias" "gallery" "sd_api_pictures")
find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf
# Define common variables
diff --git a/.github/workflows/build-portable-release-rocm.yml b/.github/workflows/build-portable-release-rocm.yml
index b9a10bac..db42b7dc 100644
--- a/.github/workflows/build-portable-release-rocm.yml
+++ b/.github/workflows/build-portable-release-rocm.yml
@@ -105,7 +105,7 @@ jobs:
cd "text-generation-webui-${VERSION_CLEAN}"
# Remove extensions that need additional requirements
- allowed=("character_bias" "gallery" "openai" "sd_api_pictures")
+ allowed=("character_bias" "gallery" "sd_api_pictures")
find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf
# Define common variables
diff --git a/.github/workflows/build-portable-release-vulkan.yml b/.github/workflows/build-portable-release-vulkan.yml
index 9748d5b8..8f5aa7c8 100644
--- a/.github/workflows/build-portable-release-vulkan.yml
+++ b/.github/workflows/build-portable-release-vulkan.yml
@@ -105,7 +105,7 @@ jobs:
cd "text-generation-webui-${VERSION_CLEAN}"
# Remove extensions that need additional requirements
- allowed=("character_bias" "gallery" "openai" "sd_api_pictures")
+ allowed=("character_bias" "gallery" "sd_api_pictures")
find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf
# Define common variables
diff --git a/.github/workflows/build-portable-release.yml b/.github/workflows/build-portable-release.yml
index e03116f6..9ace90f6 100644
--- a/.github/workflows/build-portable-release.yml
+++ b/.github/workflows/build-portable-release.yml
@@ -105,7 +105,7 @@ jobs:
cd "text-generation-webui-${VERSION_CLEAN}"
# Remove extensions that need additional requirements
- allowed=("character_bias" "gallery" "openai" "sd_api_pictures")
+ allowed=("character_bias" "gallery" "sd_api_pictures")
find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf
# Define common variables
diff --git a/docs/07 - Extensions.md b/docs/07 - Extensions.md
index 48cd30ce..779b2a34 100644
--- a/docs/07 - Extensions.md
+++ b/docs/07 - Extensions.md
@@ -20,7 +20,6 @@ If you create an extension, you are welcome to host it in a GitHub repository an
|Extension|Description|
|---------|-----------|
-|[openai](https://github.com/oobabooga/text-generation-webui/tree/main/extensions/openai)| Creates an API that mimics the OpenAI API and can be used as a drop-in replacement. |
|[superboogav2](https://github.com/oobabooga/text-generation-webui/tree/main/extensions/superboogav2)| Enhanced RAG extension with support for PDF, DOCX, and PPTX files. |
|[send_pictures](https://github.com/oobabooga/text-generation-webui/blob/main/extensions/send_pictures/)| Creates an image upload field that can be used to send images to the bot in chat mode. Captions are automatically generated using BLIP. |
|[coqui_tts](https://github.com/oobabooga/text-generation-webui/tree/main/extensions/coqui_tts)| Text-to-speech extension using Coqui XTTS v2. |
diff --git a/docs/12 - OpenAI API.md b/docs/12 - OpenAI API.md
index 637ccced..276a7e19 100644
--- a/docs/12 - OpenAI API.md
+++ b/docs/12 - OpenAI API.md
@@ -19,7 +19,7 @@ Add `--api` to your command-line flags.
### Examples
-For the documentation with all the endpoints, parameters and their types, consult `http://127.0.0.1:5000/docs` or the [typing.py](https://github.com/oobabooga/text-generation-webui/blob/main/extensions/openai/typing.py) file.
+For the documentation with all the endpoints, parameters and their types, consult `http://127.0.0.1:5000/docs` or the [typing.py](https://github.com/oobabooga/text-generation-webui/blob/main/modules/api/typing.py) file.
The official examples in the [OpenAI documentation](https://platform.openai.com/docs/api-reference) should also work, and the same parameters apply (although the API here has more optional parameters).
@@ -490,16 +490,6 @@ The following environment variables can be used (they take precedence over every
| `OPENEDAI_EMBEDDING_MODEL` | Embedding model (if applicable) | sentence-transformers/all-mpnet-base-v2 |
| `OPENEDAI_EMBEDDING_DEVICE` | Embedding device (if applicable) | cuda |
-#### Persistent settings with `settings.yaml`
-
-You can also set the following variables in your `settings.yaml` file:
-
-```
-openai-embedding_device: cuda
-openai-embedding_model: "sentence-transformers/all-mpnet-base-v2"
-openai-debug: 1
-```
-
### Third-party application setup
You can usually force an application that uses the OpenAI API to connect to the local API by using the following environment variables:
diff --git a/modules/api/__init__.py b/modules/api/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/extensions/openai/cache_embedding_model.py b/modules/api/cache_embedding_model.py
similarity index 100%
rename from extensions/openai/cache_embedding_model.py
rename to modules/api/cache_embedding_model.py
diff --git a/extensions/openai/completions.py b/modules/api/completions.py
similarity index 99%
rename from extensions/openai/completions.py
rename to modules/api/completions.py
index d0cd9802..8948bb86 100644
--- a/extensions/openai/completions.py
+++ b/modules/api/completions.py
@@ -9,9 +9,9 @@ import tiktoken
import yaml
from pydantic import ValidationError
-from extensions.openai.errors import InvalidRequestError
-from extensions.openai.typing import ToolDefinition
-from extensions.openai.utils import debug_msg
+from .errors import InvalidRequestError
+from .typing import ToolDefinition
+from .utils import debug_msg
from modules.tool_parsing import get_tool_call_id, parse_tool_call, detect_tool_call_format
from modules import shared
from modules.reasoning import extract_reasoning
diff --git a/extensions/openai/embeddings.py b/modules/api/embeddings.py
similarity index 90%
rename from extensions/openai/embeddings.py
rename to modules/api/embeddings.py
index 1420879c..ad299c9d 100644
--- a/extensions/openai/embeddings.py
+++ b/modules/api/embeddings.py
@@ -3,8 +3,8 @@ import os
import numpy as np
from transformers import AutoModel
-from extensions.openai.errors import ServiceUnavailableError
-from extensions.openai.utils import debug_msg, float_list_to_base64
+from .errors import ServiceUnavailableError
+from .utils import debug_msg, float_list_to_base64
from modules.logging_colors import logger
embeddings_params_initialized = False
@@ -17,14 +17,12 @@ def initialize_embedding_params():
'''
global embeddings_params_initialized
if not embeddings_params_initialized:
- from extensions.openai.script import params
-
global st_model, embeddings_model, embeddings_device
- st_model = os.environ.get("OPENEDAI_EMBEDDING_MODEL", params.get('embedding_model', 'all-mpnet-base-v2'))
+ st_model = os.environ.get("OPENEDAI_EMBEDDING_MODEL", 'sentence-transformers/all-mpnet-base-v2')
embeddings_model = None
# OPENEDAI_EMBEDDING_DEVICE: auto (best or cpu), cpu, cuda, ipu, xpu, mkldnn, opengl, opencl, ideep, hip, ve, fpga, ort, xla, lazy, vulkan, mps, meta, hpu, mtia, privateuseone
- embeddings_device = os.environ.get("OPENEDAI_EMBEDDING_DEVICE", params.get('embedding_device', 'cpu'))
+ embeddings_device = os.environ.get("OPENEDAI_EMBEDDING_DEVICE", 'cpu')
if embeddings_device.lower() == 'auto':
embeddings_device = None
diff --git a/extensions/openai/errors.py b/modules/api/errors.py
similarity index 100%
rename from extensions/openai/errors.py
rename to modules/api/errors.py
diff --git a/extensions/openai/images.py b/modules/api/images.py
similarity index 96%
rename from extensions/openai/images.py
rename to modules/api/images.py
index f7be3d22..95704535 100644
--- a/extensions/openai/images.py
+++ b/modules/api/images.py
@@ -6,7 +6,7 @@ import base64
import io
import time
-from extensions.openai.errors import ServiceUnavailableError
+from .errors import ServiceUnavailableError
from modules import shared
diff --git a/extensions/openai/logits.py b/modules/api/logits.py
similarity index 84%
rename from extensions/openai/logits.py
rename to modules/api/logits.py
index 280612db..e0c7ea0e 100644
--- a/extensions/openai/logits.py
+++ b/modules/api/logits.py
@@ -1,4 +1,4 @@
-from extensions.openai.completions import process_parameters
+from .completions import process_parameters
from modules.logits import get_next_logits
diff --git a/extensions/openai/models.py b/modules/api/models.py
similarity index 100%
rename from extensions/openai/models.py
rename to modules/api/models.py
diff --git a/extensions/openai/moderations.py b/modules/api/moderations.py
similarity index 97%
rename from extensions/openai/moderations.py
rename to modules/api/moderations.py
index 1ca6b8ab..ac0539d6 100644
--- a/extensions/openai/moderations.py
+++ b/modules/api/moderations.py
@@ -3,7 +3,7 @@ import time
import numpy as np
from numpy.linalg import norm
-from extensions.openai.embeddings import get_embeddings
+from .embeddings import get_embeddings
moderations_disabled = False # return 0/false
category_embeddings = None
diff --git a/extensions/openai/script.py b/modules/api/script.py
similarity index 96%
rename from extensions/openai/script.py
rename to modules/api/script.py
index a0d5deb8..356919e9 100644
--- a/extensions/openai/script.py
+++ b/modules/api/script.py
@@ -13,16 +13,15 @@ from fastapi import Depends, FastAPI, Header, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.requests import Request
from fastapi.responses import JSONResponse
-from pydub import AudioSegment
from sse_starlette import EventSourceResponse
from starlette.concurrency import iterate_in_threadpool
-import extensions.openai.completions as OAIcompletions
-import extensions.openai.logits as OAIlogits
-import extensions.openai.models as OAImodels
-from extensions.openai.tokens import token_count, token_decode, token_encode
-from extensions.openai.errors import OpenAIError
-from extensions.openai.utils import _start_cloudflared
+import modules.api.completions as OAIcompletions
+import modules.api.logits as OAIlogits
+import modules.api.models as OAImodels
+from .tokens import token_count, token_decode, token_encode
+from .errors import OpenAIError
+from .utils import _start_cloudflared
from modules import shared
from modules.logging_colors import logger
from modules.models import unload_model
@@ -53,12 +52,6 @@ from .typing import (
to_dict
)
-params = {
- 'embedding_device': 'cpu',
- 'embedding_model': 'sentence-transformers/all-mpnet-base-v2',
- 'debug': 0
-}
-
async def _wait_for_disconnect(request: Request, stop_event: threading.Event):
"""Block until the client disconnects, then signal the stop_event."""
@@ -244,6 +237,7 @@ def handle_billing_usage():
@app.post('/v1/audio/transcriptions', dependencies=check_key)
async def handle_audio_transcription(request: Request):
import speech_recognition as sr
+ from pydub import AudioSegment
r = sr.Recognizer()
@@ -275,7 +269,7 @@ async def handle_audio_transcription(request: Request):
@app.post('/v1/images/generations', response_model=ImageGenerationResponse, dependencies=check_key)
async def handle_image_generation(request_data: ImageGenerationRequest):
- import extensions.openai.images as OAIimages
+ import modules.api.images as OAIimages
response = await asyncio.to_thread(OAIimages.generations, request_data)
return JSONResponse(response)
@@ -283,7 +277,7 @@ async def handle_image_generation(request_data: ImageGenerationRequest):
@app.post("/v1/embeddings", response_model=EmbeddingsResponse, dependencies=check_key)
async def handle_embeddings(request: Request, request_data: EmbeddingsRequest):
- import extensions.openai.embeddings as OAIembeddings
+ import modules.api.embeddings as OAIembeddings
input = request_data.input
if not input:
@@ -298,7 +292,7 @@ async def handle_embeddings(request: Request, request_data: EmbeddingsRequest):
@app.post("/v1/moderations", dependencies=check_key)
async def handle_moderations(request: Request):
- import extensions.openai.moderations as OAImoderations
+ import modules.api.moderations as OAImoderations
body = await request.json()
input = body["input"]
@@ -500,7 +494,15 @@ def run_server():
uvicorn.run(app, host=server_addrs, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, access_log=False)
+_server_started = False
+
+
def setup():
+ global _server_started
+ if _server_started:
+ return
+
+ _server_started = True
if shared.args.nowebui:
run_server()
else:
diff --git a/extensions/openai/tokens.py b/modules/api/tokens.py
similarity index 100%
rename from extensions/openai/tokens.py
rename to modules/api/tokens.py
diff --git a/extensions/openai/typing.py b/modules/api/typing.py
similarity index 100%
rename from extensions/openai/typing.py
rename to modules/api/typing.py
diff --git a/extensions/openai/utils.py b/modules/api/utils.py
similarity index 93%
rename from extensions/openai/utils.py
rename to modules/api/utils.py
index 2b414769..fae181ff 100644
--- a/extensions/openai/utils.py
+++ b/modules/api/utils.py
@@ -23,8 +23,7 @@ def float_list_to_base64(float_array: np.ndarray) -> str:
def debug_msg(*args, **kwargs):
- from extensions.openai.script import params
- if os.environ.get("OPENEDAI_DEBUG", params.get('debug', 0)):
+ if os.environ.get("OPENEDAI_DEBUG", 0):
print(*args, **kwargs)
diff --git a/modules/extensions.py b/modules/extensions.py
index 4bb7b683..09db9f40 100644
--- a/modules/extensions.py
+++ b/modules/extensions.py
@@ -32,8 +32,7 @@ def load_extensions():
if name not in available_extensions:
continue
- if name != 'api':
- logger.info(f'Loading the extension "{name}"')
+ logger.info(f'Loading the extension "{name}"')
try:
# Prefer user extension, fall back to system extension
diff --git a/modules/shared.py b/modules/shared.py
index 37bc5876..69e16960 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -156,7 +156,7 @@ group.add_argument('--portable', action='store_true', help='Hide features not av
# API
group = parser.add_argument_group('API')
-group.add_argument('--api', action='store_true', help='Enable the API extension.')
+group.add_argument('--api', action='store_true', help='Enable the API server.')
group.add_argument('--public-api', action='store_true', help='Create a public URL for the API using Cloudflare.')
group.add_argument('--public-api-id', type=str, help='Tunnel ID for named Cloudflare Tunnel. Use together with public-api option.', default=None)
group.add_argument('--api-port', type=int, default=5000, help='The listening port for the API.')
@@ -435,16 +435,6 @@ def fix_loader_name(name):
return 'TensorRT-LLM'
-def add_extension(name, last=False):
- if args.extensions is None:
- args.extensions = [name]
- elif last:
- args.extensions = [x for x in args.extensions if x != name]
- args.extensions.append(name)
- elif name not in args.extensions:
- args.extensions.append(name)
-
-
def is_chat():
return True
@@ -464,10 +454,6 @@ def load_user_config():
args.loader = fix_loader_name(args.loader)
-# Activate the API extension
-if args.api or args.public_api:
- add_extension('openai', last=True)
-
# Load model-specific settings
p = Path(f'{args.model_dir}/config.yaml')
if p.exists():
diff --git a/modules/ui_session.py b/modules/ui_session.py
index 19026fbb..3f2c8a7b 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -95,8 +95,6 @@ def set_interface_arguments(extensions, bool_active):
setattr(shared.args, k, False)
for k in bool_active:
setattr(shared.args, k, True)
- if k == 'api':
- shared.add_extension('openai', last=True)
shared.need_restart = True
diff --git a/server.py b/server.py
index 1aa9fc04..cbdd2854 100644
--- a/server.py
+++ b/server.py
@@ -106,6 +106,11 @@ def create_interface():
if shared.args.extensions is not None and len(shared.args.extensions) > 0:
extensions_module.load_extensions()
+ # Start the API server if enabled
+ if shared.args.api or shared.args.public_api:
+ from modules.api.script import setup as api_setup
+ api_setup()
+
# Force some events to be triggered on page load
shared.persistent_interface_state.update({
'mode': shared.settings['mode'],
@@ -273,6 +278,12 @@ if __name__ == "__main__":
# Activate the extensions listed on settings.yaml
extensions_module.available_extensions = utils.get_available_extensions()
for extension in shared.settings['default_extensions']:
+ # The openai extension was moved to modules/api and is now
+ # activated with --api. Treat it as an alias for backwards compat.
+ if extension == 'openai':
+ shared.args.api = True
+ continue
+
shared.args.extensions = shared.args.extensions or []
if extension not in shared.args.extensions:
shared.args.extensions.append(extension)
@@ -337,6 +348,10 @@ if __name__ == "__main__":
shared.args.extensions = [x for x in (shared.args.extensions or []) if x != 'gallery']
if shared.args.extensions:
extensions_module.load_extensions()
+
+ if shared.args.api or shared.args.public_api:
+ from modules.api.script import setup as api_setup
+ api_setup()
else:
# Launch the web UI
create_interface()
From 1a910574c36b6b1d93a3bf3303335201993f503a Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 14:57:01 -0300
Subject: [PATCH 28/47] API: Fix debug_msg truthy check for OPENEDAI_DEBUG=0
---
modules/api/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/api/utils.py b/modules/api/utils.py
index fae181ff..f021c378 100644
--- a/modules/api/utils.py
+++ b/modules/api/utils.py
@@ -23,7 +23,7 @@ def float_list_to_base64(float_array: np.ndarray) -> str:
def debug_msg(*args, **kwargs):
- if os.environ.get("OPENEDAI_DEBUG", 0):
+ if int(os.environ.get("OPENEDAI_DEBUG", 0)):
print(*args, **kwargs)
From 855141967c4081f9f90a1b5b7fd091a14c543e8f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 15:03:17 -0300
Subject: [PATCH 29/47] API: Handle --extensions openai as alias for --api
---
server.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/server.py b/server.py
index cbdd2854..d224909c 100644
--- a/server.py
+++ b/server.py
@@ -288,6 +288,11 @@ if __name__ == "__main__":
if extension not in shared.args.extensions:
shared.args.extensions.append(extension)
+ # Handle --extensions openai from the command line (moved to modules/api)
+ if shared.args.extensions and 'openai' in shared.args.extensions:
+ shared.args.extensions.remove('openai')
+ shared.args.api = True
+
# Load image model if specified via CLI
if shared.args.image_model:
logger.info(f"Loading image model: {shared.args.image_model}")
From 7c79143a149d1618287ca0b526826ee04167f7d9 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 15:03:49 -0300
Subject: [PATCH 30/47] API: Fix _start_cloudflared raising after first attempt
instead of exhausting retries
---
modules/api/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/api/utils.py b/modules/api/utils.py
index f021c378..e8c505f6 100644
--- a/modules/api/utils.py
+++ b/modules/api/utils.py
@@ -50,4 +50,4 @@ def _start_cloudflared(port: int, tunnel_id: str, max_attempts: int = 3, on_star
traceback.print_exc()
time.sleep(3)
- raise Exception('Could not start cloudflared.')
+ raise Exception('Could not start cloudflared.')
From f0e3997f375d61961c7032a09145f41c254d799f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 16:04:57 -0300
Subject: [PATCH 31/47] Add missing __init__.py to modules/grammar
---
modules/grammar/__init__.py | 0
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 modules/grammar/__init__.py
diff --git a/modules/grammar/__init__.py b/modules/grammar/__init__.py
new file mode 100644
index 00000000..e69de29b
From 0216893475b415106ce631f62fc62bcd9d345f8a Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 20 Mar 2026 19:05:36 -0300
Subject: [PATCH 32/47] API: Add Anthropic-compatible /v1/messages endpoint
---
modules/api/anthropic.py | 468 +++++++++++++++++++++++++++++++++++++++
modules/api/script.py | 115 +++++++++-
modules/api/typing.py | 21 +-
3 files changed, 600 insertions(+), 4 deletions(-)
create mode 100644 modules/api/anthropic.py
diff --git a/modules/api/anthropic.py b/modules/api/anthropic.py
new file mode 100644
index 00000000..5fbf5caf
--- /dev/null
+++ b/modules/api/anthropic.py
@@ -0,0 +1,468 @@
+import json
+import time
+
+from modules import shared
+
+
+def convert_request(body: dict) -> dict:
+ """Transform Anthropic Messages API body into the dict that chat_completions_common expects."""
+ messages = []
+
+ # System message
+ system = body.get('system')
+ if system:
+ if isinstance(system, list):
+ # List of content blocks like [{"type":"text","text":"..."}]
+ text_parts = [block.get('text', '') for block in system if isinstance(block, dict) and block.get('type') == 'text']
+ system_text = '\n'.join(text_parts)
+ else:
+ system_text = str(system)
+ if system_text:
+ messages.append({"role": "system", "content": system_text})
+
+ # Convert messages
+ for msg in body.get('messages', []):
+ role = msg.get('role')
+ content = msg.get('content')
+
+ if isinstance(content, str):
+ messages.append({"role": role, "content": content})
+ continue
+
+ if not isinstance(content, list):
+ messages.append({"role": role, "content": str(content) if content else ""})
+ continue
+
+ if role == 'assistant':
+ # Split into text content, tool_calls, and skip thinking blocks
+ text_parts = []
+ tool_calls = []
+ for block in content:
+ btype = block.get('type')
+ if btype == 'text':
+ text_parts.append(block.get('text', ''))
+ elif btype == 'tool_use':
+ tool_calls.append({
+ "id": block.get('id', ''),
+ "type": "function",
+ "function": {
+ "name": block.get('name', ''),
+ "arguments": json.dumps(block.get('input', {}))
+ }
+ })
+ elif btype == 'thinking':
+ pass # Strip thinking blocks
+
+ assistant_msg = {"role": "assistant", "content": '\n'.join(text_parts) if text_parts else ""}
+ if tool_calls:
+ assistant_msg["tool_calls"] = tool_calls
+ messages.append(assistant_msg)
+
+ elif role == 'user':
+ # Handle tool_result blocks and regular content
+ regular_parts = []
+ for block in content:
+ btype = block.get('type')
+ if btype == 'tool_result':
+ # Emit any accumulated regular content first
+ if regular_parts:
+ if len(regular_parts) == 1 and regular_parts[0].get('type') == 'text':
+ messages.append({"role": "user", "content": regular_parts[0]['text']})
+ else:
+ messages.append({"role": "user", "content": regular_parts})
+ regular_parts = []
+ # Convert tool_result to OpenAI tool message
+ tool_content = block.get('content', '')
+ if isinstance(tool_content, list):
+ tool_content = '\n'.join(
+ b.get('text', '') for b in tool_content
+ if isinstance(b, dict) and b.get('type') == 'text'
+ )
+ messages.append({
+ "role": "tool",
+ "tool_call_id": block.get('tool_use_id', ''),
+ "content": str(tool_content)
+ })
+ elif btype == 'text':
+ regular_parts.append({"type": "text", "text": block.get('text', '')})
+ elif btype == 'image':
+ source = block.get('source', {})
+ if source.get('type') == 'base64':
+ media_type = source.get('media_type', 'image/png')
+ data = source.get('data', '')
+ regular_parts.append({
+ "type": "image_url",
+ "image_url": {"url": f"data:{media_type};base64,{data}"}
+ })
+ elif btype == 'thinking':
+ pass # Strip thinking blocks
+
+ if regular_parts:
+ if len(regular_parts) == 1 and regular_parts[0].get('type') == 'text':
+ messages.append({"role": "user", "content": regular_parts[0]['text']})
+ else:
+ messages.append({"role": "user", "content": regular_parts})
+ else:
+ messages.append({"role": role, "content": str(content)})
+
+ # Start with all fields from the original body (includes GenerationOptions defaults)
+ result = dict(body)
+
+ # Remove Anthropic-specific fields that don't map directly
+ for key in ('system', 'stop_sequences', 'tools', 'tool_choice', 'thinking', 'metadata'):
+ result.pop(key, None)
+
+ # Set converted fields
+ result['messages'] = messages
+ result['max_tokens'] = body.get('max_tokens', 4096)
+ result['stream'] = body.get('stream', False)
+ result['mode'] = 'instruct'
+
+ # Ensure ChatCompletionRequestParams defaults are present
+ result.setdefault('continue_', False)
+ result.setdefault('instruction_template', None)
+ result.setdefault('instruction_template_str', None)
+ result.setdefault('character', None)
+ result.setdefault('bot_name', None)
+ result.setdefault('context', None)
+ result.setdefault('greeting', None)
+ result.setdefault('user_name', None)
+ result.setdefault('user_bio', None)
+ result.setdefault('chat_template_str', None)
+ result.setdefault('chat_instruct_command', 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>')
+ result.setdefault('frequency_penalty', None)
+ result.setdefault('presence_penalty', None)
+ result.setdefault('logit_bias', None)
+ result.setdefault('logprobs', None)
+ result.setdefault('top_logprobs', None)
+ result.setdefault('n', 1)
+ result.setdefault('model', None)
+ result.setdefault('functions', None)
+ result.setdefault('function_call', None)
+ result.setdefault('stream_options', None)
+ result.setdefault('user', None)
+ result.setdefault('stop', None)
+ result.setdefault('tool_choice', None)
+
+ # Always request usage in streaming so the usage-only chunk triggers
+ # the deferred message_delta/message_stop with accurate output_tokens
+ if body.get('stream', False):
+ result['stream_options'] = {'include_usage': True}
+
+ # Map stop_sequences -> stop
+ if body.get('stop_sequences'):
+ result['stop'] = body['stop_sequences']
+
+ # Tools
+ if body.get('tools'):
+ result['tools'] = [
+ {
+ "type": "function",
+ "function": {
+ "name": t.get('name', ''),
+ "description": t.get('description', ''),
+ "parameters": t.get('input_schema', {"type": "object", "properties": {}})
+ }
+ }
+ for t in body['tools']
+ ]
+
+ # Tool choice
+ tc = body.get('tool_choice')
+ if tc and isinstance(tc, dict):
+ tc_type = tc.get('type')
+ if tc_type == 'auto':
+ result['tool_choice'] = 'auto'
+ elif tc_type == 'any':
+ result['tool_choice'] = 'required'
+ elif tc_type == 'tool':
+ result['tool_choice'] = {"type": "function", "function": {"name": tc.get('name', '')}}
+ elif tc_type == 'none':
+ result['tool_choice'] = 'none'
+ else:
+ result.setdefault('tool_choice', None)
+
+ # Thinking
+ thinking = body.get('thinking')
+ if thinking and isinstance(thinking, dict) and thinking.get('type') in ('enabled', 'adaptive'):
+ result['enable_thinking'] = True
+
+ return result
+
+
+_FINISH_REASON_MAP = {
+ "stop": "end_turn",
+ "length": "max_tokens",
+ "tool_calls": "tool_use",
+}
+
+
+def build_response(openai_resp: dict, model: str) -> dict:
+ """Transform OpenAI chat completion response dict into Anthropic Messages format."""
+ resp_id = openai_resp.get('id', 'msg_unknown')
+ if resp_id.startswith('chatcmpl-'):
+ resp_id = 'msg_' + resp_id[9:]
+
+ choice = openai_resp.get('choices', [{}])[0]
+ message = choice.get('message', {})
+
+ content = []
+
+ # Reasoning/thinking content
+ reasoning = message.get('reasoning_content')
+ if reasoning:
+ content.append({"type": "thinking", "thinking": reasoning, "signature": ""})
+
+ # Text content
+ text = message.get('content')
+ if text:
+ content.append({"type": "text", "text": text})
+
+ # Tool calls
+ tool_calls = message.get('tool_calls')
+ if tool_calls:
+ for tc in tool_calls:
+ func = tc.get('function', {})
+ try:
+ input_data = json.loads(func.get('arguments', '{}'))
+ except (json.JSONDecodeError, TypeError):
+ input_data = {}
+ content.append({
+ "type": "tool_use",
+ "id": tc.get('id', ''),
+ "name": func.get('name', ''),
+ "input": input_data
+ })
+
+ finish_reason = choice.get('finish_reason', 'stop')
+ stop_reason = _FINISH_REASON_MAP.get(finish_reason, 'end_turn')
+
+ usage = openai_resp.get('usage', {})
+
+ return {
+ "id": resp_id,
+ "type": "message",
+ "role": "assistant",
+ "content": content,
+ "model": model,
+ "stop_reason": stop_reason,
+ "stop_sequence": None,
+ "usage": {
+ "input_tokens": usage.get('prompt_tokens', 0),
+ "output_tokens": usage.get('completion_tokens', 0),
+ }
+ }
+
+
+class StreamConverter:
+ """Stateful converter: processes one OpenAI chunk at a time, yields Anthropic SSE events.
+
+ When include_usage is enabled in the OpenAI request, the final chunk with
+ finish_reason has usage=None, followed by a separate usage-only chunk
+ (choices=[], usage={...}). We defer emitting message_delta and message_stop
+ until we receive that usage chunk so output_tokens is accurate.
+ """
+
+ def __init__(self, model: str):
+ self.model = model
+ self.msg_id = "msg_%d" % int(time.time() * 1000000000)
+ self.block_index = 0
+ self.in_thinking = False
+ self.in_text = False
+ self.input_tokens = 0
+ self.output_tokens = 0
+ self.tool_calls_accum = {}
+ self.stop_reason = "end_turn"
+ self._pending_finish = False # True after we've seen finish_reason
+
+ def process_chunk(self, chunk: dict) -> list[dict]:
+ """Process a single OpenAI streaming chunk; return list of Anthropic SSE event dicts."""
+ events = []
+ choices = chunk.get('choices', [])
+ usage = chunk.get('usage')
+
+ if usage:
+ self.input_tokens = usage.get('prompt_tokens', self.input_tokens)
+ self.output_tokens = usage.get('completion_tokens', self.output_tokens)
+
+ # Usage-only chunk (choices=[]) arrives after the finish chunk
+ if not choices:
+ if self._pending_finish:
+ events.extend(self.finish())
+ return events
+
+ choice = choices[0]
+ delta = choice.get('delta', {})
+ finish_reason = choice.get('finish_reason')
+
+ # First chunk with role
+ if 'role' in delta:
+ events.append({
+ "event": "message_start",
+ "data": json.dumps({
+ "type": "message_start",
+ "message": {
+ "id": self.msg_id,
+ "type": "message",
+ "role": "assistant",
+ "content": [],
+ "model": self.model,
+ "stop_reason": None,
+ "stop_sequence": None,
+ "usage": {"input_tokens": self.input_tokens, "output_tokens": 0}
+ }
+ })
+ })
+ events.append({"event": "ping", "data": json.dumps({"type": "ping"})})
+ return events
+
+ # Reasoning content
+ reasoning_content = delta.get('reasoning_content')
+ if reasoning_content:
+ if not self.in_thinking:
+ self.in_thinking = True
+ events.append({
+ "event": "content_block_start",
+ "data": json.dumps({
+ "type": "content_block_start",
+ "index": self.block_index,
+ "content_block": {"type": "thinking", "thinking": ""}
+ })
+ })
+ events.append({
+ "event": "content_block_delta",
+ "data": json.dumps({
+ "type": "content_block_delta",
+ "index": self.block_index,
+ "delta": {"type": "thinking_delta", "thinking": reasoning_content}
+ })
+ })
+ return events
+
+ # Text content
+ text_content = delta.get('content')
+ if text_content:
+ if self.in_thinking:
+ events.append({
+ "event": "content_block_stop",
+ "data": json.dumps({"type": "content_block_stop", "index": self.block_index})
+ })
+ self.in_thinking = False
+ self.block_index += 1
+
+ if not self.in_text:
+ self.in_text = True
+ events.append({
+ "event": "content_block_start",
+ "data": json.dumps({
+ "type": "content_block_start",
+ "index": self.block_index,
+ "content_block": {"type": "text", "text": ""}
+ })
+ })
+ events.append({
+ "event": "content_block_delta",
+ "data": json.dumps({
+ "type": "content_block_delta",
+ "index": self.block_index,
+ "delta": {"type": "text_delta", "text": text_content}
+ })
+ })
+ return events
+
+ # Tool calls in delta
+ chunk_tool_calls = delta.get('tool_calls')
+ if chunk_tool_calls:
+ for tc in chunk_tool_calls:
+ tc_id = tc.get('id', '')
+ tc_idx = tc.get('index', 0)
+ func = tc.get('function', {})
+ if tc_id:
+ self.tool_calls_accum[tc_idx] = {
+ "id": tc_id,
+ "name": func.get('name', ''),
+ "arguments": func.get('arguments', '')
+ }
+ elif tc_idx in self.tool_calls_accum:
+ self.tool_calls_accum[tc_idx]["arguments"] += func.get('arguments', '')
+
+ # Final chunk — close open content blocks, defer message_delta/stop for usage
+ if finish_reason is not None:
+ self.stop_reason = _FINISH_REASON_MAP.get(finish_reason, 'end_turn')
+
+ if self.in_thinking:
+ events.append({
+ "event": "content_block_stop",
+ "data": json.dumps({"type": "content_block_stop", "index": self.block_index})
+ })
+ self.in_thinking = False
+ self.block_index += 1
+
+ if self.in_text:
+ events.append({
+ "event": "content_block_stop",
+ "data": json.dumps({"type": "content_block_stop", "index": self.block_index})
+ })
+ self.in_text = False
+ self.block_index += 1
+
+ for tc_idx in sorted(self.tool_calls_accum.keys()):
+ tc = self.tool_calls_accum[tc_idx]
+ arguments_str = tc["arguments"] or "{}"
+
+ events.append({
+ "event": "content_block_start",
+ "data": json.dumps({
+ "type": "content_block_start",
+ "index": self.block_index,
+ "content_block": {
+ "type": "tool_use",
+ "id": tc["id"],
+ "name": tc["name"],
+ "input": {}
+ }
+ })
+ })
+ # Emit the full input as a single input_json_delta so SDK
+ # clients that reconstruct from deltas get the correct data
+ events.append({
+ "event": "content_block_delta",
+ "data": json.dumps({
+ "type": "content_block_delta",
+ "index": self.block_index,
+ "delta": {
+ "type": "input_json_delta",
+ "partial_json": arguments_str
+ }
+ })
+ })
+ events.append({
+ "event": "content_block_stop",
+ "data": json.dumps({"type": "content_block_stop", "index": self.block_index})
+ })
+ self.block_index += 1
+
+ # Defer message_delta/stop — usage chunk may follow
+ self._pending_finish = True
+
+ return events
+
+ def finish(self) -> list[dict]:
+ """Emit deferred message_delta and message_stop. Safe to call multiple times."""
+ if not self._pending_finish:
+ return []
+ self._pending_finish = False
+ return [
+ {
+ "event": "message_delta",
+ "data": json.dumps({
+ "type": "message_delta",
+ "delta": {"stop_reason": self.stop_reason, "stop_sequence": None},
+ "usage": {"input_tokens": self.input_tokens, "output_tokens": self.output_tokens}
+ })
+ },
+ {
+ "event": "message_stop",
+ "data": json.dumps({"type": "message_stop"})
+ }
+ ]
diff --git a/modules/api/script.py b/modules/api/script.py
index 356919e9..a94247fa 100644
--- a/modules/api/script.py
+++ b/modules/api/script.py
@@ -10,6 +10,7 @@ from threading import Thread
import uvicorn
from fastapi import Depends, FastAPI, Header, HTTPException
+from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.requests import Request
from fastapi.responses import JSONResponse
@@ -19,6 +20,7 @@ from starlette.concurrency import iterate_in_threadpool
import modules.api.completions as OAIcompletions
import modules.api.logits as OAIlogits
import modules.api.models as OAImodels
+import modules.api.anthropic as Anthropic
from .tokens import token_count, token_decode, token_encode
from .errors import OpenAIError
from .utils import _start_cloudflared
@@ -28,6 +30,7 @@ from modules.models import unload_model
from modules.text_generation import stop_everything_event # used by /v1/internal/stop-generation
from .typing import (
+ AnthropicRequest,
ChatCompletionRequest,
ChatCompletionResponse,
ChatPromptResponse,
@@ -74,9 +77,23 @@ def verify_admin_key(authorization: str = Header(None)) -> None:
raise HTTPException(status_code=401, detail="Unauthorized")
+def verify_anthropic_key(x_api_key: str = Header(None, alias="x-api-key")) -> None:
+ expected_api_key = shared.args.api_key
+ if expected_api_key and (x_api_key is None or x_api_key != expected_api_key):
+ raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+class AnthropicError(Exception):
+ def __init__(self, message: str, error_type: str = "invalid_request_error", status_code: int = 400):
+ self.message = message
+ self.error_type = error_type
+ self.status_code = status_code
+
+
app = FastAPI()
check_key = [Depends(verify_api_key)]
check_admin_key = [Depends(verify_admin_key)]
+check_anthropic_key = [Depends(verify_anthropic_key)]
# Configure CORS settings to allow all origins, methods, and headers
app.add_middleware(
@@ -102,6 +119,28 @@ async def openai_error_handler(request: Request, exc: OpenAIError):
)
+@app.exception_handler(AnthropicError)
+async def anthropic_error_handler(request: Request, exc: AnthropicError):
+ return JSONResponse(
+ status_code=exc.status_code,
+ content={"type": "error", "error": {"type": exc.error_type, "message": exc.message}}
+ )
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_error_handler(request: Request, exc: RequestValidationError):
+ if request.url.path.startswith("/v1/messages"):
+ messages = "; ".join(
+ f"{'.'.join(str(l) for l in e['loc'])}: {e['msg']}" for e in exc.errors()
+ )
+ return JSONResponse(
+ status_code=400,
+ content={"type": "error", "error": {"type": "invalid_request_error", "message": messages}}
+ )
+
+ return JSONResponse(status_code=422, content={"detail": exc.errors()})
+
+
@app.middleware("http")
async def validate_host_header(request: Request, call_next):
# Be strict about only approving access to localhost by default
@@ -211,6 +250,76 @@ async def openai_chat_completions(request: Request, request_data: ChatCompletion
return JSONResponse(response)
+@app.post('/v1/messages', dependencies=check_anthropic_key)
+async def anthropic_messages(request: Request, request_data: AnthropicRequest):
+ body = to_dict(request_data)
+ model = body.get('model') or shared.model_name or 'unknown'
+
+ try:
+ converted = Anthropic.convert_request(body)
+ except Exception as e:
+ raise AnthropicError(message=str(e))
+
+ try:
+ return await _anthropic_generate(request, request_data, converted, model)
+ except OpenAIError as e:
+ error_type = "invalid_request_error" if e.code < 500 else "api_error"
+ if e.code == 503:
+ error_type = "overloaded_error"
+ raise AnthropicError(message=e.message, error_type=error_type, status_code=e.code)
+ except Exception as e:
+ raise AnthropicError(message=str(e) or "Internal server error", error_type="api_error", status_code=500)
+
+
+async def _anthropic_generate(request, request_data, converted, model):
+ if request_data.stream:
+ stop_event = threading.Event()
+
+ async def generator():
+ converter = Anthropic.StreamConverter(model)
+ response = OAIcompletions.stream_chat_completions(converted, is_legacy=False, stop_event=stop_event)
+ try:
+ async for resp in iterate_in_threadpool(response):
+ disconnected = await request.is_disconnected()
+ if disconnected:
+ break
+
+ for event in converter.process_chunk(resp):
+ yield event
+
+ for event in converter.finish():
+ yield event
+ except OpenAIError as e:
+ error_type = "invalid_request_error" if e.code < 500 else "api_error"
+ if e.code == 503:
+ error_type = "overloaded_error"
+ yield {
+ "event": "error",
+ "data": json.dumps({"type": "error", "error": {"type": error_type, "message": e.message}})
+ }
+ finally:
+ stop_event.set()
+ response.close()
+
+ return EventSourceResponse(generator(), sep="\n")
+
+ else:
+ stop_event = threading.Event()
+ monitor = asyncio.create_task(_wait_for_disconnect(request, stop_event))
+ try:
+ openai_resp = await asyncio.to_thread(
+ OAIcompletions.chat_completions,
+ converted,
+ is_legacy=False,
+ stop_event=stop_event
+ )
+ finally:
+ stop_event.set()
+ monitor.cancel()
+
+ return JSONResponse(Anthropic.build_response(openai_resp, model))
+
+
@app.get("/v1/models", dependencies=check_key)
@app.get("/v1/models/{model}", dependencies=check_key)
async def handle_models(request: Request):
@@ -469,15 +578,15 @@ def run_server():
port,
shared.args.public_api_id,
max_attempts=3,
- on_start=lambda url: logger.info(f'OpenAI-compatible API URL:\n\n{url}/v1\n')
+ on_start=lambda url: logger.info(f'API URL (OpenAI + Anthropic compatible):\n\n{url}/v1\n')
)
else:
url_proto = 'https://' if (ssl_certfile and ssl_keyfile) else 'http://'
urls = [f'{url_proto}{addr}:{port}/v1' for addr in server_addrs]
if len(urls) > 1:
- logger.info('OpenAI-compatible API URLs:\n\n' + '\n'.join(urls) + '\n')
+ logger.info('API URLs (OpenAI + Anthropic compatible):\n\n' + '\n'.join(urls) + '\n')
else:
- logger.info('OpenAI-compatible API URL:\n\n' + '\n'.join(urls) + '\n')
+ logger.info('API URL (OpenAI + Anthropic compatible):\n\n' + '\n'.join(urls) + '\n')
# Log API keys
if shared.args.api_key:
diff --git a/modules/api/typing.py b/modules/api/typing.py
index 80831c44..1d486e8f 100644
--- a/modules/api/typing.py
+++ b/modules/api/typing.py
@@ -144,7 +144,7 @@ class CompletionResponse(BaseModel):
class ChatCompletionRequestParams(BaseModel):
- messages: List[dict]
+ messages: List[dict] = Field(..., min_length=1)
model: str | None = Field(default=None, description="Unused parameter. To change the model, use the /v1/internal/model/load endpoint.")
frequency_penalty: float | None = shared.args.frequency_penalty
function_call: str | dict | None = Field(default=None, description="Unused parameter.")
@@ -282,6 +282,25 @@ class LoadLorasRequest(BaseModel):
lora_names: List[str]
+class AnthropicRequestParams(BaseModel):
+ model: str | None = None
+ messages: List[dict] = Field(..., min_length=1)
+ max_tokens: int
+ system: str | list | None = None
+ temperature: float | None = shared.args.temperature
+ top_p: float | None = shared.args.top_p
+ stop_sequences: list[str] | None = None
+ stream: bool = False
+ tools: list[dict] | None = None
+ tool_choice: dict | None = None
+ thinking: dict | None = None
+ metadata: dict | None = None
+
+
+class AnthropicRequest(GenerationOptions, AnthropicRequestParams):
+ pass
+
+
class ImageGenerationRequest(BaseModel):
"""Image-specific parameters for generation."""
prompt: str
From f2c909725ef667821a0e2ef5d68f4a2b86f0fd49 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 21 Mar 2026 11:09:06 -0700
Subject: [PATCH 33/47] API: Use top_p=0.95 by default
---
modules/shared.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/shared.py b/modules/shared.py
index 69e16960..16ccbe77 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -175,7 +175,7 @@ group.add_argument('--dynatemp-high', type=float, default=_d['dynatemp_high'], m
group.add_argument('--dynatemp-exponent', type=float, default=_d['dynatemp_exponent'], metavar='N', help='Dynamic temperature exponent')
group.add_argument('--smoothing-factor', type=float, default=_d['smoothing_factor'], metavar='N', help='Smoothing factor')
group.add_argument('--smoothing-curve', type=float, default=_d['smoothing_curve'], metavar='N', help='Smoothing curve')
-group.add_argument('--top-p', type=float, default=_d['top_p'], metavar='N', help='Top P')
+group.add_argument('--top-p', type=float, default=0.95, metavar='N', help='Top P')
group.add_argument('--top-k', type=int, default=_d['top_k'], metavar='N', help='Top K')
group.add_argument('--min-p', type=float, default=_d['min_p'], metavar='N', help='Min P')
group.add_argument('--top-n-sigma', type=float, default=_d['top_n_sigma'], metavar='N', help='Top N Sigma')
From 2c4f36433986001f80fdbb0f9095aa68f43274d2 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 21 Mar 2026 18:38:11 -0700
Subject: [PATCH 34/47] Update API docs to mention Anthropic support
---
README.md | 2 +-
docs/12 - OpenAI API.md | 4 ++--
modules/api/script.py | 6 +++---
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/README.md b/README.md
index cabb81fc..7e5566ec 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ A Gradio web UI for running Large Language Models locally. 100% private and offl
- **File attachments**: Upload text files, PDF documents, and .docx documents to talk about their contents.
- **Vision (multimodal)**: Attach images to messages for visual understanding ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Multimodal-Tutorial)).
- **Tool-calling**: Models can call custom functions during chat — web search, page fetching, math, and more. Each tool is a single `.py` file, easy to create and extend ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Tool-Calling-Tutorial)).
-- **OpenAI-compatible API**: Chat and Completions endpoints with tool-calling support. Use as a local drop-in replacement for the OpenAI API ([examples](https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API#examples)).
+- **OpenAI/Anthropic-compatible API**: Chat, Completions, and Messages endpoints with tool-calling support. Use as a local drop-in replacement for the OpenAI/Anthropic APIs ([examples](https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API#examples)).
- **Training**: Fine-tune LoRAs on multi-turn chat or raw text datasets. Supports resuming interrupted runs ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/05-%E2%80%90-Training-Tab)).
- **Image generation**: A dedicated tab for `diffusers` models like **Z-Image-Turbo**. Features 4-bit/8-bit quantization and a persistent gallery with metadata ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Image-Generation-Tutorial)).
- **Easy setup**: [Portable builds](https://github.com/oobabooga/text-generation-webui/releases) (zero setup, just unzip and run) for GGUF models on Windows/Linux/macOS, or a one-click installer for the full feature set.
diff --git a/docs/12 - OpenAI API.md b/docs/12 - OpenAI API.md
index 276a7e19..2a7a7f69 100644
--- a/docs/12 - OpenAI API.md
+++ b/docs/12 - OpenAI API.md
@@ -1,6 +1,6 @@
-## OpenAI compatible API
+## OpenAI/Anthropic-compatible API
-The main API for this project is meant to be a drop-in replacement to the OpenAI API, including Chat and Completions endpoints.
+The main API for this project is meant to be a drop-in replacement for the OpenAI and Anthropic APIs, including Chat, Completions, and Messages endpoints.
* It is 100% offline and private.
* It doesn't create any logs.
diff --git a/modules/api/script.py b/modules/api/script.py
index a94247fa..5913c2c5 100644
--- a/modules/api/script.py
+++ b/modules/api/script.py
@@ -578,15 +578,15 @@ def run_server():
port,
shared.args.public_api_id,
max_attempts=3,
- on_start=lambda url: logger.info(f'API URL (OpenAI + Anthropic compatible):\n\n{url}/v1\n')
+ on_start=lambda url: logger.info(f'OpenAI/Anthropic-compatible API URL:\n\n{url}/v1\n')
)
else:
url_proto = 'https://' if (ssl_certfile and ssl_keyfile) else 'http://'
urls = [f'{url_proto}{addr}:{port}/v1' for addr in server_addrs]
if len(urls) > 1:
- logger.info('API URLs (OpenAI + Anthropic compatible):\n\n' + '\n'.join(urls) + '\n')
+ logger.info('OpenAI/Anthropic-compatible API URLs:\n\n' + '\n'.join(urls) + '\n')
else:
- logger.info('API URL (OpenAI + Anthropic compatible):\n\n' + '\n'.join(urls) + '\n')
+ logger.info('OpenAI/Anthropic-compatible API URL:\n\n' + '\n'.join(urls) + '\n')
# Log API keys
if shared.args.api_key:
From 9488df3e489c97cc26018d9ae1dc6a4bc0384f1b Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 21 Mar 2026 20:47:26 -0700
Subject: [PATCH 35/47] llama.cpp: Don't suppress llama-server logs
---
modules/llama_cpp_server.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index 2ae01ddc..b77a8605 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -588,8 +588,11 @@ def filter_stderr_with_progress(process_stderr):
print(display_line, end=end_char, file=sys.stderr, flush=True)
last_was_progress = (progress < 1.0)
- # skip noise lines
- elif not (line.startswith(('srv ', 'slot ')) or 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line):
+ # skip health check polling and parser warnings
+ elif 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line:
+ continue
+
+ else:
# if we were in progress, finish that line first
if last_was_progress:
print(file=sys.stderr)
From 1dda5e47111eaf8cb90f25ffb94e47296def5c8f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 21 Mar 2026 20:58:45 -0700
Subject: [PATCH 36/47] Follow-up to previous commit
---
modules/llama_cpp_server.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index b77a8605..5cbf2122 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -588,8 +588,8 @@ def filter_stderr_with_progress(process_stderr):
print(display_line, end=end_char, file=sys.stderr, flush=True)
last_was_progress = (progress < 1.0)
- # skip health check polling and parser warnings
- elif 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line:
+ # skip noise lines
+ elif 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line or (last_was_progress and ('memory_seq_rm' in line or 'context checkpoint' in line)):
continue
else:
From bde496ea5daf9f7fa9a0ac90f8f8f25166738112 Mon Sep 17 00:00:00 2001
From: Phrosty1
Date: Sun, 22 Mar 2026 20:48:56 -0400
Subject: [PATCH 37/47] Fix prompt corruption when continuing with context
truncation (#7439)
---
modules/chat.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/modules/chat.py b/modules/chat.py
index 148d559a..f8088e0f 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -434,6 +434,8 @@ def generate_chat_prompt(user_input, state, **kwargs):
messages.append({"role": "user", "content": "fake user message replace me"})
def make_prompt(messages):
+ if _continue:
+ messages = copy.deepcopy(messages)
last_message = messages[-1].copy()
if _continue:
if state['mode'] == 'chat-instruct':
From 9ec20d9730db3f41270da12f51f7ce138fb8705c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sun, 22 Mar 2026 19:16:24 -0700
Subject: [PATCH 38/47] Strip thinking blocks before tool-call parsing
---
modules/tool_parsing.py | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/modules/tool_parsing.py b/modules/tool_parsing.py
index 7a7ed5d8..ec49f77f 100644
--- a/modules/tool_parsing.py
+++ b/modules/tool_parsing.py
@@ -2,6 +2,8 @@ import json
import random
import re
+from modules.reasoning import extract_reasoning
+
def _make_tool_call(name, arguments):
return {"type": "function", "function": {"name": name, "arguments": arguments}}
@@ -41,6 +43,10 @@ def streaming_tool_buffer_check(text, markers=None, tool_names=None, check_bare_
check_bare_names: Whether to do partial-prefix matching on tool
names (for models with unknown template format).
'''
+ # Strip thinking blocks so tool-call syntax inside doesn't
+ # trigger false positives.
+ _, text = extract_reasoning(text)
+
# Full marker found in text → buffer permanently.
# Always checks ALL known markers regardless of template (cheap safety net).
for marker in TOOL_CALL_OPENING_MARKERS:
@@ -543,12 +549,19 @@ def detect_tool_call_format(template_str):
def parse_tool_call(answer: str, tool_names: list[str], return_prefix: bool = False, parsers: list = None):
+ # Strip thinking blocks so tool-call syntax inside is ignored.
+ original_answer = answer
+ _, answer = extract_reasoning(answer)
+ # Offset between original and stripped text, used to map start_pos
+ # back to the original string when returning a prefix.
+ reasoning_offset = len(original_answer) - len(answer)
+
matches = []
start_pos = None
def _return(matches, start_pos):
if return_prefix:
- prefix = answer[:start_pos] if matches and start_pos is not None else ''
+ prefix = original_answer[:start_pos + reasoning_offset] if matches and start_pos is not None else ''
return matches, prefix
return matches
From 307d0c92be2a4f8ac97f2be6c2cc3af1b9c8ad6f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 23 Mar 2026 06:35:14 -0700
Subject: [PATCH 39/47] UI polish
---
css/main.css | 8 ++++----
modules/ui.py | 1 +
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/css/main.css b/css/main.css
index 22fac5c5..a8c30a3f 100644
--- a/css/main.css
+++ b/css/main.css
@@ -54,7 +54,7 @@ div.svelte-iyf88w {
height: 39.594px;
align-self: end;
line-height: 1em;
- border-radius: 0.375rem;
+ border-radius: 0.75rem;
flex: none;
}
@@ -1420,7 +1420,7 @@ audio {
}
.dark .thinking-block {
- background-color: var(--darker-gray);
+ background-color: var(--selected-item-color-dark);
border: 1px solid var(--input-border-color);
}
@@ -1558,7 +1558,7 @@ strong {
min-height: 200px;
max-height: 65vh;
padding: 10px;
- border-radius: 5px;
+ border-radius: 0.5rem;
border: 1px solid #ccc;
background-color: var(--light-theme-gray);
font-family: inherit;
@@ -1586,7 +1586,7 @@ strong {
.edit-control-button {
padding: 6px 12px;
border: 1px solid #ccc;
- border-radius: 4px;
+ border-radius: 0.75rem;
cursor: pointer;
background-color: #f8f9fa;
color: #212529;
diff --git a/modules/ui.py b/modules/ui.py
index 20bc8373..02b5a9fb 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -115,6 +115,7 @@ if not shared.args.old_colors:
input_shadow_focus='none',
input_shadow_focus_dark='none',
button_large_radius='0.75rem',
+ button_small_radius='0.75rem',
button_large_padding='6px 12px',
input_radius='0.5rem',
block_radius='0.375rem',
From 02f18a1d65881cb3ed291050a191d8cf712b7115 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 23 Mar 2026 07:06:38 -0700
Subject: [PATCH 40/47] API: Add thinking block signature field, fix error
codes, clean up logging
---
modules/api/anthropic.py | 2 +-
modules/api/embeddings.py | 4 ++--
modules/api/moderations.py | 2 --
modules/api/script.py | 9 +++++++--
4 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/modules/api/anthropic.py b/modules/api/anthropic.py
index 5fbf5caf..3fab09a6 100644
--- a/modules/api/anthropic.py
+++ b/modules/api/anthropic.py
@@ -326,7 +326,7 @@ class StreamConverter:
"data": json.dumps({
"type": "content_block_start",
"index": self.block_index,
- "content_block": {"type": "thinking", "thinking": ""}
+ "content_block": {"type": "thinking", "thinking": "", "signature": ""}
})
})
events.append({
diff --git a/modules/api/embeddings.py b/modules/api/embeddings.py
index ad299c9d..16cf0482 100644
--- a/modules/api/embeddings.py
+++ b/modules/api/embeddings.py
@@ -39,14 +39,14 @@ def load_embedding_model(model: str):
initialize_embedding_params()
global embeddings_device, embeddings_model
try:
- print(f"Try embedding model: {model} on {embeddings_device}")
+ logger.info(f"Try embedding model: {model} on {embeddings_device}")
if 'jina-embeddings' in model:
embeddings_model = AutoModel.from_pretrained(model, trust_remote_code=True) # trust_remote_code is needed to use the encode method
embeddings_model = embeddings_model.to(embeddings_device)
else:
embeddings_model = SentenceTransformer(model, device=embeddings_device)
- print(f"Loaded embedding model: {model}")
+ logger.info(f"Loaded embedding model: {model}")
except Exception as e:
embeddings_model = None
raise ServiceUnavailableError(f"Error: Failed to load embedding model: {model}", internal_message=repr(e))
diff --git a/modules/api/moderations.py b/modules/api/moderations.py
index ac0539d6..a41763cf 100644
--- a/modules/api/moderations.py
+++ b/modules/api/moderations.py
@@ -64,6 +64,4 @@ def moderations(input):
'category_scores': category_scores,
}])
- print(results)
-
return results
diff --git a/modules/api/script.py b/modules/api/script.py
index 5913c2c5..85f4974f 100644
--- a/modules/api/script.py
+++ b/modules/api/script.py
@@ -506,12 +506,17 @@ async def handle_load_model(request_data: LoadModelRequest):
return JSONResponse(content="OK")
except Exception:
traceback.print_exc()
- raise HTTPException(status_code=400, detail="Failed to load the model.")
+ raise HTTPException(status_code=500, detail="Failed to load the model.")
@app.post("/v1/internal/model/unload", dependencies=check_admin_key)
async def handle_unload_model():
- unload_model()
+ try:
+ unload_model()
+ return JSONResponse(content="OK")
+ except Exception:
+ traceback.print_exc()
+ raise HTTPException(status_code=500, detail="Failed to unload the model.")
@app.get("/v1/internal/lora/list", response_model=LoraListResponse, dependencies=check_admin_key)
From 286bbb685d7bc585b8d82fd0e8d23515aeff9cb0 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 23 Mar 2026 20:22:46 -0700
Subject: [PATCH 41/47] Revert "Follow-up to previous commit"
This reverts commit 1dda5e47111eaf8cb90f25ffb94e47296def5c8f.
---
modules/llama_cpp_server.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index 5cbf2122..b77a8605 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -588,8 +588,8 @@ def filter_stderr_with_progress(process_stderr):
print(display_line, end=end_char, file=sys.stderr, flush=True)
last_was_progress = (progress < 1.0)
- # skip noise lines
- elif 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line or (last_was_progress and ('memory_seq_rm' in line or 'context checkpoint' in line)):
+ # skip health check polling and parser warnings
+ elif 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line:
continue
else:
From a7ef430b38c2f6e7c9a043b2f94ec6c2108d1480 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 23 Mar 2026 20:22:51 -0700
Subject: [PATCH 42/47] Revert "llama.cpp: Don't suppress llama-server logs"
This reverts commit 9488df3e489c97cc26018d9ae1dc6a4bc0384f1b.
---
modules/llama_cpp_server.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index b77a8605..2ae01ddc 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -588,11 +588,8 @@ def filter_stderr_with_progress(process_stderr):
print(display_line, end=end_char, file=sys.stderr, flush=True)
last_was_progress = (progress < 1.0)
- # skip health check polling and parser warnings
- elif 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line:
- continue
-
- else:
+ # skip noise lines
+ elif not (line.startswith(('srv ', 'slot ')) or 'log_server_r: request: GET /health' in line or 'No parser definition detected' in line):
# if we were in progress, finish that line first
if last_was_progress:
print(file=sys.stderr)
From c9d2240f5045baed0f234f3937614bdbe63af340 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 24 Mar 2026 06:45:39 -0700
Subject: [PATCH 43/47] Update README
---
README.md | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md
index 7e5566ec..ab6cc2e5 100644
--- a/README.md
+++ b/README.md
@@ -23,21 +23,20 @@ A Gradio web UI for running Large Language Models locally. 100% private and offl
## Features
+- **Easy setup**: [Portable builds](https://github.com/oobabooga/text-generation-webui/releases) (zero setup, just unzip and run) for GGUF models on Windows/Linux/macOS, or a one-click installer for the full feature set.
- **Multiple backends**: [llama.cpp](https://github.com/ggerganov/llama.cpp), [Transformers](https://github.com/huggingface/transformers), [ExLlamaV3](https://github.com/turboderp-org/exllamav3), and [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). Switch between backends and models without restarting.
-- **File attachments**: Upload text files, PDF documents, and .docx documents to talk about their contents.
-- **Vision (multimodal)**: Attach images to messages for visual understanding ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Multimodal-Tutorial)).
-- **Tool-calling**: Models can call custom functions during chat — web search, page fetching, math, and more. Each tool is a single `.py` file, easy to create and extend ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Tool-Calling-Tutorial)).
- **OpenAI/Anthropic-compatible API**: Chat, Completions, and Messages endpoints with tool-calling support. Use as a local drop-in replacement for the OpenAI/Anthropic APIs ([examples](https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API#examples)).
+- **Tool-calling**: Models can call custom functions during chat — web search, page fetching, math, and more. Each tool is a single `.py` file, easy to create and extend ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Tool-Calling-Tutorial)).
+- **Vision (multimodal)**: Attach images to messages for visual understanding ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Multimodal-Tutorial)).
+- **File attachments**: Upload text files, PDF documents, and .docx documents to talk about their contents.
- **Training**: Fine-tune LoRAs on multi-turn chat or raw text datasets. Supports resuming interrupted runs ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/05-%E2%80%90-Training-Tab)).
- **Image generation**: A dedicated tab for `diffusers` models like **Z-Image-Turbo**. Features 4-bit/8-bit quantization and a persistent gallery with metadata ([tutorial](https://github.com/oobabooga/text-generation-webui/wiki/Image-Generation-Tutorial)).
-- **Easy setup**: [Portable builds](https://github.com/oobabooga/text-generation-webui/releases) (zero setup, just unzip and run) for GGUF models on Windows/Linux/macOS, or a one-click installer for the full feature set.
- 100% offline and private, with zero telemetry, external resources, or remote update requests.
- `instruct` mode for instruction-following (like ChatGPT), and `chat-instruct`/`chat` modes for talking to custom characters. Prompts are automatically formatted with Jinja2 templates.
- Edit messages, navigate between message versions, and branch conversations at any point.
- Free-form text generation in the Notebook tab without being limited to chat turns.
- Multiple sampling parameters and generation options for sophisticated text generation control.
-- Aesthetic UI with dark and light themes.
-- Syntax highlighting for code blocks and LaTeX rendering for mathematical expressions.
+- Dark/light themes, syntax highlighting for code blocks, and LaTeX rendering for mathematical expressions.
- Extension support, with numerous built-in and user-contributed extensions available. See the [wiki](https://github.com/oobabooga/text-generation-webui/wiki/07-%E2%80%90-Extensions) and [extensions directory](https://github.com/oobabooga/text-generation-webui-extensions) for details.
## How to install
@@ -429,7 +428,7 @@ API generation defaults:
That's it. The UI will detect it automatically.
-To check what will fit your GPU, you can use the [VRAM Calculator](https://huggingface.co/spaces/oobabooga/accurate-gguf-vram-calculator).
+To estimate how much memory a model will use, you can use the [GGUF Memory Calculator](https://huggingface.co/spaces/oobabooga/accurate-gguf-vram-calculator).
Other model types (Transformers, EXL3)
From 5b8da154b7aa4475718b819abba8acc1354e34eb Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 24 Mar 2026 09:34:59 -0700
Subject: [PATCH 44/47] Update llama.cpp
---
requirements/full/requirements.txt | 4 ++--
requirements/full/requirements_amd.txt | 4 ++--
requirements/full/requirements_apple_intel.txt | 2 +-
requirements/full/requirements_apple_silicon.txt | 2 +-
requirements/full/requirements_cpu_only.txt | 4 ++--
requirements/portable/requirements.txt | 4 ++--
requirements/portable/requirements_amd.txt | 4 ++--
requirements/portable/requirements_apple_intel.txt | 2 +-
requirements/portable/requirements_apple_silicon.txt | 2 +-
requirements/portable/requirements_cpu_only.txt | 4 ++--
requirements/portable/requirements_cuda131.txt | 4 ++--
requirements/portable/requirements_vulkan.txt | 4 ++--
12 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index ad68ad59..56619627 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -40,8 +40,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav3/releases/download/v0.0.26/exllamav3-0.0.26+cu128.torch2.9.0-cp313-cp313-win_amd64.whl; platform_system == "Windows" and python_version == "3.13"
https://github.com/turboderp-org/exllamav3/releases/download/v0.0.26/exllamav3-0.0.26+cu128.torch2.9.0-cp313-cp313-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.13"
https://github.com/kingbri1/flash-attention/releases/download/v2.8.3/flash_attn-2.8.3+cu128torch2.9.0cxx11abiFALSE-cp313-cp313-win_amd64.whl; platform_system == "Windows" and python_version == "3.13"
diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt
index b11e50b7..620683cc 100644
--- a/requirements/full/requirements_amd.txt
+++ b/requirements/full/requirements_amd.txt
@@ -37,5 +37,5 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+rocm7.2-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+rocm7.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+rocm7.2-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+rocm7.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt
index d147af3f..b1f109b2 100644
--- a/requirements/full/requirements_apple_intel.txt
+++ b/requirements/full/requirements_apple_intel.txt
@@ -37,4 +37,4 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0-py3-none-macosx_13_0_x86_64.whl; platform_system == "Darwin"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0-py3-none-macosx_13_0_x86_64.whl; platform_system == "Darwin"
diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt
index d284c5d5..a54476a9 100644
--- a/requirements/full/requirements_apple_silicon.txt
+++ b/requirements/full/requirements_apple_silicon.txt
@@ -37,4 +37,4 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin"
diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt
index 3952054e..be82c904 100644
--- a/requirements/full/requirements_cpu_only.txt
+++ b/requirements/full/requirements_cpu_only.txt
@@ -37,5 +37,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cpu-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cpu-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cpu-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cpu-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt
index abf7690c..188da380 100644
--- a/requirements/portable/requirements.txt
+++ b/requirements/portable/requirements.txt
@@ -23,5 +23,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_amd.txt b/requirements/portable/requirements_amd.txt
index 0d66c16c..4562b6d0 100644
--- a/requirements/portable/requirements_amd.txt
+++ b/requirements/portable/requirements_amd.txt
@@ -23,5 +23,5 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+rocm7.2-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+rocm7.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+rocm7.2-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+rocm7.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt
index 0658239a..04dcf25e 100644
--- a/requirements/portable/requirements_apple_intel.txt
+++ b/requirements/portable/requirements_apple_intel.txt
@@ -23,4 +23,4 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0-py3-none-macosx_13_0_x86_64.whl; platform_system == "Darwin"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0-py3-none-macosx_13_0_x86_64.whl; platform_system == "Darwin"
diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt
index b66e2b38..4b8af78a 100644
--- a/requirements/portable/requirements_apple_silicon.txt
+++ b/requirements/portable/requirements_apple_silicon.txt
@@ -23,4 +23,4 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin"
diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt
index bb815bb2..5b0eaf89 100644
--- a/requirements/portable/requirements_cpu_only.txt
+++ b/requirements/portable/requirements_cpu_only.txt
@@ -23,5 +23,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cpu-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cpu-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cpu-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cpu-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_cuda131.txt b/requirements/portable/requirements_cuda131.txt
index d57ba40b..90b3234f 100644
--- a/requirements/portable/requirements_cuda131.txt
+++ b/requirements/portable/requirements_cuda131.txt
@@ -23,5 +23,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu131-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+cu131-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cu131-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+cu131-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt
index 6abd8920..ea72b4ec 100644
--- a/requirements/portable/requirements_vulkan.txt
+++ b/requirements/portable/requirements_vulkan.txt
@@ -23,5 +23,5 @@ sse-starlette==1.6.5
tiktoken
# Vulkan wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.94.0/llama_cpp_binaries-0.94.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.98.0/llama_cpp_binaries-0.98.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
From 5814e745be03d1f6f4cc6614e7a10d45282024b8 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 24 Mar 2026 11:14:22 -0700
Subject: [PATCH 45/47] UI: Minor polish
---
css/main.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/css/main.css b/css/main.css
index a8c30a3f..d42bd6ae 100644
--- a/css/main.css
+++ b/css/main.css
@@ -582,7 +582,7 @@ audio {
#chat-input textarea {
background: #f3f4f6;
- padding: 0.65rem 2.5rem 0.6rem;
+ padding: 0.675rem 2.5rem 0.6rem;
margin-top: 0.15rem;
border: 1px solid #d2d2d8;
border-radius: 1.5rem;
From 750502695c4339dc525d50cf428960d7ffbeeb05 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 24 Mar 2026 11:39:24 -0700
Subject: [PATCH 46/47] Fix GPT-OSS tool-calling after 9ec20d97
---
modules/reasoning.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/modules/reasoning.py b/modules/reasoning.py
index 9c92719b..aa1939b8 100644
--- a/modules/reasoning.py
+++ b/modules/reasoning.py
@@ -72,10 +72,9 @@ def extract_reasoning(text, html_escaped=False):
if content_pos != -1:
content_start = content_pos + len(content_esc)
else:
- # Content tag expected but not yet present (e.g. partial
- # streaming) — suppress intermediate tags between end_tag
- # and content_tag so they don't leak as content.
- content_start = len(text)
+ # Content tag not present — fall back to content after
+ # end_tag (e.g. GPT-OSS tool calls skip the final channel).
+ content_start = end_pos + len(end_esc)
else:
content_start = end_pos + len(end_esc)
From f48a2b79d022a3f503085d6daeb3706b3b6dc2e0 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 24 Mar 2026 11:45:33 -0700
Subject: [PATCH 47/47] UI: Minor polish
---
css/main.css | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/css/main.css b/css/main.css
index d42bd6ae..009b7c0a 100644
--- a/css/main.css
+++ b/css/main.css
@@ -1420,7 +1420,7 @@ audio {
}
.dark .thinking-block {
- background-color: var(--selected-item-color-dark);
+ background-color: transparent;
border: 1px solid var(--input-border-color);
}