oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-09 16:43:46 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add slider for --ubatch-size for llama.cpp loader, change defaults for better MoE performance (#7316)

Browse source
This commit is contained in:
GodEmperor785 2025-11-21 20:56:02 +01:00 • committed by GitHub
parent 8f0048663d
commit 400bb0694b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/ui_model_menu.py
View file

@ -84,6 +84,7 @@ def create_ui():
shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=256, value=shared.args.threads)
shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch)
shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size)
shared.gradio['ubatch_size'] = gr.Slider(label="ubatch_size", minimum=1, maximum=4096, step=1, value=shared.args.ubatch_size)
shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='List of proportions to split the model across multiple GPUs. Example: 60,40')
shared.gradio['extra_flags'] = gr.Textbox(label='extra-flags', info='Additional flags to pass to llama-server. Format: "flag1=value1,flag2,flag3=value3". Example: "override-tensor=exps=CPU"', value=shared.args.extra_flags)
shared.gradio['cpu_memory'] = gr.Number(label="Maximum CPU memory in GiB. Use this for CPU offloading.", value=shared.args.cpu_memory)