oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2025-12-06 07:12:10 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make llama.cpp the default loader

Browse source
This commit is contained in:
oobabooga 2025-04-21 16:36:35 -07:00
parent 86c3ed3218
commit 15989c2ed8
2 changed files with 18 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
modules/loaders.py
View file

@ -4,6 +4,23 @@ from collections import OrderedDict
import gradio as gr import gradio as gr
loaders_and_params = OrderedDict({ loaders_and_params = OrderedDict({
'llama.cpp': [
'n_gpu_layers',
'threads',
'threads_batch',
'batch_size',
'n_ctx',
'cache_type',
'tensor_split',
'rope_freq_base',
'compress_pos_emb',
'flash_attn',
'row_split',
'no_kv_offload',
'no_mmap',
'mlock',
'numa',
],
'Transformers': [ 'Transformers': [
'gpu_split', 'gpu_split',
'cpu_memory', 'cpu_memory',
@ -23,23 +40,6 @@ loaders_and_params = OrderedDict({
'trust_remote_code', 'trust_remote_code',
'no_use_fast', 'no_use_fast',
], ],
'llama.cpp': [
'n_gpu_layers',
'threads',
'threads_batch',
'batch_size',
'n_ctx',
'cache_type',
'tensor_split',
'rope_freq_base',
'compress_pos_emb',
'flash_attn',
'row_split',
'no_kv_offload',
'no_mmap',
'mlock',
'numa',
],
'ExLlamav3_HF': [ 'ExLlamav3_HF': [
'max_seq_len', 'max_seq_len',
'gpu_split', 'gpu_split',

2
server.py
View file

@ -85,7 +85,7 @@ def create_interface():
# Force some events to be triggered on page load # Force some events to be triggered on page load
shared.persistent_interface_state.update({ shared.persistent_interface_state.update({
'loader': shared.args.loader or 'Transformers', 'loader': shared.args.loader or 'llama.cpp',
'mode': shared.settings['mode'] if shared.settings['mode'] == 'instruct' else gr.update(), 'mode': shared.settings['mode'] if shared.settings['mode'] == 'instruct' else gr.update(),
'character_menu': shared.args.character or shared.settings['character'], 'character_menu': shared.args.character or shared.settings['character'],
'instruction_template_str': shared.settings['instruction_template_str'], 'instruction_template_str': shared.settings['instruction_template_str'],