oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-04 14:17:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

New llama.cpp loader (#6846)

Browse source
This commit is contained in:
oobabooga 2025-04-18 09:59:37 -03:00 • committed by GitHub
parent 5c2f8d828e
commit ae54d8faaa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 471 additions and 999 deletions
Download patch file Download diff file Expand all files Collapse all files

4
modules/models_settings.py
View file

@ -48,7 +48,7 @@ def get_model_metadata(model):
)
# GGUF metadata
if model_settings['loader'] in ['llama.cpp', 'llamacpp_HF']:
if model_settings['loader'] == 'llama.cpp':
path = Path(f'{shared.args.model_dir}/{model}')
if path.is_file():
model_file = path
@ -163,8 +163,6 @@ def infer_loader(model_name, model_settings, hf_quant_method=None):
path_to_model = Path(f'{shared.args.model_dir}/{model_name}')
if not path_to_model.exists():
loader = None
elif len(list(path_to_model.glob('*.gguf'))) > 0 and path_to_model.is_dir() and (path_to_model / 'tokenizer_config.json').exists():
loader = 'llamacpp_HF'
elif len(list(path_to_model.glob('*.gguf'))) > 0:
loader = 'llama.cpp'
elif re.match(r'.*\.gguf', model_name.lower()):