oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-06 07:03:37 +00:00
Code Issues Projects Releases Packages Wiki Activity

llama.cpp: Use -1 instead of 0 for auto gpu_layers

Browse source
This commit is contained in:
oobabooga 2026-03-04 19:21:45 -08:00
parent d45aa6606a
commit fbfcd59fe0
4 changed files with 8 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/shared.py
View file

@ -88,7 +88,7 @@ group.add_argument('--spec-ngram-min-hits', type=int, default=1, help='Minimum n
# llama.cpp
group = parser.add_argument_group('llama.cpp')
group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=0, metavar='N', help='Number of layers to offload to the GPU. 0 means auto (llama.cpp decides via --fit).')
group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=-1, metavar='N', help='Number of layers to offload to the GPU. Set to -1 for auto mode, where llama.cpp decides via --fit.')
group.add_argument('--cpu-moe', action='store_true', help='Move the experts to the CPU (for MoE models).')
group.add_argument('--mmproj', type=str, default=None, help='Path to the mmproj file for vision models.')
group.add_argument('--streaming-llm', action='store_true', help='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.')