oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-06 23:23:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

llama.cpp: Document the --device-draft syntax

Browse source
This commit is contained in:
oobabooga 2025-04-24 09:20:11 -07:00
parent f1b64df8dd
commit 93fd4ad25d
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/shared.py
View file

@ -134,7 +134,7 @@ group = parser.add_argument_group('Speculative decoding')
group.add_argument('--model-draft', type=str, default=None, help='Path to the draft model for speculative decoding.')
group.add_argument('--draft-max', type=int, default=4, help='Number of tokens to draft for speculative decoding.')
group.add_argument('--gpu-layers-draft', type=int, default=0, help='Number of layers to offload to the GPU for the draft model.')
group.add_argument('--device-draft', type=str, default=None, help='Comma-separated list of devices to use for offloading the draft model.')
group.add_argument('--device-draft', type=str, default=None, help='Comma-separated list of devices to use for offloading the draft model. Example: CUDA0,CUDA1')
group.add_argument('--ctx-size-draft', type=int, default=0, help='Size of the prompt context for the draft model. If 0, uses the same as the main model.')
# ExLlamaV2