oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-04 14:17:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

llama.cpp: Add speculative decoding (#6891)

Browse source
This commit is contained in:
oobabooga 2025-04-23 20:10:16 -03:00 • committed by GitHub
parent 9424ba17c8
commit e99c20bcb0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 61 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

9
modules/shared.py
View file

@ -13,6 +13,7 @@ from modules.logging_colors import logger
model = None
tokenizer = None
model_name = 'None'
draft_model_name = 'None'
is_seq2seq = False
model_dirty_from_training = False
lora_names = []
@ -127,6 +128,14 @@ group.add_argument('--numa', action='store_true', help='Activate NUMA task alloc
group.add_argument('--no-kv-offload', action='store_true', help='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.')
group.add_argument('--row-split', action='store_true', help='Split the model by rows across GPUs. This may improve multi-gpu performance.')
# Speculative decoding
group = parser.add_argument_group('Speculative decoding')
group.add_argument('--model-draft', type=str, default=None, help='Path to the draft model for speculative decoding.')
group.add_argument('--draft-max', type=int, default=4, help='Number of tokens to draft for speculative decoding.')
group.add_argument('--gpu-layers-draft', type=int, default=0, help='Number of layers to offload to the GPU for the draft model.')
group.add_argument('--device-draft', type=str, default=None, help='Comma-separated list of devices to use for offloading the draft model.')
group.add_argument('--ctx-size-draft', type=int, default=0, help='Size of the prompt context for the draft model. If 0, uses the same as the main model.')
# ExLlamaV2
group = parser.add_argument_group('ExLlamaV2')
group.add_argument('--gpu-split', type=str, help='Comma-separated list of VRAM (in GB) to use per GPU device for model layers. Example: 20,7,7.')