oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-04 14:17:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

llama.cpp: Add StreamingLLM (--streaming-llm)

Browse source
This commit is contained in:
oobabooga 2025-04-25 16:21:35 -07:00
parent d35818f4e1
commit 877cf44c08
3 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/llama_cpp_server.py
View file

@ -301,6 +301,8 @@ class LlamaServer:
cmd += ["--device-draft", shared.args.device_draft]
if shared.args.ctx_size_draft > 0:
cmd += ["--ctx-size-draft", str(shared.args.ctx_size_draft)]
if shared.args.streaming_llm:
cmd += ["--cache-reuse", "1"]
if shared.args.extra_flags:
# Clean up the input
extra_flags = shared.args.extra_flags.strip()