oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-06 07:03:37 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add ik_llama.cpp support via --ik flag

Browse source
This commit is contained in:
oobabooga 2026-03-26 06:49:39 -07:00
parent e154140021
commit 4cbea02ed4
2 changed files with 38 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
modules/shared.py
View file

@ -110,6 +110,7 @@ group.add_argument('--numa', action='store_true', help='Activate NUMA task alloc
group.add_argument('--parallel', type=int, default=1, help='Number of parallel request slots. The context size is divided equally among slots. For example, to have 4 slots with 8192 context each, set ctx_size to 32768.')
group.add_argument('--fit-target', type=str, default='512', help='Target VRAM margin per device for auto GPU layers, comma-separated list of values in MiB. A single value is broadcast across all devices.')
group.add_argument('--extra-flags', type=str, default=None, help='Extra flags to pass to llama-server. Example: "--jinja --rpc 192.168.1.100:50052"')
group.add_argument('--ik', action='store_true', help='Use ik_llama.cpp instead of upstream llama.cpp. To install: build ik_llama.cpp, then delete all files inside <venv>/lib/pythonX.Y/site-packages/llama_cpp_binaries/bin/ and copy or symlink the ik_llama.cpp build outputs into that folder.')
# Transformers/Accelerate
group = parser.add_argument_group('Transformers/Accelerate')