diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index a5e90c94..2af9aa8a 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -339,7 +339,6 @@ class LlamaServer: cmd = [ self.server_path, "--model", self.model_path, - "--ctx-size", str(shared.args.ctx_size), "--batch-size", str(shared.args.batch_size), "--ubatch-size", str(shared.args.ubatch_size), "--port", str(self.port), @@ -347,6 +346,9 @@ class LlamaServer: "--flash-attn", "on", ] + if shared.args.ctx_size > 0: + cmd += ["--ctx-size", str(shared.args.ctx_size)] + if shared.args.gpu_layers >= 0: cmd += ["--gpu-layers", str(shared.args.gpu_layers), "--fit", "off"] else: @@ -449,7 +451,8 @@ class LlamaServer: print() gpu_layers_str = "auto" if shared.args.gpu_layers < 0 else str(shared.args.gpu_layers) - logger.info(f"Using gpu_layers={gpu_layers_str} | ctx_size={shared.args.ctx_size} | cache_type={cache_type}") + ctx_size_str = "auto" if shared.args.ctx_size == 0 else str(shared.args.ctx_size) + logger.info(f"Using gpu_layers={gpu_layers_str} | ctx_size={ctx_size_str} | cache_type={cache_type}") # Start the server with pipes for output self.process = subprocess.Popen( cmd, diff --git a/modules/models.py b/modules/models.py index 8c0f1c37..2dd69c63 100644 --- a/modules/models.py +++ b/modules/models.py @@ -54,7 +54,8 @@ def load_model(model_name, loader=None): shared.settings.update({k: v for k, v in metadata.items() if k in shared.settings}) if loader.lower().startswith('exllama') or loader.lower().startswith('tensorrt') or loader == 'llama.cpp': - shared.settings['truncation_length'] = shared.args.ctx_size + if shared.args.ctx_size > 0: + shared.settings['truncation_length'] = shared.args.ctx_size shared.is_multimodal = False if loader.lower() in ('exllamav3', 'llama.cpp') and hasattr(model, 'is_multimodal'): diff --git a/modules/models_settings.py b/modules/models_settings.py index 1ef436e0..1d784a87 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -418,7 +418,7 @@ def update_gpu_layers_and_vram(loader, model, gpu_layers, ctx_size, cache_type): Compute the estimated VRAM usage for the given GPU layers and return an HTML string for the UI display. """ - if loader != 'llama.cpp' or model in ["None", None] or not model.endswith(".gguf") or gpu_layers < 0: + if loader != 'llama.cpp' or model in ["None", None] or not model.endswith(".gguf") or gpu_layers < 0 or ctx_size == 0: return "
Estimated VRAM to load the model:
" vram_usage = estimate_vram(model, gpu_layers, ctx_size, cache_type) diff --git a/modules/shared.py b/modules/shared.py index 787f04d9..c80db298 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -71,7 +71,7 @@ group.add_argument('--loader', type=str, help='Choose the model loader manually, # Cache group = parser.add_argument_group('Context and cache') -group.add_argument('--ctx-size', '--n_ctx', '--max_seq_len', type=int, default=8192, metavar='N', help='Context size in tokens.') +group.add_argument('--ctx-size', '--n_ctx', '--max_seq_len', type=int, default=8192, metavar='N', help='Context size in tokens. llama.cpp: 0 = auto if gpu-layers is also -1.') group.add_argument('--cache-type', '--cache_type', type=str, default='fp16', metavar='N', help='KV cache type; valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8 (can specify k_bits and v_bits separately, e.g. q4_q8).') # Speculative decoding @@ -88,7 +88,7 @@ group.add_argument('--spec-ngram-min-hits', type=int, default=1, help='Minimum n # llama.cpp group = parser.add_argument_group('llama.cpp') -group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=-1, metavar='N', help='Number of layers to offload to the GPU. Set to -1 for auto mode, where llama.cpp decides via --fit.') +group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=-1, metavar='N', help='Number of layers to offload to the GPU. -1 = auto.') group.add_argument('--cpu-moe', action='store_true', help='Move the experts to the CPU (for MoE models).') group.add_argument('--mmproj', type=str, default=None, help='Path to the mmproj file for vision models.') group.add_argument('--streaming-llm', action='store_true', help='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 430123eb..de9901e3 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -41,8 +41,8 @@ def create_ui(): gr.Markdown("## Main options") with gr.Row(): with gr.Column(): - shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=-1, maximum=get_initial_gpu_layers_max(), step=1, value=shared.args.gpu_layers, info='Number of layers to offload to the GPU. -1 = auto (llama.cpp decides via --fit).') - shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=256, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. Common values: 4096, 8192, 16384, 32768, 65536, 131072.') + shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=-1, maximum=get_initial_gpu_layers_max(), step=1, value=shared.args.gpu_layers, info='Number of layers to offload to the GPU. -1 = auto.') + shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=0, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. llama.cpp: 0 = auto if gpu-layers is also -1. Common values: 4096, 8192, 16384, 32768, 65536, 131072.') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') shared.gradio['attn_implementation'] = gr.Dropdown(label="attn-implementation", choices=['sdpa', 'eager', 'flash_attention_2'], value=shared.args.attn_implementation, info='Attention implementation.') shared.gradio['cache_type'] = gr.Dropdown(label="cache-type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py index dc07d291..23882084 100644 --- a/modules/ui_parameters.py +++ b/modules/ui_parameters.py @@ -128,7 +128,7 @@ def create_event_handlers(): def get_truncation_length(): - if 'ctx_size' in shared.provided_arguments or shared.args.ctx_size != shared.args_defaults.ctx_size: + if shared.args.ctx_size > 0 and ('ctx_size' in shared.provided_arguments or shared.args.ctx_size != shared.args_defaults.ctx_size): return shared.args.ctx_size else: return shared.settings['truncation_length']