diff --git a/README.md b/README.md index 2f7fe4f5..907d8c38 100644 --- a/README.md +++ b/README.md @@ -235,9 +235,9 @@ List of command-line flags ```txt usage: server.py [-h] [--multi-user] [--model MODEL] [--lora LORA [LORA ...]] [--model-dir MODEL_DIR] [--lora-dir LORA_DIR] [--model-menu] [--settings SETTINGS] [--extensions EXTENSIONS [EXTENSIONS ...]] [--verbose] [--idle-timeout IDLE_TIMEOUT] [--loader LOADER] [--cpu] [--cpu-memory CPU_MEMORY] [--disk] [--disk-cache-dir DISK_CACHE_DIR] - [--load-in-8bit] [--bf16] [--no-cache] [--trust-remote-code] [--force-safetensors] [--no_use_fast] [--attn-implementation IMPLEMENTATION] [--torch-compile] [--load-in-4bit] - [--use_double_quant] [--compute_dtype COMPUTE_DTYPE] [--quant_type QUANT_TYPE] [--flash-attn] [--threads THREADS] [--threads-batch THREADS_BATCH] [--batch-size BATCH_SIZE] [--no-mmap] - [--mlock] [--gpu-layers N] [--tensor-split TENSOR_SPLIT] [--numa] [--no-kv-offload] [--row-split] [--extra-flags EXTRA_FLAGS] [--streaming-llm] [--ctx-size N] [--cache-type N] + [--load-in-8bit] [--bf16] [--no-cache] [--trust-remote-code] [--force-safetensors] [--no_use_fast] [--attn-implementation IMPLEMENTATION] [--load-in-4bit] [--use_double_quant] + [--compute_dtype COMPUTE_DTYPE] [--quant_type QUANT_TYPE] [--flash-attn] [--threads THREADS] [--threads-batch THREADS_BATCH] [--batch-size BATCH_SIZE] [--no-mmap] [--mlock] + [--gpu-layers N] [--tensor-split TENSOR_SPLIT] [--numa] [--no-kv-offload] [--row-split] [--extra-flags EXTRA_FLAGS] [--streaming-llm] [--ctx-size N] [--cache-type N] [--model-draft MODEL_DRAFT] [--draft-max DRAFT_MAX] [--gpu-layers-draft GPU_LAYERS_DRAFT] [--device-draft DEVICE_DRAFT] [--ctx-size-draft CTX_SIZE_DRAFT] [--gpu-split GPU_SPLIT] [--autosplit] [--cfg-cache] [--no_flash_attn] [--no_xformers] [--no_sdpa] [--num_experts_per_token N] [--enable_tp] [--cpp-runner] [--deepspeed] [--nvme-offload-dir NVME_OFFLOAD_DIR] [--local_rank LOCAL_RANK] [--alpha_value ALPHA_VALUE] [--rope_freq_base ROPE_FREQ_BASE] [--compress_pos_emb COMPRESS_POS_EMB] [--listen] [--listen-port LISTEN_PORT] @@ -278,8 +278,7 @@ Transformers/Accelerate: --trust-remote-code Set trust_remote_code=True while loading the model. Necessary for some models. --force-safetensors Set use_safetensors=True while loading the model. This prevents arbitrary code execution. --no_use_fast Set use_fast=False while loading the tokenizer (it's True by default). Use this if you have any problems related to use_fast. - --attn-implementation IMPLEMENTATION Attention implementation. Valid options: eager, sdpa, flash_attention_2. - --torch-compile Compile the model with torch.compile for improved performance. + --attn-implementation IMPLEMENTATION Attention implementation. Valid options: sdpa, eager, flash_attention_2. bitsandbytes 4-bit: --load-in-4bit Load the model with 4-bit precision (using bitsandbytes). diff --git a/modules/loaders.py b/modules/loaders.py index 2b2c6b78..f515aeca 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -39,7 +39,6 @@ loaders_and_params = OrderedDict({ 'quant_type', 'load_in_8bit', 'load_in_4bit', - 'torch_compile', 'attn_implementation', 'cpu', 'disk', diff --git a/modules/shared.py b/modules/shared.py index d368062b..5e3e11c0 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -62,7 +62,6 @@ group.add_argument('--trust-remote-code', action='store_true', help='Set trust_r group.add_argument('--force-safetensors', action='store_true', help='Set use_safetensors=True while loading the model. This prevents arbitrary code execution.') group.add_argument('--no_use_fast', action='store_true', help='Set use_fast=False while loading the tokenizer (it\'s True by default). Use this if you have any problems related to use_fast.') group.add_argument('--attn-implementation', type=str, default='sdpa', metavar="IMPLEMENTATION", help='Attention implementation. Valid options: sdpa, eager, flash_attention_2.') -group.add_argument('--torch-compile', action='store_true', help='Compile the model with torch.compile for improved performance.') # bitsandbytes 4-bit group = parser.add_argument_group('bitsandbytes 4-bit') diff --git a/modules/transformers_loader.py b/modules/transformers_loader.py index e4163b6d..2f7367a4 100644 --- a/modules/transformers_loader.py +++ b/modules/transformers_loader.py @@ -258,9 +258,6 @@ def load_model_HF(model_name): print() model = LoaderClass.from_pretrained(path_to_model, **params) - if shared.args.torch_compile: - model = torch.compile(model) - return model diff --git a/modules/ui.py b/modules/ui.py index 9d6777f0..98acc038 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -142,7 +142,6 @@ def list_model_elements(): 'num_experts_per_token', 'load_in_8bit', 'load_in_4bit', - 'torch_compile', 'flash_attn', 'attn_implementation', 'cpu', diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 2018a943..ba7c529e 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -52,7 +52,6 @@ def create_ui(): shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming-llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) shared.gradio['load_in_4bit'] = gr.Checkbox(label="load-in-4bit", value=shared.args.load_in_4bit) - shared.gradio['torch_compile'] = gr.Checkbox(label="torch-compile", value=shared.args.torch_compile, info='Compile the model with torch.compile for improved performance.') shared.gradio['use_double_quant'] = gr.Checkbox(label="use_double_quant", value=shared.args.use_double_quant, info='Used by load-in-4bit.') shared.gradio['autosplit'] = gr.Checkbox(label="autosplit", value=shared.args.autosplit, info='Automatically split the model tensors across the available GPUs.') shared.gradio['enable_tp'] = gr.Checkbox(label="enable_tp", value=shared.args.enable_tp, info='Enable Tensor Parallelism (TP).') @@ -131,7 +130,7 @@ def create_ui(): def create_event_handlers(): mu = shared.args.multi_user if mu: - return + return shared.gradio['loader'].change(loaders.make_loader_params_visible, gradio('loader'), gradio(loaders.get_all_params()), show_progress=False)