From c12a53c998ce39ec762b9f7895861f1d94c2d827 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 1 May 2025 19:46:56 -0700 Subject: [PATCH 01/90] Use turboderp's exllamav2 wheels --- requirements/full/requirements.txt | 6 +++--- requirements/full/requirements_amd.txt | 4 ++-- requirements/full/requirements_amd_noavx2.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 2 +- requirements/full/requirements_apple_silicon.txt | 2 +- requirements/full/requirements_noavx2.txt | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index 6f265eba..c0ace41b 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -34,8 +34,8 @@ https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_c https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" https://github.com/oobabooga/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu124torch2.6.0cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt index c8e75ee7..91582eb3 100644 --- a/requirements/full/requirements_amd.txt +++ b/requirements/full/requirements_amd.txt @@ -30,5 +30,5 @@ tiktoken # AMD wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+rocm6.1.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.1.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt index e54d6d9c..7b86050e 100644 --- a/requirements/full/requirements_amd_noavx2.txt +++ b/requirements/full/requirements_amd_noavx2.txt @@ -30,5 +30,5 @@ tiktoken # AMD wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+rocm6.1.2avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.1.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index d714ea3d..cc747edb 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -32,4 +32,4 @@ tiktoken https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6-py3-none-any.whl -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index 89f4f576..67b3260e 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -33,4 +33,4 @@ https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_c https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6-py3-none-any.whl -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index e216c9cd..3575d352 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -34,8 +34,8 @@ https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_c https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" https://github.com/oobabooga/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu124torch2.6.0cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" From f8aaf3c23a793b60ce7452213304acb493be98af Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 1 May 2025 19:50:46 -0700 Subject: [PATCH 02/90] Use ROCm 6.2.4 on AMD --- README.md | 2 +- one_click.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4b541b9e..3280186c 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ conda activate textgen |--------|---------|---------| | Linux/WSL | NVIDIA | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu124` | | Linux/WSL | CPU only | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cpu` | -| Linux | AMD | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/rocm6.1` | +| Linux | AMD | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/rocm6.2.4` | | MacOS + MPS | Any | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0` | | Windows | NVIDIA | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu124` | | Windows | CPU only | `pip3 install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0` | diff --git a/one_click.py b/one_click.py index 065afd99..cb16b813 100644 --- a/one_click.py +++ b/one_click.py @@ -222,7 +222,7 @@ def update_pytorch_and_python(): if "+cu" in torver: install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124" elif "+rocm" in torver: - install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.1" + install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4" elif "+cpu" in torver: install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cpu" elif "+cxx11" in torver: @@ -273,7 +273,7 @@ def install_webui(): "What is your GPU?", { 'A': 'NVIDIA - CUDA 12.4', - 'B': 'AMD - Linux/macOS only, requires ROCm 6.1', + 'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4', 'C': 'Apple M Series', 'D': 'Intel Arc (beta)', 'N': 'CPU mode' @@ -314,7 +314,7 @@ def install_webui(): if selected_gpu == "NVIDIA": install_pytorch += "--index-url https://download.pytorch.org/whl/cu124" elif selected_gpu == "AMD": - install_pytorch += "--index-url https://download.pytorch.org/whl/rocm6.1" + install_pytorch += "--index-url https://download.pytorch.org/whl/rocm6.2.4" elif selected_gpu in ["APPLE", "NONE"]: install_pytorch += "--index-url https://download.pytorch.org/whl/cpu" elif selected_gpu == "INTEL": From d5c407cf35453ba2d06eea942942ff11cdc7993b Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 1 May 2025 20:05:36 -0700 Subject: [PATCH 03/90] Use Vulkan instead of ROCm for llama.cpp on AMD --- requirements/full/requirements_amd.txt | 3 ++- requirements/full/requirements_amd_noavx2.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt index 91582eb3..24eeee6a 100644 --- a/requirements/full/requirements_amd.txt +++ b/requirements/full/requirements_amd.txt @@ -29,6 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+rocm6.1.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt index 7b86050e..99716f3c 100644 --- a/requirements/full/requirements_amd_noavx2.txt +++ b/requirements/full/requirements_amd_noavx2.txt @@ -29,6 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+rocm6.1.2avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" From 9e3867dc8358baf153d6f7c182496dad158696a4 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 09:36:15 -0700 Subject: [PATCH 04/90] llama.cpp: Fix manual random seeds --- modules/text_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index 8d091868..b9bf9b16 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -480,7 +480,7 @@ def generate_reply_custom(question, original_question, state, stopping_strings=N For models that do not use the transformers library for sampling """ - seed = set_manual_seed(state['seed']) + state['seed'] = set_manual_seed(state['seed']) t0 = time.time() reply = '' try: From 3f26b0408bd02f500acc8c090a7e50ee286051b5 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 16:17:22 -0700 Subject: [PATCH 05/90] Fix after 9e3867dc8358baf153d6f7c182496dad158696a4 --- modules/text_generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index b9bf9b16..8fd65dc4 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -500,7 +500,7 @@ def generate_reply_custom(question, original_question, state, stopping_strings=N t1 = time.time() original_tokens = len(encode(original_question)[0]) new_tokens = len(encode(original_question + reply)[0]) - original_tokens - print(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {seed})') + print(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {state["seed"]})') return From 905afced1c8339833280de254cd597b389a3dade Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 16:32:22 -0700 Subject: [PATCH 06/90] Add a --portable flag to hide things in portable mode --- modules/presets.py | 9 ++++++++- modules/shared.py | 1 + modules/ui_model_menu.py | 17 +++++++++++------ modules/ui_parameters.py | 2 +- server.py | 5 +++-- start_linux.sh | 2 +- start_macos.sh | 2 +- start_windows.bat | 2 +- 8 files changed, 27 insertions(+), 13 deletions(-) diff --git a/modules/presets.py b/modules/presets.py index a432bf52..50d0f985 100644 --- a/modules/presets.py +++ b/modules/presets.py @@ -11,7 +11,7 @@ from modules.logging_colors import logger def default_preset(): - return { + result = { 'temperature': 1, 'dynatemp_low': 1, 'dynatemp_high': 1, @@ -50,6 +50,13 @@ def default_preset(): 'dry_sequence_breakers': '"\\n", ":", "\\"", "*"', } + if shared.args.portable: + samplers = result['sampler_priority'].split('\n') + samplers = [sampler for sampler in samplers if sampler in ["dry", "top_k", "typ_p", "top_p", "min_p", "xtc", "temperature", "repetition_penalty"]] + result['sampler_priority'] = '\n'.join(samplers) + + return result + def presets_params(): return [k for k in default_preset()] diff --git a/modules/shared.py b/modules/shared.py index fb10c014..39b0bdaa 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -190,6 +190,7 @@ group.add_argument('--ssl-keyfile', type=str, help='The path to the SSL certific group.add_argument('--ssl-certfile', type=str, help='The path to the SSL certificate cert file.', default=None) group.add_argument('--subpath', type=str, help='Customize the subpath for gradio, use with reverse proxy') group.add_argument('--old-colors', action='store_true', help='Use the legacy Gradio colors, before the December/2024 update.') +group.add_argument('--portable', action='store_true', help='Hide features not available in portable mode like training.') # API group = parser.add_argument_group('API') diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index d13bcff7..4a49d209 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -35,14 +35,17 @@ def create_ui(): shared.gradio['save_model_settings'] = gr.Button("Save settings", elem_classes='refresh-button', interactive=not mu) with gr.Column(): - with gr.Row(): - shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)', elem_classes='slim-dropdown', interactive=not mu) - ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) - shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) + if shared.args.portable: + pass + else: + with gr.Row(): + shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)', elem_classes='slim-dropdown', interactive=not mu) + ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) + shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) with gr.Row(): with gr.Column(): - shared.gradio['loader'] = gr.Dropdown(label="Model loader", choices=loaders.loaders_and_params.keys(), value=None) + shared.gradio['loader'] = gr.Dropdown(label="Model loader", choices=loaders.loaders_and_params.keys() if not shared.args.portable else ['llama.cpp'], value=None) with gr.Blocks(): with gr.Row(): with gr.Column(): @@ -150,7 +153,9 @@ def create_event_handlers(): ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( save_model_settings, gradio('model_menu', 'interface_state'), gradio('model_status'), show_progress=False) - shared.gradio['lora_menu_apply'].click(load_lora_wrapper, gradio('lora_menu'), gradio('model_status'), show_progress=False) + if not shared.args.portable: + shared.gradio['lora_menu_apply'].click(load_lora_wrapper, gradio('lora_menu'), gradio('model_status'), show_progress=False) + shared.gradio['download_model_button'].click(download_model_wrapper, gradio('custom_model_menu', 'download_specific_file'), gradio('model_status'), show_progress=True) shared.gradio['get_file_list'].click(partial(download_model_wrapper, return_links=True), gradio('custom_model_menu', 'download_specific_file'), gradio('model_status'), show_progress=True) shared.gradio['autoload_model'].change(lambda x: gr.update(visible=not x), gradio('autoload_model'), gradio('load_model')) diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py index 3f609d71..071b30b6 100644 --- a/modules/ui_parameters.py +++ b/modules/ui_parameters.py @@ -21,7 +21,7 @@ def create_ui(default_preset): shared.gradio['random_preset'] = gr.Button('🎲', elem_classes='refresh-button') with gr.Column(): - shared.gradio['filter_by_loader'] = gr.Dropdown(label="Filter by loader", choices=["All"] + list(loaders.loaders_and_params.keys()), value="All", elem_classes='slim-dropdown') + shared.gradio['filter_by_loader'] = gr.Dropdown(label="Filter by loader", choices=["All"] + list(loaders.loaders_and_params.keys()) if not shared.args.portable else ['llama.cpp'], value="All", elem_classes='slim-dropdown') with gr.Row(): with gr.Column(): diff --git a/server.py b/server.py index 169578a5..b0b9e633 100644 --- a/server.py +++ b/server.py @@ -90,7 +90,7 @@ def create_interface(): 'instruction_template_str': shared.settings['instruction_template_str'], 'prompt_menu-default': shared.settings['prompt-default'], 'prompt_menu-notebook': shared.settings['prompt-notebook'], - 'filter_by_loader': shared.args.loader or 'All' + 'filter_by_loader': (shared.args.loader or 'All') if not shared.args.portable else 'llama.cpp' }) if Path("user_data/cache/pfp_character.png").exists(): @@ -127,7 +127,8 @@ def create_interface(): ui_parameters.create_ui(shared.settings['preset']) # Parameters tab ui_model_menu.create_ui() # Model tab - training.create_ui() # Training tab + if not shared.args.portable: + training.create_ui() # Training tab ui_session.create_ui() # Session tab # Generation events diff --git a/start_linux.sh b/start_linux.sh index 00082f07..c74f1272 100755 --- a/start_linux.sh +++ b/start_linux.sh @@ -4,7 +4,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" # Portable install case if [ -d "portable_env" ]; then - ./portable_env/bin/python3 server.py --api --auto-launch "$@" + ./portable_env/bin/python3 server.py --portable --api --auto-launch "$@" exit $? fi diff --git a/start_macos.sh b/start_macos.sh index 628f59cc..7a060ba6 100755 --- a/start_macos.sh +++ b/start_macos.sh @@ -4,7 +4,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" # Portable install case if [ -d "portable_env" ]; then - ./portable_env/bin/python3 server.py --api --auto-launch --api-port 5005 "$@" + ./portable_env/bin/python3 server.py --portable --api --auto-launch --api-port 5005 "$@" exit $? fi diff --git a/start_windows.bat b/start_windows.bat index 451b85e0..1616ee27 100755 --- a/start_windows.bat +++ b/start_windows.bat @@ -5,7 +5,7 @@ cd /D "%~dp0" @rem Portable install case if exist "portable_env" ( - .\portable_env\python.exe server.py --api --auto-launch %* + .\portable_env\python.exe server.py --portable --api --auto-launch %* exit /b %errorlevel% ) From 4cea720da8cca27cbb5e8ac560019a55e6afb73a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 16:38:28 -0700 Subject: [PATCH 07/90] UI: Remove the "Autoload the model" feature --- modules/shared.py | 1 - modules/ui_model_menu.py | 9 ++------- user_data/settings-template.yaml | 1 - 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 39b0bdaa..cfedb992 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -60,7 +60,6 @@ settings = { 'custom_stopping_strings': '', 'custom_token_bans': '', 'negative_prompt': '', - 'autoload_model': False, 'dark_theme': True, 'default_extensions': [], 'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}", diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 4a49d209..9361ef91 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -30,7 +30,7 @@ def create_ui(): with gr.Row(): shared.gradio['model_menu'] = gr.Dropdown(choices=utils.get_available_models(), value=lambda: shared.model_name, label='Model', elem_classes='slim-dropdown', interactive=not mu) ui.create_refresh_button(shared.gradio['model_menu'], lambda: None, lambda: {'choices': utils.get_available_models()}, 'refresh-button', interactive=not mu) - shared.gradio['load_model'] = gr.Button("Load", visible=not shared.settings['autoload_model'], elem_classes='refresh-button', interactive=not mu) + shared.gradio['load_model'] = gr.Button("Load", elem_classes='refresh-button', interactive=not mu) shared.gradio['unload_model'] = gr.Button("Unload", elem_classes='refresh-button', interactive=not mu) shared.gradio['save_model_settings'] = gr.Button("Save settings", elem_classes='refresh-button', interactive=not mu) @@ -108,9 +108,6 @@ def create_ui(): shared.gradio['ctx_size_draft'] = gr.Number(label="ctx-size-draft", precision=0, step=256, value=shared.args.ctx_size_draft, info='Size of the prompt context for the draft model. If 0, uses the same as the main model.') with gr.Column(): - with gr.Row(): - shared.gradio['autoload_model'] = gr.Checkbox(value=shared.settings['autoload_model'], label='Autoload the model', info='Whether to load the model as soon as it is selected in the Model dropdown.', interactive=not mu) - with gr.Tab("Download"): shared.gradio['custom_model_menu'] = gr.Textbox(label="Download model or LoRA", info="Enter the Hugging Face username/model path, for instance: facebook/galactica-125m. To specify a branch, add it at the end after a \":\" character like this: facebook/galactica-125m:main. To download a single file, enter its name in the second box.", interactive=not mu) shared.gradio['download_specific_file'] = gr.Textbox(placeholder="File name (for GGUF models)", show_label=False, max_lines=1, interactive=not mu) @@ -135,11 +132,10 @@ def create_event_handlers(): # In this event handler, the interface state is read and updated # with the model defaults (if any), and then the model is loaded - # unless "autoload_model" is unchecked shared.gradio['model_menu'].change( ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( handle_load_model_event_initial, gradio('model_menu', 'interface_state'), gradio(ui.list_interface_input_elements()) + gradio('interface_state'), show_progress=False).then( - load_model_wrapper, gradio('model_menu', 'loader', 'autoload_model'), gradio('model_status'), show_progress=True).success( + partial(load_model_wrapper, autoload=False), gradio('model_menu', 'loader'), gradio('model_status'), show_progress=True).success( handle_load_model_event_final, gradio('truncation_length', 'loader', 'interface_state'), gradio('truncation_length', 'filter_by_loader'), show_progress=False) shared.gradio['load_model'].click( @@ -158,7 +154,6 @@ def create_event_handlers(): shared.gradio['download_model_button'].click(download_model_wrapper, gradio('custom_model_menu', 'download_specific_file'), gradio('model_status'), show_progress=True) shared.gradio['get_file_list'].click(partial(download_model_wrapper, return_links=True), gradio('custom_model_menu', 'download_specific_file'), gradio('model_status'), show_progress=True) - shared.gradio['autoload_model'].change(lambda x: gr.update(visible=not x), gradio('autoload_model'), gradio('load_model')) shared.gradio['customized_template_submit'].click(save_instruction_template, gradio('model_menu', 'customized_template'), gradio('model_status'), show_progress=True) diff --git a/user_data/settings-template.yaml b/user_data/settings-template.yaml index 20896da3..ce0f77e1 100644 --- a/user_data/settings-template.yaml +++ b/user_data/settings-template.yaml @@ -31,7 +31,6 @@ seed: -1 custom_stopping_strings: '' custom_token_bans: '' negative_prompt: '' -autoload_model: false dark_theme: true default_extensions: [] instruction_template_str: |- From 3526b7923c9f5a3b3ba55056e445a660a03d2bc6 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 17:40:53 -0700 Subject: [PATCH 08/90] Remove extensions with requirements from portable builds --- .github/workflows/build-portable-release-cuda.yml | 2 ++ .github/workflows/build-portable-release-vulkan.yml | 2 ++ .github/workflows/build-portable-release.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/build-portable-release-cuda.yml b/.github/workflows/build-portable-release-cuda.yml index fb9e61b0..571cbac0 100644 --- a/.github/workflows/build-portable-release-cuda.yml +++ b/.github/workflows/build-portable-release-cuda.yml @@ -102,6 +102,8 @@ jobs: shell: bash run: | rm -rf .git cmd* update_wizard* Colab-TextGen-GPU.ipynb docker + allowed=("character_bias" "gallery" "openai" "sd_api_pictures") + find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf # Define common variables CUDA_VERSION="${{ matrix.cuda }}" diff --git a/.github/workflows/build-portable-release-vulkan.yml b/.github/workflows/build-portable-release-vulkan.yml index 8de29791..4e88d4d9 100644 --- a/.github/workflows/build-portable-release-vulkan.yml +++ b/.github/workflows/build-portable-release-vulkan.yml @@ -101,6 +101,8 @@ jobs: shell: bash run: | rm -rf .git cmd* update_wizard* Colab-TextGen-GPU.ipynb docker + allowed=("character_bias" "gallery" "openai" "sd_api_pictures") + find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf # Define common variables AVX_SUPPORT="${{ matrix.avx }}" diff --git a/.github/workflows/build-portable-release.yml b/.github/workflows/build-portable-release.yml index bdf96cec..6910ce2c 100644 --- a/.github/workflows/build-portable-release.yml +++ b/.github/workflows/build-portable-release.yml @@ -101,6 +101,8 @@ jobs: shell: bash run: | rm -rf .git cmd* update_wizard* Colab-TextGen-GPU.ipynb docker + allowed=("character_bias" "gallery" "openai" "sd_api_pictures") + find extensions/ -mindepth 1 -maxdepth 1 -type d | grep -v -E "$(printf '%s|' "${allowed[@]}" | sed 's/|$//')" | xargs rm -rf # Define common variables AVX_SUPPORT="${{ matrix.avx }}" From d08acb4af9c2a4f4d0f7fd97babb217c0890e1c8 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 20:50:52 -0700 Subject: [PATCH 09/90] UI: Rename enable_thinking -> Enable thinking --- modules/ui_parameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py index 071b30b6..733d0901 100644 --- a/modules/ui_parameters.py +++ b/modules/ui_parameters.py @@ -82,7 +82,7 @@ def create_ui(default_preset): shared.gradio['auto_max_new_tokens'] = gr.Checkbox(value=shared.settings['auto_max_new_tokens'], label='auto_max_new_tokens', info='Expand max_new_tokens to the available context length.') shared.gradio['ban_eos_token'] = gr.Checkbox(value=shared.settings['ban_eos_token'], label='Ban the eos_token', info='Forces the model to never end the generation prematurely.') shared.gradio['add_bos_token'] = gr.Checkbox(value=shared.settings['add_bos_token'], label='Add the bos_token to the beginning of prompts', info='Disabling this can make the replies more creative.') - shared.gradio['enable_thinking'] = gr.Checkbox(value=shared.settings['enable_thinking'], label='enable_thinking', info='Used by Qwen3 to toggle mode.') + shared.gradio['enable_thinking'] = gr.Checkbox(value=shared.settings['enable_thinking'], label='Enable thinking', info='Used by Qwen3 to toggle mode.') shared.gradio['skip_special_tokens'] = gr.Checkbox(value=shared.settings['skip_special_tokens'], label='Skip special tokens', info='Some specific models need this unset.') shared.gradio['stream'] = gr.Checkbox(value=shared.settings['stream'], label='Activate text streaming') shared.gradio['static_cache'] = gr.Checkbox(value=shared.settings['static_cache'], label='Static KV cache', info='Use a static cache for improved performance.') From b21bd8bb1e79466be945abfe417e92e52b63ec6f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 22:41:49 -0700 Subject: [PATCH 10/90] UI: Invert user/assistant message colors in instruct mode The goal is to make assistant messages more readable. --- css/html_instruct_style.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/css/html_instruct_style.css b/css/html_instruct_style.css index 4613b380..b98544a1 100644 --- a/css/html_instruct_style.css +++ b/css/html_instruct_style.css @@ -61,11 +61,11 @@ } .dark .chat .user-message { - background: transparent; + background: var(--light-gray); } .dark .chat .assistant-message { - background: var(--light-gray); + background: transparent; } .chat .user-message .text, From b71ef50e9d01c15a09c67b95e2032fed535c63ba Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 2 May 2025 23:45:58 -0700 Subject: [PATCH 11/90] UI: Add a min-height to prevent constant scrolling during chat streaming --- css/chat_style-Dark.css | 2 ++ css/chat_style-TheEncrypted777.css | 2 ++ css/chat_style-cai-chat.css | 1 + css/main.css | 5 +++++ modules/html_generator.py | 9 ++++++--- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/css/chat_style-Dark.css b/css/chat_style-Dark.css index 368a2a16..3b2bd385 100644 --- a/css/chat_style-Dark.css +++ b/css/chat_style-Dark.css @@ -1,5 +1,6 @@ .message { display: grid; + align-items: start; grid-template-columns: 60px minmax(0, 1fr); padding-bottom: 28px; font-size: 18px; @@ -102,6 +103,7 @@ @media screen and (width <= 688px) { .message { display: grid; + align-items: start; grid-template-columns: 60px minmax(0, 1fr); padding-bottom: 25px; font-size: 15px; diff --git a/css/chat_style-TheEncrypted777.css b/css/chat_style-TheEncrypted777.css index 6404f41d..25d26db8 100644 --- a/css/chat_style-TheEncrypted777.css +++ b/css/chat_style-TheEncrypted777.css @@ -2,6 +2,7 @@ .message { display: grid; + align-items: start; grid-template-columns: 60px minmax(0, 1fr); padding-bottom: 28px; font-size: 18px; @@ -100,6 +101,7 @@ @media screen and (width <= 688px) { .message { display: grid; + align-items: start; grid-template-columns: 60px minmax(0, 1fr); padding-bottom: 25px; font-size: 15px; diff --git a/css/chat_style-cai-chat.css b/css/chat_style-cai-chat.css index 93276bd3..223f6150 100644 --- a/css/chat_style-cai-chat.css +++ b/css/chat_style-cai-chat.css @@ -1,5 +1,6 @@ .message { display: grid; + align-items: start; grid-template-columns: 60px minmax(0, 1fr); padding-bottom: 2em; font-size: 15px; diff --git a/css/main.css b/css/main.css index d6e5ac83..cf0dfde7 100644 --- a/css/main.css +++ b/css/main.css @@ -403,6 +403,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { .chat-parent { height: calc(100dvh - 98px - var(--input-delta)); overflow: auto !important; + /* scroll-behavior: smooth; */ border-radius: 0 !important; margin-bottom: var(--input-delta) !important; } @@ -1382,3 +1383,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { 50% { opacity: 1; } 100% { opacity: 0.6; } } + +.streaming { + min-height: 70vh; +} diff --git a/modules/html_generator.py b/modules/html_generator.py index 67d15b6e..a6f5f930 100644 --- a/modules/html_generator.py +++ b/modules/html_generator.py @@ -365,8 +365,9 @@ def generate_instruct_html(history): f'' ) + streaming_class = " streaming" if i == len(history["visible"]) - 1 else "" output += ( - f'
' f'
' f'
{converted_visible[1]}
' @@ -414,8 +415,9 @@ def generate_cai_chat_html(history, name1, name2, style, character, reset_cache= f'
' ) + streaming_class = " streaming" if i == len(history["visible"]) - 1 else "" output += ( - f'
' f'
{img_bot}
' f'
' @@ -452,8 +454,9 @@ def generate_chat_html(history, name1, name2, reset_cache=False): f'
' ) + streaming_class = " streaming" if i == len(history["visible"]) - 1 else "" output += ( - f'
' f'
' f'
{converted_visible[1]}
' From ea60f14674a89d3a71e5504edacb8f64f148b57c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 3 May 2025 06:06:50 -0700 Subject: [PATCH 12/90] UI: Show the list of files if the user tries to download a GGUF repository --- modules/ui_model_menu.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 9361ef91..2c593df6 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -192,6 +192,26 @@ def load_lora_wrapper(selected_loras): def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), return_links=False, check=False): try: + # Handle direct GGUF URLs + if repo_id.startswith("https://") and ("huggingface.co" in repo_id) and (repo_id.endswith(".gguf") or repo_id.endswith(".gguf?download=true")): + try: + path = repo_id.split("huggingface.co/")[1] + + # Extract the repository ID (first two parts of the path) + parts = path.split("/") + if len(parts) >= 2: + extracted_repo_id = f"{parts[0]}/{parts[1]}" + + # Extract the filename (last part of the path) + filename = repo_id.split("/")[-1] + if "?download=true" in filename: + filename = filename.replace("?download=true", "") + + repo_id = extracted_repo_id + specific_file = filename + except: + pass + if repo_id == "": yield ("Please enter a model path") return @@ -205,6 +225,18 @@ def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), retur yield ("Getting the download links from Hugging Face") links, sha256, is_lora, is_llamacpp = downloader.get_download_links_from_huggingface(model, branch, text_only=False, specific_file=specific_file) + + # Check for multiple GGUF files + gguf_files = [link for link in links if link.lower().endswith('.gguf')] + if len(gguf_files) > 1 and not specific_file: + output = "Multiple GGUF files found. Please copy one of the following filenames to the 'File name' field:\n\n```\n" + for link in gguf_files: + output += f"{Path(link).name}\n" + + output += "```" + yield output + return + if return_links: output = "```\n" for link in links: From 4c2e3b168bc1751dbb3f1b222fdd749ad7a5d36e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 3 May 2025 06:51:20 -0700 Subject: [PATCH 13/90] llama.cpp: Add a retry mechanism when getting the logits (sometimes it fails) --- modules/llama_cpp_server.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index d9187db8..2ebeb560 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -210,14 +210,15 @@ class LlamaServer: pprint.PrettyPrinter(indent=4, sort_dicts=False).pprint(printable_payload) print() - response = self.session.post(url, json=payload) - result = response.json() + for retry in range(5): + response = self.session.post(url, json=payload) + result = response.json() - if "completion_probabilities" in result: - if use_samplers: - return result["completion_probabilities"][0]["top_probs"] - else: - return result["completion_probabilities"][0]["top_logprobs"] + if "completion_probabilities" in result: + if use_samplers: + return result["completion_probabilities"][0]["top_probs"] + else: + return result["completion_probabilities"][0]["top_logprobs"] else: raise Exception(f"Unexpected response format: 'completion_probabilities' not found in {result}") From 5f5569e9ac21ffdcb335b0557909ad102104fc8f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 06:20:24 -0700 Subject: [PATCH 14/90] Update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3280186c..8a7b2467 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Download from: https://github.com/oobabooga/text-generation-webui/releases To restart the web UI later, just run the same `start_` script. If you need to reinstall, delete the `installer_files` folder created during setup and run the script again. -You can use command-line flags, like `./start_linux.sh --help`, or add them to `user_data/CMD_FLAGS.txt` (such as `--api` to enable API use). To update the project, run `update_wizard_linux.sh`, `update_wizard_windows.bat`, `update_wizard_macos.sh`, or `update_wizard_wsl.bat`. +You can use command-line flags, like `./start_linux.sh --help`, or add them to `user_data/CMD_FLAGS.txt` (such as `--api` to enable API use). To update the project, run `update_wizard_linux.sh`, `update_wizard_windows.bat`, or `update_wizard_macos.sh`.
@@ -55,7 +55,7 @@ Setup details and information about installing manually The script uses Miniconda to set up a Conda environment in the `installer_files` folder. -If you ever need to install something manually in the `installer_files` environment, you can launch an interactive shell using the cmd script: `cmd_linux.sh`, `cmd_windows.bat`, `cmd_macos.sh`, or `cmd_wsl.bat`. +If you ever need to install something manually in the `installer_files` environment, you can launch an interactive shell using the cmd script: `cmd_linux.sh`, `cmd_windows.bat`, or `cmd_macos.sh`. * There is no need to run any of those scripts (`start_`, `update_wizard_`, or `cmd_`) as admin/root. * To install the requirements for extensions, you can use the `extensions_reqs` script for your OS. At the end, this script will install the main requirements for the project to make sure that they take precedence in case of version conflicts. From b7a5c7db8de89159144fadb59920045efc3fe544 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 07:14:42 -0700 Subject: [PATCH 15/90] llama.cpp: Handle short arguments in --extra-flags --- modules/llama_cpp_server.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 2ebeb560..7244001a 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -317,9 +317,15 @@ class LlamaServer: for flag_item in extra_flags.split(','): if '=' in flag_item: flag, value = flag_item.split('=', 1) - cmd += [f"--{flag}", value] + if len(flag) <= 3: + cmd += [f"-{flag}", value] + else: + cmd += [f"--{flag}", value] else: - cmd.append(f"--{flag_item}") + if len(flag_item) <= 3: + cmd.append(f"-{flag_item}") + else: + cmd.append(f"--{flag_item}") env = os.environ.copy() if os.name == 'posix': From 7853fb1c8d701bb8b720b3907bdc50017911d6a6 Mon Sep 17 00:00:00 2001 From: oobabooga Date: Sun, 4 May 2025 18:58:37 -0300 Subject: [PATCH 16/90] Optimize the Chat tab (#6948) --- css/main.css | 34 +++++++++++----------------------- js/main.js | 8 +------- modules/ui_chat.py | 2 +- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/css/main.css b/css/main.css index cf0dfde7..64e96ccc 100644 --- a/css/main.css +++ b/css/main.css @@ -389,7 +389,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { .chat { margin-left: auto; margin-right: auto; - min-height: var(--chat-height); + flex: 1; overflow-y: auto; display: flex; flex-direction: column; @@ -401,11 +401,9 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } .chat-parent { - height: calc(100dvh - 98px - var(--input-delta)); + flex: 1; overflow: auto !important; - /* scroll-behavior: smooth; */ border-radius: 0 !important; - margin-bottom: var(--input-delta) !important; } .chat-parent .prose { @@ -422,8 +420,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } .chat-parent.bigchat { - height: calc(100dvh - 98px - var(--input-delta)) !important; - margin-bottom: var(--input-delta) !important; + flex: 1; } .chat > .messages { @@ -604,8 +601,6 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } .chat-input-positioned { - position: absolute; - bottom: 0; max-width: 54rem; left: 50%; transform: translateX(-50%); @@ -790,7 +785,8 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } #chat-input-container { - min-width: 0 !important; + display: flex; + flex-direction: column; } #chat-input-container > .form { @@ -799,9 +795,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } #chat-input-row { - padding-bottom: 1.5em; - padding-left: 1rem; - padding-right: 1rem; + padding: 1rem; } #chat-input-row.bigchat { @@ -809,22 +803,16 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } #chat-col { - padding-bottom: 100px; + height: 100dvh; + display: flex; + flex-direction: column; + padding-bottom: 0; } @media screen and (width <= 924px) { #chat-col { - padding-bottom: 100px; + height: calc(100dvh - 132px); margin-top: 32px; - position: relative; /* Ensure positioning for the pseudo-element */ - } - - .chat-parent { - height: calc(100dvh - 98px - var(--input-delta) - 32px); - } - - .chat-parent.bigchat { - height: calc(100dvh - 98px - var(--input-delta) - 32px) !important; } } diff --git a/js/main.js b/js/main.js index 33b7d6bd..408815db 100644 --- a/js/main.js +++ b/js/main.js @@ -442,12 +442,6 @@ function updateCssProperties() { // Check if the chat container is visible if (chatContainer.clientHeight > 0) { - const chatContainerParentHeight = chatContainer.parentNode.clientHeight; - const newChatHeight = `${chatContainerParentHeight - chatInputHeight - 80}px`; - - document.documentElement.style.setProperty("--chat-height", newChatHeight); - document.documentElement.style.setProperty("--input-delta", `${chatInputHeight - 40}px`); - // Adjust scrollTop based on input height change if (chatInputHeight !== currentChatInputHeight) { const deltaHeight = chatInputHeight - currentChatInputHeight; @@ -720,7 +714,7 @@ function isMobile() { // Function to initialize sidebars function initializeSidebars() { const isOnMobile = isMobile(); - + if (isOnMobile) { // Mobile state: Hide sidebars and set closed states [pastChatsRow, chatControlsRow, headerBar].forEach(el => { diff --git a/modules/ui_chat.py b/modules/ui_chat.py index 0d588549..0856cfab 100644 --- a/modules/ui_chat.py +++ b/modules/ui_chat.py @@ -46,8 +46,8 @@ def create_ui(): with gr.Row(): with gr.Column(elem_id='chat-col'): - shared.gradio['html_display'] = gr.HTML(value=chat_html_wrapper({'internal': [], 'visible': []}, '', '', 'chat', 'cai-chat', '')['html'], visible=True) shared.gradio['display'] = gr.JSON(value={}, visible=False) # Hidden buffer + shared.gradio['html_display'] = gr.HTML(value=chat_html_wrapper({'internal': [], 'visible': []}, '', '', 'chat', 'cai-chat', '')['html'], visible=True) with gr.Row(elem_id="chat-input-row"): with gr.Column(scale=1, elem_id='gr-hover-container'): gr.HTML(value='
☰
', elem_id='gr-hover') From d1866219261c5fa1e9d8c0a9c6c380b965ca7cc7 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 15:19:46 -0700 Subject: [PATCH 17/90] UI: Fixes after previous commit --- css/main.css | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/css/main.css b/css/main.css index 64e96ccc..f76a2787 100644 --- a/css/main.css +++ b/css/main.css @@ -787,6 +787,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { #chat-input-container { display: flex; flex-direction: column; + min-width: 0 !important; } #chat-input-container > .form { @@ -807,12 +808,13 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { display: flex; flex-direction: column; padding-bottom: 0; + gap: 0; } @media screen and (width <= 924px) { #chat-col { - height: calc(100dvh - 132px); margin-top: 32px; + height: calc(100dvh - 32px); } } From 84ab1f95bedd2433cec165502375901fdaa56a98 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 15:21:52 -0700 Subject: [PATCH 18/90] UI: Increase the chat area a bit --- css/main.css | 1 + 1 file changed, 1 insertion(+) diff --git a/css/main.css b/css/main.css index f76a2787..d5d5e771 100644 --- a/css/main.css +++ b/css/main.css @@ -797,6 +797,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { #chat-input-row { padding: 1rem; + padding-top: 0; } #chat-input-row.bigchat { From d9da16edba88b09dcbfe97a7be302c65ed244ebb Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 16:53:52 -0700 Subject: [PATCH 19/90] UI: Remove the chat input textarea border --- css/main.css | 1 + 1 file changed, 1 insertion(+) diff --git a/css/main.css b/css/main.css index d5d5e771..b3e699fa 100644 --- a/css/main.css +++ b/css/main.css @@ -581,6 +581,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { #chat-input textarea { padding: 0.65rem 2.5rem; + border: 0; } #chat-input textarea::placeholder { From 690d693913f68d25d08fd74db902495766c12e5e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 18:04:52 -0700 Subject: [PATCH 20/90] UI: Add padding to only show the last message/reply after sending a message To avoid scrolling --- css/chat_style-Dark.css | 3 ++- css/chat_style-TheEncrypted777.css | 3 ++- css/chat_style-cai-chat-square.css | 3 ++- css/chat_style-cai-chat.css | 3 ++- css/chat_style-messenger.css | 3 ++- css/chat_style-wpp.css | 3 ++- css/html_instruct_style.css | 4 ---- css/main.css | 4 ---- js/main.js | 10 ++++++++++ modules/html_generator.py | 9 +++------ 10 files changed, 25 insertions(+), 20 deletions(-) diff --git a/css/chat_style-Dark.css b/css/chat_style-Dark.css index 3b2bd385..1ad46bc0 100644 --- a/css/chat_style-Dark.css +++ b/css/chat_style-Dark.css @@ -2,7 +2,8 @@ display: grid; align-items: start; grid-template-columns: 60px minmax(0, 1fr); - padding-bottom: 28px; + padding-bottom: 14px; + padding-top: 14px; font-size: 18px; font-family: Roboto, Arial, sans-serif; /* Modern font */ line-height: 1.5; diff --git a/css/chat_style-TheEncrypted777.css b/css/chat_style-TheEncrypted777.css index 25d26db8..9e1230b7 100644 --- a/css/chat_style-TheEncrypted777.css +++ b/css/chat_style-TheEncrypted777.css @@ -4,7 +4,8 @@ display: grid; align-items: start; grid-template-columns: 60px minmax(0, 1fr); - padding-bottom: 28px; + padding-bottom: 14px; + padding-top: 14px; font-size: 18px; font-family: 'Noto Sans', Arial, sans-serif; line-height: 1.428571429; diff --git a/css/chat_style-cai-chat-square.css b/css/chat_style-cai-chat-square.css index 854fff60..015f6927 100644 --- a/css/chat_style-cai-chat-square.css +++ b/css/chat_style-cai-chat-square.css @@ -16,6 +16,7 @@ } .message { - padding-bottom: 2em; + padding-bottom: 1em; + padding-top: 1em; grid-template-columns: 70px minmax(0, 1fr); } diff --git a/css/chat_style-cai-chat.css b/css/chat_style-cai-chat.css index 223f6150..0e91101f 100644 --- a/css/chat_style-cai-chat.css +++ b/css/chat_style-cai-chat.css @@ -2,7 +2,8 @@ display: grid; align-items: start; grid-template-columns: 60px minmax(0, 1fr); - padding-bottom: 2em; + padding-bottom: 1em; + padding-top: 1em; font-size: 15px; font-family: 'Noto Sans', Helvetica, Arial, sans-serif; line-height: 22.5px !important; diff --git a/css/chat_style-messenger.css b/css/chat_style-messenger.css index f0fd1578..6518d6ca 100644 --- a/css/chat_style-messenger.css +++ b/css/chat_style-messenger.css @@ -1,5 +1,6 @@ .message { - padding-bottom: 25px; + padding-bottom: 12.5px; + padding-top: 12.5px; font-size: 15px; font-family: 'Noto Sans', Helvetica, Arial, sans-serif; line-height: 1.428571429; diff --git a/css/chat_style-wpp.css b/css/chat_style-wpp.css index 30ca61f3..1442dd0a 100644 --- a/css/chat_style-wpp.css +++ b/css/chat_style-wpp.css @@ -1,5 +1,6 @@ .message { - padding-bottom: 25px; + padding-bottom: 12.5px; + padding-top: 12.5px; font-size: 15px; font-family: 'Noto Sans', Helvetica, Arial, sans-serif; line-height: 1.428571429; diff --git a/css/html_instruct_style.css b/css/html_instruct_style.css index b98544a1..f4339311 100644 --- a/css/html_instruct_style.css +++ b/css/html_instruct_style.css @@ -8,10 +8,6 @@ padding-top: 0 !important; } -.chat > .messages > :last-child { - margin-bottom: 1.7rem !important; -} - .chat .message-body p, .chat .message-body li { font-size: 1rem !important; line-height: 28px !important; diff --git a/css/main.css b/css/main.css index b3e699fa..9915735d 100644 --- a/css/main.css +++ b/css/main.css @@ -1375,7 +1375,3 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { 50% { opacity: 1; } 100% { opacity: 0.6; } } - -.streaming { - min-height: 70vh; -} diff --git a/js/main.js b/js/main.js index 408815db..e6611788 100644 --- a/js/main.js +++ b/js/main.js @@ -150,6 +150,16 @@ const observer = new MutationObserver(function(mutations) { if (!isScrolled && targetElement.scrollTop !== targetElement.scrollHeight) { targetElement.scrollTop = targetElement.scrollHeight; } + + const chatElement = document.getElementById("chat"); + if (chatElement) { + const messagesContainer = chatElement.querySelector(".messages"); + const lastChild = messagesContainer?.lastElementChild; + const prevSibling = lastChild?.previousElementSibling; + if (lastChild && prevSibling) { + lastChild.style.minHeight = `calc(100vh - ${prevSibling.offsetHeight}px - 102px)`; + } + } }); // Configure the observer to watch for changes in the subtree and attributes diff --git a/modules/html_generator.py b/modules/html_generator.py index a6f5f930..67d15b6e 100644 --- a/modules/html_generator.py +++ b/modules/html_generator.py @@ -365,9 +365,8 @@ def generate_instruct_html(history): f'
' ) - streaming_class = " streaming" if i == len(history["visible"]) - 1 else "" output += ( - f'
' f'
' f'
{converted_visible[1]}
' @@ -415,9 +414,8 @@ def generate_cai_chat_html(history, name1, name2, style, character, reset_cache= f'
' ) - streaming_class = " streaming" if i == len(history["visible"]) - 1 else "" output += ( - f'
' f'
{img_bot}
' f'
' @@ -454,9 +452,8 @@ def generate_chat_html(history, name1, name2, reset_cache=False): f'
' ) - streaming_class = " streaming" if i == len(history["visible"]) - 1 else "" output += ( - f'
' f'
' f'
{converted_visible[1]}
' From 2da197bba4b1547d51086e312d877d942e810be2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 18:29:05 -0700 Subject: [PATCH 21/90] Refinement after previous commit --- js/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/main.js b/js/main.js index e6611788..205cf88e 100644 --- a/js/main.js +++ b/js/main.js @@ -157,7 +157,7 @@ const observer = new MutationObserver(function(mutations) { const lastChild = messagesContainer?.lastElementChild; const prevSibling = lastChild?.previousElementSibling; if (lastChild && prevSibling) { - lastChild.style.minHeight = `calc(100vh - ${prevSibling.offsetHeight}px - 102px)`; + lastChild.style.minHeight = `calc(max(70vh, 100vh - ${prevSibling.offsetHeight}px - 102px))`; } } }); From d0211afb3c513bde0d8662bd686ddb0dc87354cd Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 18:52:01 -0700 Subject: [PATCH 22/90] Save the chat history right after sending a message --- modules/chat.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/chat.py b/modules/chat.py index 98913d5c..feac6bdd 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -483,6 +483,8 @@ def generate_chat_reply_wrapper(text, state, regenerate=False, _continue=False): history = state['history'] for i, history in enumerate(generate_chat_reply(text, state, regenerate, _continue, loading_message=True, for_ui=True)): yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu']), history + if i == 0: + save_history(history, state['unique_id'], state['character_menu'], state['mode']) save_history(history, state['unique_id'], state['character_menu'], state['mode']) From df7bb0db1fe6478d037debf73272b10cef1f75c7 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 20:03:55 -0700 Subject: [PATCH 23/90] Rename --n-gpu-layers to --gpu-layers --- modules/shared.py | 2 +- modules/ui_model_menu.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index cfedb992..b952c4a1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -120,7 +120,7 @@ group.add_argument('--threads-batch', type=int, default=0, help='Number of threa group.add_argument('--batch-size', type=int, default=256, help='Maximum number of prompt tokens to batch together when calling llama_eval.') group.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.') group.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') -group.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.') +group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=0, metavar='N', help='Number of layers to offload to the GPU.') group.add_argument('--tensor-split', type=str, default=None, help='Split the model across multiple GPUs. Comma-separated list of proportions. Example: 60,40.') group.add_argument('--numa', action='store_true', help='Activate NUMA task allocation for llama.cpp.') group.add_argument('--no-kv-offload', action='store_true', help='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 2c593df6..943645cf 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -49,7 +49,7 @@ def create_ui(): with gr.Blocks(): with gr.Row(): with gr.Column(): - shared.gradio['n_gpu_layers'] = gr.Slider(label="n-gpu-layers", minimum=0, maximum=256, value=shared.args.n_gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') + shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.n_gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=256, value=shared.args.threads) shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch) shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size) From f3da45f65d76f8c48fd95678ecc841afb0ddd04e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sun, 4 May 2025 20:37:15 -0700 Subject: [PATCH 24/90] ExLlamaV3_HF: Change max_chunk_size to 256 --- modules/exllamav3_hf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/exllamav3_hf.py b/modules/exllamav3_hf.py index 12b22f64..417df473 100644 --- a/modules/exllamav3_hf.py +++ b/modules/exllamav3_hf.py @@ -119,7 +119,7 @@ class Exllamav3HF(PreTrainedModel, GenerationMixin): reset = True # Maximum number of tokens to process in a single forward pass - max_chunk_size = 2048 + max_chunk_size = 256 # Make the forward call if labels is None: From b817bb33fd7b26a24c81798dabb36af4620d4a53 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 04:54:25 -0700 Subject: [PATCH 25/90] Minor fix after df7bb0db1fe6478d037debf73272b10cef1f75c7 --- modules/llama_cpp_server.py | 2 +- modules/ui_model_menu.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 7244001a..0ddb3fff 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -256,7 +256,7 @@ class LlamaServer: self.server_path, "--model", self.model_path, "--ctx-size", str(shared.args.ctx_size), - "--n-gpu-layers", str(shared.args.n_gpu_layers), + "--gpu-layers", str(shared.args.gpu_layers), "--batch-size", str(shared.args.batch_size), "--port", str(self.port), ] diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 943645cf..e05d2256 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -49,7 +49,7 @@ def create_ui(): with gr.Blocks(): with gr.Row(): with gr.Column(): - shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.n_gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') + shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=256, value=shared.args.threads) shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch) shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size) From 475e012ee8e0cbeb53fade01359cd649b9b5d470 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 06:16:11 -0700 Subject: [PATCH 26/90] UI: Improve the light theme colors --- css/main.css | 3 ++- modules/ui.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/css/main.css b/css/main.css index 9915735d..b1d6e345 100644 --- a/css/main.css +++ b/css/main.css @@ -2,7 +2,7 @@ --darker-gray: #202123; --dark-gray: #343541; --light-gray: #444654; - --light-theme-gray: #f5f5f5; + --light-theme-gray: #f3f4f6; --border-color-dark: #525252; --header-width: 112px; --selected-item-color-dark: #32333e; @@ -580,6 +580,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } #chat-input textarea { + background: var(--light-theme-gray); padding: 0.65rem 2.5rem; border: 0; } diff --git a/modules/ui.py b/modules/ui.py index fb016f87..d08c1435 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -61,7 +61,7 @@ if not shared.args.old_colors: background_fill_primary_dark='var(--darker-gray)', body_background_fill="white", block_background_fill="transparent", - body_text_color="#333", + body_text_color='rgb(64, 64, 64)', button_secondary_background_fill="#f4f4f4", button_secondary_border_color="var(--border-color-primary)", From 6001d279c64d92c1f2a312142e41119807694729 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 07:42:13 -0700 Subject: [PATCH 27/90] Light theme improvement --- css/main.css | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/css/main.css b/css/main.css index b1d6e345..38585a1c 100644 --- a/css/main.css +++ b/css/main.css @@ -979,6 +979,11 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { cursor: pointer; } +#past-chats .selected, +#past-chats label:hover { + background-color: rgb(224, 224, 224) !important; +} + #past-chats-buttons, #delete-chat-row, #rename-row { @@ -987,7 +992,6 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { gap: 9px; } - #past-chats-row, #chat-controls { width: 260px; From 967b70327ea10a9c5cc7c932583993687b9d4ba7 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 07:56:47 -0700 Subject: [PATCH 28/90] Light theme improvement --- css/html_instruct_style.css | 2 +- css/main.css | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/css/html_instruct_style.css b/css/html_instruct_style.css index f4339311..fb984338 100644 --- a/css/html_instruct_style.css +++ b/css/html_instruct_style.css @@ -42,7 +42,7 @@ } .chat .user-message { - background: #f5f5f5; + background: #f3f4f6; padding: 1.5rem 1rem; padding-bottom: 2rem; border-radius: 0; diff --git a/css/main.css b/css/main.css index 38585a1c..d6a0d220 100644 --- a/css/main.css +++ b/css/main.css @@ -2,7 +2,7 @@ --darker-gray: #202123; --dark-gray: #343541; --light-gray: #444654; - --light-theme-gray: #f3f4f6; + --light-theme-gray: #f9fbff; --border-color-dark: #525252; --header-width: 112px; --selected-item-color-dark: #32333e; @@ -580,7 +580,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } #chat-input textarea { - background: var(--light-theme-gray); + background: #f3f4f6; padding: 0.65rem 2.5rem; border: 0; } @@ -981,7 +981,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { #past-chats .selected, #past-chats label:hover { - background-color: rgb(224, 224, 224) !important; + background-color: #dbeafe !important; } #past-chats-buttons, @@ -1123,8 +1123,8 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { border: 0 !important; } -.dark #past-chats .selected, -.dark #past-chats label:hover { +.dark gradio-app .gradio-container.gradio-container-4-37-2 .contain #past-chats .selected, +.dark gradio-app .gradio-container.gradio-container-4-37-2 .contain #past-chats label:hover { background-color: var(--selected-item-color-dark) !important; } @@ -1161,7 +1161,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } .header_bar button.selected { - background: #E0E0E0; + background: #dbeafe; } #chat-controls, From bf5290bc0ff15f6894a4eb5785e8df60831ecb25 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 08:04:12 -0700 Subject: [PATCH 29/90] Fix the hover menu in light theme --- css/main.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/css/main.css b/css/main.css index d6a0d220..59165a62 100644 --- a/css/main.css +++ b/css/main.css @@ -742,7 +742,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { .hover-menu button { width: 100%; - background: transparent !important; + background: white !important; border-radius: 0 !important; justify-content: space-between; margin: 0 !important; From 53d8e4650202f5891364197011098b3af34fe6ac Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 12:28:17 -0700 Subject: [PATCH 30/90] Ensure environment isolation in portable installs --- start_linux.sh | 9 +++++---- start_macos.sh | 9 +++++---- start_windows.bat | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/start_linux.sh b/start_linux.sh index c74f1272..e2b00558 100755 --- a/start_linux.sh +++ b/start_linux.sh @@ -1,5 +1,10 @@ #!/usr/bin/env bash +# environment isolation +export PYTHONNOUSERSITE=1 +unset PYTHONPATH +unset PYTHONHOME + cd "$(dirname "${BASH_SOURCE[0]}")" # Portable install case @@ -61,10 +66,6 @@ if [ ! -e "$INSTALL_ENV_DIR/bin/python" ]; then exit fi -# environment isolation -export PYTHONNOUSERSITE=1 -unset PYTHONPATH -unset PYTHONHOME export CUDA_PATH="$INSTALL_ENV_DIR" export CUDA_HOME="$CUDA_PATH" diff --git a/start_macos.sh b/start_macos.sh index 7a060ba6..bff11bc1 100755 --- a/start_macos.sh +++ b/start_macos.sh @@ -1,5 +1,10 @@ #!/bin/bash +# environment isolation +export PYTHONNOUSERSITE=1 +unset PYTHONPATH +unset PYTHONHOME + cd "$(dirname "${BASH_SOURCE[0]}")" # Portable install case @@ -61,10 +66,6 @@ if [ ! -e "$INSTALL_ENV_DIR/bin/python" ]; then exit fi -# environment isolation -export PYTHONNOUSERSITE=1 -unset PYTHONPATH -unset PYTHONHOME export CUDA_PATH="$INSTALL_ENV_DIR" export CUDA_HOME="$CUDA_PATH" diff --git a/start_windows.bat b/start_windows.bat index 1616ee27..f5e66ec2 100755 --- a/start_windows.bat +++ b/start_windows.bat @@ -1,6 +1,11 @@ @echo off setlocal enabledelayedexpansion +@rem environment isolation +set PYTHONNOUSERSITE=1 +set PYTHONPATH= +set PYTHONHOME= + cd /D "%~dp0" @rem Portable install case @@ -87,10 +92,6 @@ if not exist "%INSTALL_ENV_DIR%" ( @rem check if conda environment was actually created if not exist "%INSTALL_ENV_DIR%\python.exe" ( echo. && echo Conda environment is empty. && goto end ) -@rem environment isolation -set PYTHONNOUSERSITE=1 -set PYTHONPATH= -set PYTHONHOME= set "CUDA_PATH=%INSTALL_ENV_DIR%" set "CUDA_HOME=%CUDA_PATH%" From 8137eb8ef46ac6950cb96094e3cc30b0a72dee76 Mon Sep 17 00:00:00 2001 From: mamei16 Date: Mon, 5 May 2025 23:05:23 +0200 Subject: [PATCH 31/90] Dynamic Chat Message UI Update Speed (#6952) --- modules/shared.py | 1 - modules/text_generation.py | 18 ++++++++---------- modules/ui.py | 1 - modules/ui_parameters.py | 2 -- user_data/settings-template.yaml | 1 - 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index b952c4a1..b4dfbfd1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -47,7 +47,6 @@ settings = { 'max_new_tokens_max': 4096, 'prompt_lookup_num_tokens': 0, 'max_tokens_second': 0, - 'max_updates_second': 12, 'auto_max_new_tokens': True, 'ban_eos_token': False, 'add_bos_token': True, diff --git a/modules/text_generation.py b/modules/text_generation.py index 8fd65dc4..7e48a2f6 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -64,41 +64,39 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap all_stop_strings += st shared.stop_everything = False - last_update = -1 reply = '' is_stream = state['stream'] if len(all_stop_strings) > 0 and not state['stream']: state = copy.deepcopy(state) state['stream'] = True - min_update_interval = 0 - if state.get('max_updates_second', 0) > 0: - min_update_interval = 1 / state['max_updates_second'] - # Generate + last_update = -1 + latency_threshold = 1 / 1000 for reply in generate_func(question, original_question, state, stopping_strings, is_chat=is_chat): + cur_time = time.monotonic() reply, stop_found = apply_stopping_strings(reply, all_stop_strings) if escape_html: reply = html.escape(reply) if is_stream: - cur_time = time.time() - # Limit number of tokens/second to make text readable in real time if state['max_tokens_second'] > 0: diff = 1 / state['max_tokens_second'] - (cur_time - last_update) if diff > 0: time.sleep(diff) - last_update = time.time() + last_update = time.monotonic() yield reply # Limit updates to avoid lag in the Gradio UI # API updates are not limited else: - if cur_time - last_update > min_update_interval: - last_update = cur_time + # If 'generate_func' takes less than 0.001 seconds to yield the next token + # (equivalent to more than 1000 tok/s), assume that the UI is lagging behind and skip yielding + if (cur_time - last_update) > latency_threshold: yield reply + last_update = time.monotonic() if stop_found or (state['max_tokens_second'] > 0 and shared.stop_everything): break diff --git a/modules/ui.py b/modules/ui.py index d08c1435..b3d4bccf 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -192,7 +192,6 @@ def list_interface_input_elements(): 'max_new_tokens', 'prompt_lookup_num_tokens', 'max_tokens_second', - 'max_updates_second', 'do_sample', 'dynamic_temperature', 'temperature_last', diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py index 733d0901..84f9fbfc 100644 --- a/modules/ui_parameters.py +++ b/modules/ui_parameters.py @@ -71,8 +71,6 @@ def create_ui(default_preset): shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.') shared.gradio['prompt_lookup_num_tokens'] = gr.Slider(value=shared.settings['prompt_lookup_num_tokens'], minimum=0, maximum=10, step=1, label='prompt_lookup_num_tokens', info='Activates Prompt Lookup Decoding.') shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.') - shared.gradio['max_updates_second'] = gr.Slider(value=shared.settings['max_updates_second'], minimum=0, maximum=24, step=1, label='Maximum UI updates/second', info='Set this if you experience lag in the UI during streaming.') - with gr.Column(): with gr.Row(): with gr.Column(): diff --git a/user_data/settings-template.yaml b/user_data/settings-template.yaml index ce0f77e1..db481e84 100644 --- a/user_data/settings-template.yaml +++ b/user_data/settings-template.yaml @@ -18,7 +18,6 @@ max_new_tokens_min: 1 max_new_tokens_max: 4096 prompt_lookup_num_tokens: 0 max_tokens_second: 0 -max_updates_second: 12 auto_max_new_tokens: true ban_eos_token: false add_bos_token: true From 85bf2e15b98117ef5630e81bf4a002440fffe2c2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 14:14:48 -0700 Subject: [PATCH 32/90] API: Remove obsolete multimodal extension handling Multimodal support will be added back once it's implemented in llama-server. --- extensions/openai/completions.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index 75e2cc11..46c76199 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -96,30 +96,6 @@ def convert_history(history): user_input_last = True system_message = "" - # Multimodal: convert OpenAI format to multimodal extension format - if any('content' in entry and isinstance(entry['content'], list) for entry in history): - new_history = [] - for entry in history: - if isinstance(entry['content'], list): - for item in entry['content']: - if not isinstance(item, dict): - continue - - image_url = None - content = None - if item['type'] == 'image_url' and isinstance(item['image_url'], dict): - image_url = item['image_url']['url'] - elif item['type'] == 'text' and isinstance(item['text'], str): - content = item['text'] - if image_url: - new_history.append({"image_url": image_url, "role": "user"}) - if content: - new_history.append({"content": content, "role": "user"}) - else: - new_history.append(entry) - - history = new_history - for entry in history: if "image_url" in entry: image_url = entry['image_url'] From f82667f0b4c0824420a6637efee3c680ddbe25f3 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 14:17:00 -0700 Subject: [PATCH 33/90] Remove more multimodal extension references --- extensions/openai/completions.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index 46c76199..a7d8b4e4 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -1,13 +1,8 @@ -import base64 import copy -import re import time from collections import deque -from io import BytesIO -import requests import tiktoken -from PIL import Image from extensions.openai.errors import InvalidRequestError from extensions.openai.utils import debug_msg @@ -97,28 +92,7 @@ def convert_history(history): system_message = "" for entry in history: - if "image_url" in entry: - image_url = entry['image_url'] - if "base64" in image_url: - image_url = re.sub('^data:image/.+;base64,', '', image_url) - img = Image.open(BytesIO(base64.b64decode(image_url))) - else: - try: - my_res = requests.get(image_url) - img = Image.open(BytesIO(my_res.content)) - except Exception: - raise 'Image cannot be loaded from the URL!' - - buffered = BytesIO() - if img.mode in ("RGBA", "P"): - img = img.convert("RGB") - - img.save(buffered, format="JPEG") - img_str = base64.b64encode(buffered.getvalue()).decode('utf-8') - content = f'' - else: - content = entry["content"] - + content = entry["content"] role = entry["role"] if role == "user": From 941e0663da48345150ae77d7c6b6eb54e21d671d Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 14:18:05 -0700 Subject: [PATCH 34/90] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a7b2467..6cc84c50 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ If you ever need to install something manually in the `installer_files` environm * There is no need to run any of those scripts (`start_`, `update_wizard_`, or `cmd_`) as admin/root. * To install the requirements for extensions, you can use the `extensions_reqs` script for your OS. At the end, this script will install the main requirements for the project to make sure that they take precedence in case of version conflicts. * For additional instructions about AMD and WSL setup, consult [the documentation](https://github.com/oobabooga/text-generation-webui/wiki). -* For automated installation, you can use the `GPU_CHOICE`, `USE_CUDA118`, `LAUNCH_AFTER_INSTALL`, and `INSTALL_EXTENSIONS` environment variables. For instance: `GPU_CHOICE=A USE_CUDA118=FALSE LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh`. +* For automated installation, you can use the `GPU_CHOICE`, `LAUNCH_AFTER_INSTALL`, and `INSTALL_EXTENSIONS` environment variables. For instance: `GPU_CHOICE=A LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh`. ### Manual installation using Conda From 987505ead345b0e113d636311f6a5faa4fcbe986 Mon Sep 17 00:00:00 2001 From: Evgenii Novikov Date: Tue, 6 May 2025 00:03:33 +0200 Subject: [PATCH 35/90] docker: Fix app uid typo in cpu docker compose (#6957) --- docker/cpu/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/cpu/docker-compose.yml b/docker/cpu/docker-compose.yml index c9d415ae..9aba314a 100644 --- a/docker/cpu/docker-compose.yml +++ b/docker/cpu/docker-compose.yml @@ -22,7 +22,7 @@ services: TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5} BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-} APP_GID: ${APP_GID:-6972} - APP_UID: ${APP_UID-6972} + APP_UID: ${APP_UID:-6972} env_file: .env user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}" ports: From 99bd66445f90df58a3f3832b35cca94dc397d1be Mon Sep 17 00:00:00 2001 From: Alireza Ghasemi Date: Tue, 6 May 2025 00:04:06 +0200 Subject: [PATCH 36/90] SuperboogaV2: minor update to avoid json serialization errors #6945 --- extensions/superboogav2/chromadb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/superboogav2/chromadb.py b/extensions/superboogav2/chromadb.py index 6e93dd92..f4f77821 100644 --- a/extensions/superboogav2/chromadb.py +++ b/extensions/superboogav2/chromadb.py @@ -292,6 +292,8 @@ class ChromaCollector(): for doc in documents: doc_tokens = encode(doc)[0] + if isinstance(doc_tokens, np.ndarray): + doc_tokens = doc_tokens.tolist() doc_token_count = len(doc_tokens) if current_token_count + doc_token_count > max_token_count: # If adding this document would exceed the max token count, From 76f947e3cf1c71e4105f708f02b2ca163a69987c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 15:58:29 -0700 Subject: [PATCH 37/90] UI: Minor style change --- css/main.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/css/main.css b/css/main.css index 59165a62..520ff972 100644 --- a/css/main.css +++ b/css/main.css @@ -1380,3 +1380,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { 50% { opacity: 1; } 100% { opacity: 0.6; } } + +strong { + font-weight: bold; +} From 530223bf0b196257e41ec948c2e92e1c3e507e9f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 16:00:49 -0700 Subject: [PATCH 38/90] UI: Fix the hover menu colors --- css/main.css | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/css/main.css b/css/main.css index 520ff972..b8ba8256 100644 --- a/css/main.css +++ b/css/main.css @@ -761,6 +761,10 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { background: var(--button-secondary-background-fill-hover) !important; } +.dark .hover-menu button:hover { + background: var(--selected-item-color-dark) !important; +} + .transparent-substring { opacity: 0.333; } @@ -1109,12 +1113,9 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { color: #9ca3af; } -.dark .hover-menu { - background-color: var(--darker-gray); -} - .dark .hover-menu button { border-color: var(--border-color-primary); + background-color: var(--darker-gray) !important; } .dark #chat-controls, From 4e8f628d3c206e8362cea5b5f7557abe33351bc0 Mon Sep 17 00:00:00 2001 From: Evgenii Novikov Date: Tue, 6 May 2025 01:05:15 +0200 Subject: [PATCH 39/90] docker: App uid typo in other docker composes (#6958) --- docker/amd/docker-compose.yml | 2 +- docker/intel/docker-compose.yml | 2 +- docker/nvidia/docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/amd/docker-compose.yml b/docker/amd/docker-compose.yml index 4709ae94..8866e9ed 100644 --- a/docker/amd/docker-compose.yml +++ b/docker/amd/docker-compose.yml @@ -22,7 +22,7 @@ services: TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5} BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-} APP_GID: ${APP_GID:-6972} - APP_UID: ${APP_UID-6972} + APP_UID: ${APP_UID:-6972} env_file: .env user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}" ports: diff --git a/docker/intel/docker-compose.yml b/docker/intel/docker-compose.yml index 31e9dde0..78e06698 100644 --- a/docker/intel/docker-compose.yml +++ b/docker/intel/docker-compose.yml @@ -22,7 +22,7 @@ services: TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5} BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-} APP_GID: ${APP_GID:-6972} - APP_UID: ${APP_UID-6972} + APP_UID: ${APP_UID:-6972} env_file: .env user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}" ports: diff --git a/docker/nvidia/docker-compose.yml b/docker/nvidia/docker-compose.yml index 835dd838..0392078e 100644 --- a/docker/nvidia/docker-compose.yml +++ b/docker/nvidia/docker-compose.yml @@ -22,7 +22,7 @@ services: TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5} BUILD_EXTENSIONS: ${BUILD_EXTENSIONS:-} APP_GID: ${APP_GID:-6972} - APP_UID: ${APP_UID-6972} + APP_UID: ${APP_UID:-6972} env_file: .env user: "${APP_RUNTIME_UID:-6972}:${APP_RUNTIME_GID:-6972}" ports: From cbef35054cb598b033e17b0442e8dad2da6873c4 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 5 May 2025 17:46:09 -0700 Subject: [PATCH 40/90] UI: CSS fix --- css/main.css | 1 + 1 file changed, 1 insertion(+) diff --git a/css/main.css b/css/main.css index b8ba8256..746f1f9e 100644 --- a/css/main.css +++ b/css/main.css @@ -426,6 +426,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { .chat > .messages { display: flex; flex-direction: column; + min-height: calc(100vh - 102px); } .chat > .messages > :first-child { From d1c0154d664e51d1ee6ea82d9c0e799d96367d4a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 06:38:39 -0700 Subject: [PATCH 41/90] llama.cpp: Add top_n_sigma, fix typical_p in sampler priority --- modules/llama_cpp_server.py | 5 ++++- modules/presets.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 0ddb3fff..b9902cd7 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -66,6 +66,7 @@ class LlamaServer: "top_k": state["top_k"], "top_p": state["top_p"], "min_p": state["min_p"], + "top_n_sigma": state["top_n_sigma"] if state["top_n_sigma"] > 0 else -1, "tfs_z": state["tfs"], "typical_p": state["typical_p"], "repeat_penalty": state["repetition_penalty"], @@ -102,8 +103,10 @@ class LlamaServer: penalty_found = False for s in samplers: - if s.strip() in ["dry", "top_k", "typ_p", "top_p", "min_p", "xtc", "temperature"]: + if s.strip() in ["dry", "top_k", "top_p", "top_n_sigma", "min_p", "temperature", "xtc"]: filtered_samplers.append(s.strip()) + elif s.strip() == "typical_p": + filtered_samplers.append("typ_p") elif not penalty_found and s.strip() == "repetition_penalty": filtered_samplers.append("penalties") penalty_found = True diff --git a/modules/presets.py b/modules/presets.py index 50d0f985..5a9a5873 100644 --- a/modules/presets.py +++ b/modules/presets.py @@ -52,7 +52,7 @@ def default_preset(): if shared.args.portable: samplers = result['sampler_priority'].split('\n') - samplers = [sampler for sampler in samplers if sampler in ["dry", "top_k", "typ_p", "top_p", "min_p", "xtc", "temperature", "repetition_penalty"]] + samplers = [sampler for sampler in samplers if sampler in ["dry", "top_k", "top_p", "top_n_sigma", "min_p", "temperature", "xtc", "typical_p", "repetition_penalty"]] result['sampler_priority'] = '\n'.join(samplers) return result From 89590adc14c941814c2d54795cfc78fab959d9e7 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 06:41:17 -0700 Subject: [PATCH 42/90] Update llama.cpp --- requirements/full/requirements.txt | 4 ++-- requirements/full/requirements_amd.txt | 4 ++-- requirements/full/requirements_amd_noavx2.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 4 ++-- .../full/requirements_apple_silicon.txt | 6 +++--- requirements/full/requirements_cpu_only.txt | 4 ++-- .../full/requirements_cpu_only_noavx2.txt | 4 ++-- requirements/full/requirements_noavx2.txt | 4 ++-- requirements/portable/requirements.txt | 4 ++-- requirements/portable/requirements_amd.txt | 18 ------------------ .../portable/requirements_amd_noavx2.txt | 18 ------------------ .../portable/requirements_apple_intel.txt | 4 ++-- .../portable/requirements_apple_silicon.txt | 6 +++--- .../portable/requirements_cpu_only.txt | 4 ++-- .../portable/requirements_cpu_only_noavx2.txt | 4 ++-- requirements/portable/requirements_noavx2.txt | 4 ++-- requirements/portable/requirements_vulkan.txt | 4 ++-- .../portable/requirements_vulkan_noavx2.txt | 4 ++-- 18 files changed, 34 insertions(+), 70 deletions(-) delete mode 100644 requirements/portable/requirements_amd.txt delete mode 100644 requirements/portable/requirements_amd_noavx2.txt diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index c0ace41b..a60ea7b4 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt index 24eeee6a..431cd740 100644 --- a/requirements/full/requirements_amd.txt +++ b/requirements/full/requirements_amd.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt index 99716f3c..0c581f86 100644 --- a/requirements/full/requirements_amd_noavx2.txt +++ b/requirements/full/requirements_amd_noavx2.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index cc747edb..f7213efe 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index 67b3260e..4aac3dea 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -29,8 +29,8 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt index 47ad5759..ac277d61 100644 --- a/requirements/full/requirements_cpu_only.txt +++ b/requirements/full/requirements_cpu_only.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt index 334f11df..cc412d33 100644 --- a/requirements/full/requirements_cpu_only_noavx2.txt +++ b/requirements/full/requirements_cpu_only_noavx2.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index 3575d352..78265f1a 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt index c720daa7..1240d335 100644 --- a/requirements/portable/requirements.txt +++ b/requirements/portable/requirements.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_amd.txt b/requirements/portable/requirements_amd.txt deleted file mode 100644 index 7d9c00c0..00000000 --- a/requirements/portable/requirements_amd.txt +++ /dev/null @@ -1,18 +0,0 @@ -fastapi==0.112.4 -gradio==4.37.* -jinja2==3.1.6 -markdown -numpy==1.26.* -pydantic==2.8.2 -pyyaml -requests -rich -tqdm - -# API -flask_cloudflared==0.0.14 -sse-starlette==1.6.5 -tiktoken - -# AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+rocm6.1.2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_amd_noavx2.txt b/requirements/portable/requirements_amd_noavx2.txt deleted file mode 100644 index d718c1b1..00000000 --- a/requirements/portable/requirements_amd_noavx2.txt +++ /dev/null @@ -1,18 +0,0 @@ -fastapi==0.112.4 -gradio==4.37.* -jinja2==3.1.6 -markdown -numpy==1.26.* -pydantic==2.8.2 -pyyaml -requests -rich -tqdm - -# API -flask_cloudflared==0.0.14 -sse-starlette==1.6.5 -tiktoken - -# AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+rocm6.1.2avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt index 9e184b53..6b165b7c 100644 --- a/requirements/portable/requirements_apple_intel.txt +++ b/requirements/portable/requirements_apple_intel.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt index ec059716..1b2b5cf2 100644 --- a/requirements/portable/requirements_apple_silicon.txt +++ b/requirements/portable/requirements_apple_silicon.txt @@ -15,6 +15,6 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt index d473b824..2793d743 100644 --- a/requirements/portable/requirements_cpu_only.txt +++ b/requirements/portable/requirements_cpu_only.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt index d3fffb43..6d7316a6 100644 --- a/requirements/portable/requirements_cpu_only_noavx2.txt +++ b/requirements/portable/requirements_cpu_only_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt index cdfa6a01..e56eba08 100644 --- a/requirements/portable/requirements_noavx2.txt +++ b/requirements/portable/requirements_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt index 1a7ce6ed..a7f8c703 100644 --- a/requirements/portable/requirements_vulkan.txt +++ b/requirements/portable/requirements_vulkan.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_vulkan_noavx2.txt b/requirements/portable/requirements_vulkan_noavx2.txt index 4737321d..5b427fd2 100644 --- a/requirements/portable/requirements_vulkan_noavx2.txt +++ b/requirements/portable/requirements_vulkan_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.8.0/llama_cpp_binaries-0.8.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" From 605cc9ab14533dd20cc11363f020fb9947cfb723 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 06:42:15 -0700 Subject: [PATCH 43/90] Update exllamav3 --- requirements/full/requirements.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 2 +- requirements/full/requirements_apple_silicon.txt | 2 +- requirements/full/requirements_noavx2.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index a60ea7b4..3b50c674 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -32,8 +32,8 @@ tiktoken # CUDA wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index f7213efe..ba23ea9c 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -31,5 +31,5 @@ tiktoken # Mac wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6-py3-none-any.whl +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index 4aac3dea..c245ab74 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -32,5 +32,5 @@ tiktoken https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6-py3-none-any.whl +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index 78265f1a..d8bbf6d1 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -32,8 +32,8 @@ tiktoken # CUDA wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a6/exllamav3-0.0.1a6+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" From 1927afe89457dce8eb805b2275ba7c8a9680a967 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 08:18:49 -0700 Subject: [PATCH 44/90] Fix top_n_sigma not showing for llama.cpp --- modules/loaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/loaders.py b/modules/loaders.py index 738198b1..217d569c 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -300,6 +300,7 @@ loaders_samplers = { 'xtc_threshold', 'xtc_probability', 'tfs', + 'top_n_sigma', 'dry_multiplier', 'dry_allowed_length', 'dry_base', From 05115e42ee1ab7a2848b883e469885ce9504f04a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 08:27:21 -0700 Subject: [PATCH 45/90] Set top_n_sigma before temperature by default --- modules/presets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/presets.py b/modules/presets.py index 5a9a5873..cf706605 100644 --- a/modules/presets.py +++ b/modules/presets.py @@ -46,7 +46,7 @@ def default_preset(): 'do_sample': True, 'dynamic_temperature': False, 'temperature_last': False, - 'sampler_priority': 'repetition_penalty\npresence_penalty\nfrequency_penalty\ndry\ntemperature\ndynamic_temperature\nquadratic_sampling\ntop_n_sigma\ntop_k\ntop_p\ntypical_p\nepsilon_cutoff\neta_cutoff\ntfs\ntop_a\nmin_p\nmirostat\nxtc\nencoder_repetition_penalty\nno_repeat_ngram', + 'sampler_priority': 'repetition_penalty\npresence_penalty\nfrequency_penalty\ndry\ntop_n_sigma\ntemperature\ndynamic_temperature\nquadratic_sampling\ntop_k\ntop_p\ntypical_p\nepsilon_cutoff\neta_cutoff\ntfs\ntop_a\nmin_p\nmirostat\nxtc\nencoder_repetition_penalty\nno_repeat_ngram', 'dry_sequence_breakers': '"\\n", ":", "\\"", "*"', } From c4f36db0d859e1819550e576a7fbd513c990c64d Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 08:41:13 -0700 Subject: [PATCH 46/90] llama.cpp: remove tfs (it doesn't get used) --- modules/llama_cpp_server.py | 1 - modules/loaders.py | 1 - 2 files changed, 2 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index b9902cd7..d8d2f61b 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -67,7 +67,6 @@ class LlamaServer: "top_p": state["top_p"], "min_p": state["min_p"], "top_n_sigma": state["top_n_sigma"] if state["top_n_sigma"] > 0 else -1, - "tfs_z": state["tfs"], "typical_p": state["typical_p"], "repeat_penalty": state["repetition_penalty"], "repeat_last_n": state["repetition_penalty_range"], diff --git a/modules/loaders.py b/modules/loaders.py index 217d569c..b29679bd 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -299,7 +299,6 @@ loaders_samplers = { 'typical_p', 'xtc_threshold', 'xtc_probability', - 'tfs', 'top_n_sigma', 'dry_multiplier', 'dry_allowed_length', From 5ef564a22e8df21a7480d5c8d6e32919f35f14c7 Mon Sep 17 00:00:00 2001 From: Downtown-Case Date: Tue, 6 May 2025 15:03:33 -0500 Subject: [PATCH 47/90] Fix model config loading in shared.py for Python 3.13 (#6961) --- modules/shared.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index b4dfbfd1..6fd4604c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -310,11 +310,13 @@ if args.api or args.public_api: add_extension('openai', last=True) # Load model-specific settings -with Path(f'{args.model_dir}/config.yaml') as p: - if p.exists(): - model_config = yaml.safe_load(open(p, 'r').read()) - else: - model_config = {} +p = Path(f'{args.model_dir}/config.yaml') +if p.exists(): + model_config = yaml.safe_load(open(p, 'r').read()) +else: + model_config = {} +del p + # Load custom model-specific settings user_config = load_user_config() From e4fb2475d25e1dccfa39f5d943bcde61ef517245 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 14:02:01 -0700 Subject: [PATCH 48/90] UI: Multiple small style improvements (light/dark themes) --- css/html_instruct_style.css | 2 +- css/main.css | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/css/html_instruct_style.css b/css/html_instruct_style.css index fb984338..6ad250aa 100644 --- a/css/html_instruct_style.css +++ b/css/html_instruct_style.css @@ -66,7 +66,7 @@ .chat .user-message .text, .chat .assistant-message .text { - max-width: 645px; + max-width: 700px; margin-left: auto; margin-right: auto; } diff --git a/css/main.css b/css/main.css index 746f1f9e..30089aca 100644 --- a/css/main.css +++ b/css/main.css @@ -545,7 +545,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { border-radius: 5px; font-size: 82%; padding: 1px 3px; - background: white !important; + background: #f3f4f6 !important; color: #1f2328; } @@ -559,18 +559,17 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { padding: 15px; } -.message-body :not(pre) > code::before { - content: "`"; -} - -.message-body :not(pre) > code::after { - content: "`"; -} - .message-body :not(pre) > code { white-space: normal !important; font-weight: bold; - font-family: unset; + font-size: 0.95em; + font-family: Menlo,"Roboto Mono","Courier New",Courier,monospace,Inter,sans-serif; + padding: .15rem .3rem; + background-color: #ececec; +} + +.dark .message-body :not(pre) > code { + background-color: rgb(255 255 255 / 12.5%); } #chat-input { @@ -584,6 +583,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { background: #f3f4f6; padding: 0.65rem 2.5rem; border: 0; + box-shadow: 0; } #chat-input textarea::placeholder { @@ -759,7 +759,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } .hover-menu button:hover { - background: var(--button-secondary-background-fill-hover) !important; + background: #dbeafe !important; } .dark .hover-menu button:hover { From b28fa86db6921adc8a42038f7062b72a27cb68b1 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 6 May 2025 17:51:55 -0700 Subject: [PATCH 49/90] Default --gpu-layers to 256 --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 6fd4604c..f2698bd2 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -119,7 +119,7 @@ group.add_argument('--threads-batch', type=int, default=0, help='Number of threa group.add_argument('--batch-size', type=int, default=256, help='Maximum number of prompt tokens to batch together when calling llama_eval.') group.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.') group.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') -group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=0, metavar='N', help='Number of layers to offload to the GPU.') +group.add_argument('--gpu-layers', '--n-gpu-layers', type=int, default=256, metavar='N', help='Number of layers to offload to the GPU.') group.add_argument('--tensor-split', type=str, default=None, help='Split the model across multiple GPUs. Comma-separated list of proportions. Example: 60,40.') group.add_argument('--numa', action='store_true', help='Activate NUMA task allocation for llama.cpp.') group.add_argument('--no-kv-offload', action='store_true', help='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') From d2bae7694c0798f9f51bc61a1f7b20d93059f106 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 07:26:23 -0700 Subject: [PATCH 50/90] UI: Change the ctx-size description --- modules/ui_model_menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index e05d2256..8dea457e 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -54,7 +54,7 @@ def create_ui(): shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch) shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size) shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) - shared.gradio['ctx_size'] = gr.Number(label='ctx-size', precision=0, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model. Common values: 2048, 4096, 8192, 16384, 32768, 65536.') + shared.gradio['ctx_size'] = gr.Number(label='ctx-size', precision=0, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model. Common values: 2048, 4096, 8192, 16384, 32768, 65536, 131072.') shared.gradio['cache_type'] = gr.Dropdown(label="cache_type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='List of proportions to split the model across multiple GPUs. Example: 60,40') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') From 348d4860c278eda1dedff15c05082e2d3358c3f3 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 07:58:59 -0700 Subject: [PATCH 51/90] UI: Create a "Main options" section in the Model tab --- modules/ui_model_menu.py | 70 ++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 8dea457e..28b7222d 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -47,52 +47,27 @@ def create_ui(): with gr.Column(): shared.gradio['loader'] = gr.Dropdown(label="Model loader", choices=loaders.loaders_and_params.keys() if not shared.args.portable else ['llama.cpp'], value=None) with gr.Blocks(): + gr.Markdown("## Main options") with gr.Row(): with gr.Column(): shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') - shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=256, value=shared.args.threads) - shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch) - shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size) - shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) + shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') shared.gradio['ctx_size'] = gr.Number(label='ctx-size', precision=0, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model. Common values: 2048, 4096, 8192, 16384, 32768, 65536, 131072.') shared.gradio['cache_type'] = gr.Dropdown(label="cache_type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') - shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='List of proportions to split the model across multiple GPUs. Example: 60,40') - shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') - shared.gradio['extra_flags'] = gr.Textbox(label='extra-flags', info='Additional flags to pass to llama-server. Format: "flag1=value1,flag2,flag3=value3". Example: "override-tensor=exps=CPU"', value=shared.args.extra_flags) - shared.gradio['cpu_memory'] = gr.Number(label="Maximum CPU memory in GiB. Use this for CPU offloading.", value=shared.args.cpu_memory) - shared.gradio['alpha_value'] = gr.Number(label='alpha_value', value=shared.args.alpha_value, precision=2, info='Positional embeddings alpha factor for NTK RoPE scaling. Recommended values (NTKv1): 1.75 for 1.5x context, 2.5 for 2x context. Use either this or compress_pos_emb, not both.') - shared.gradio['rope_freq_base'] = gr.Number(label='rope_freq_base', value=shared.args.rope_freq_base, precision=0, info='Positional embeddings frequency base for NTK RoPE scaling. Related to alpha_value by rope_freq_base = 10000 * alpha_value ^ (64 / 63). 0 = from model.') - shared.gradio['compress_pos_emb'] = gr.Number(label='compress_pos_emb', value=shared.args.compress_pos_emb, precision=2, info='Positional embeddings compression factor. Should be set to (context length) / (model\'s original context length). Equal to 1/rope_freq_scale.') - shared.gradio['compute_dtype'] = gr.Dropdown(label="compute_dtype", choices=["bfloat16", "float16", "float32"], value=shared.args.compute_dtype, info='Used by load-in-4bit.') - shared.gradio['quant_type'] = gr.Dropdown(label="quant_type", choices=["nf4", "fp4"], value=shared.args.quant_type, info='Used by load-in-4bit.') - shared.gradio['num_experts_per_token'] = gr.Number(label="Number of experts per token", value=shared.args.num_experts_per_token, info='Only applies to MoE models like Mixtral.') + shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) with gr.Column(): + shared.gradio['flash_attn'] = gr.Checkbox(label="flash_attn", value=shared.args.flash_attn, info='Use flash-attention.') + shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming_llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) shared.gradio['load_in_4bit'] = gr.Checkbox(label="load-in-4bit", value=shared.args.load_in_4bit) shared.gradio['torch_compile'] = gr.Checkbox(label="torch-compile", value=shared.args.torch_compile, info='Compile the model with torch.compile for improved performance.') - shared.gradio['flash_attn'] = gr.Checkbox(label="flash_attn", value=shared.args.flash_attn, info='Use flash-attention.') shared.gradio['use_flash_attention_2'] = gr.Checkbox(label="use_flash_attention_2", value=shared.args.use_flash_attention_2, info='Set use_flash_attention_2=True while loading the model.') - shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming_llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') - shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu, info='llama.cpp: Use llama-cpp-python compiled without GPU acceleration. Transformers: use PyTorch in CPU mode.') - shared.gradio['disk'] = gr.Checkbox(label="disk", value=shared.args.disk) - shared.gradio['row_split'] = gr.Checkbox(label="row_split", value=shared.args.row_split, info='Split the model by rows across GPUs. This may improve multi-gpu performance.') - shared.gradio['no_kv_offload'] = gr.Checkbox(label="no_kv_offload", value=shared.args.no_kv_offload, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') - shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap) - shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) - shared.gradio['numa'] = gr.Checkbox(label="numa", value=shared.args.numa, info='NUMA support can help on some systems with non-uniform memory access.') shared.gradio['use_double_quant'] = gr.Checkbox(label="use_double_quant", value=shared.args.use_double_quant, info='Used by load-in-4bit.') - shared.gradio['use_eager_attention'] = gr.Checkbox(label="use_eager_attention", value=shared.args.use_eager_attention, info='Set attn_implementation= eager while loading the model.') - shared.gradio['bf16'] = gr.Checkbox(label="bf16", value=shared.args.bf16) shared.gradio['autosplit'] = gr.Checkbox(label="autosplit", value=shared.args.autosplit, info='Automatically split the model tensors across the available GPUs.') shared.gradio['enable_tp'] = gr.Checkbox(label="enable_tp", value=shared.args.enable_tp, info='Enable Tensor Parallelism (TP).') - shared.gradio['no_flash_attn'] = gr.Checkbox(label="no_flash_attn", value=shared.args.no_flash_attn) - shared.gradio['no_xformers'] = gr.Checkbox(label="no_xformers", value=shared.args.no_xformers) - shared.gradio['no_sdpa'] = gr.Checkbox(label="no_sdpa", value=shared.args.no_sdpa) - shared.gradio['cfg_cache'] = gr.Checkbox(label="cfg-cache", value=shared.args.cfg_cache, info='Necessary to use CFG with this loader.') shared.gradio['cpp_runner'] = gr.Checkbox(label="cpp-runner", value=shared.args.cpp_runner, info='Enable inference with ModelRunnerCpp, which is faster than the default ModelRunner.') shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Set trust_remote_code=True while loading the tokenizer/model. To enable this option, start the web UI with the --trust-remote-code flag.', interactive=shared.args.trust_remote_code) - shared.gradio['no_use_fast'] = gr.Checkbox(label="no_use_fast", value=shared.args.no_use_fast, info='Set use_fast=False while loading the tokenizer.') shared.gradio['exllamav2_info'] = gr.Markdown("ExLlamav2_HF is recommended over ExLlamav2 for better integration with extensions and more consistent sampling behavior across loaders.") shared.gradio['tensorrt_llm_info'] = gr.Markdown('* TensorRT-LLM has to be installed manually in a separate Python 3.10 environment at the moment. For a guide, consult the description of [this PR](https://github.com/oobabooga/text-generation-webui/pull/5715). \n\n* `ctx_size` is only used when `cpp-runner` is checked.\n\n* `cpp_runner` does not support streaming at the moment.') @@ -102,11 +77,44 @@ def create_ui(): shared.gradio['model_draft'] = gr.Dropdown(label="model-draft", choices=utils.get_available_models(), value=lambda: shared.args.model_draft, elem_classes='slim-dropdown', info='Draft model. Speculative decoding only works with models sharing the same vocabulary (e.g., same model family).', interactive=not mu) ui.create_refresh_button(shared.gradio['model_draft'], lambda: None, lambda: {'choices': utils.get_available_models()}, 'refresh-button', interactive=not mu) - shared.gradio['draft_max'] = gr.Number(label="draft-max", precision=0, step=1, value=shared.args.draft_max, info='Number of tokens to draft for speculative decoding.') shared.gradio['gpu_layers_draft'] = gr.Slider(label="gpu-layers-draft", minimum=0, maximum=256, value=shared.args.gpu_layers_draft, info='Number of layers to offload to the GPU for the draft model.') + shared.gradio['draft_max'] = gr.Number(label="draft-max", precision=0, step=1, value=shared.args.draft_max, info='Number of tokens to draft for speculative decoding. Recommended value: 4.') shared.gradio['device_draft'] = gr.Textbox(label="device-draft", value=shared.args.device_draft, info='Comma-separated list of devices to use for offloading the draft model. Example: CUDA0,CUDA1') shared.gradio['ctx_size_draft'] = gr.Number(label="ctx-size-draft", precision=0, step=256, value=shared.args.ctx_size_draft, info='Size of the prompt context for the draft model. If 0, uses the same as the main model.') + gr.Markdown("## Other options") + with gr.Accordion("See more options", open=False, elem_classes='tgw-accordion'): + with gr.Row(): + with gr.Column(): + shared.gradio['threads'] = gr.Slider(label="threads", minimum=0, step=1, maximum=256, value=shared.args.threads) + shared.gradio['threads_batch'] = gr.Slider(label="threads_batch", minimum=0, step=1, maximum=256, value=shared.args.threads_batch) + shared.gradio['batch_size'] = gr.Slider(label="batch_size", minimum=1, maximum=4096, step=1, value=shared.args.batch_size) + shared.gradio['tensor_split'] = gr.Textbox(label='tensor_split', info='List of proportions to split the model across multiple GPUs. Example: 60,40') + shared.gradio['extra_flags'] = gr.Textbox(label='extra-flags', info='Additional flags to pass to llama-server. Format: "flag1=value1,flag2,flag3=value3". Example: "override-tensor=exps=CPU"', value=shared.args.extra_flags) + shared.gradio['cpu_memory'] = gr.Number(label="Maximum CPU memory in GiB. Use this for CPU offloading.", value=shared.args.cpu_memory) + shared.gradio['alpha_value'] = gr.Number(label='alpha_value', value=shared.args.alpha_value, precision=2, info='Positional embeddings alpha factor for NTK RoPE scaling. Recommended values (NTKv1): 1.75 for 1.5x context, 2.5 for 2x context. Use either this or compress_pos_emb, not both.') + shared.gradio['rope_freq_base'] = gr.Number(label='rope_freq_base', value=shared.args.rope_freq_base, precision=0, info='Positional embeddings frequency base for NTK RoPE scaling. Related to alpha_value by rope_freq_base = 10000 * alpha_value ^ (64 / 63). 0 = from model.') + shared.gradio['compress_pos_emb'] = gr.Number(label='compress_pos_emb', value=shared.args.compress_pos_emb, precision=2, info='Positional embeddings compression factor. Should be set to (context length) / (model\'s original context length). Equal to 1/rope_freq_scale.') + shared.gradio['compute_dtype'] = gr.Dropdown(label="compute_dtype", choices=["bfloat16", "float16", "float32"], value=shared.args.compute_dtype, info='Used by load-in-4bit.') + shared.gradio['quant_type'] = gr.Dropdown(label="quant_type", choices=["nf4", "fp4"], value=shared.args.quant_type, info='Used by load-in-4bit.') + shared.gradio['num_experts_per_token'] = gr.Number(label="Number of experts per token", value=shared.args.num_experts_per_token, info='Only applies to MoE models like Mixtral.') + + with gr.Column(): + shared.gradio['cpu'] = gr.Checkbox(label="cpu", value=shared.args.cpu, info='llama.cpp: Use llama-cpp-python compiled without GPU acceleration. Transformers: use PyTorch in CPU mode.') + shared.gradio['disk'] = gr.Checkbox(label="disk", value=shared.args.disk) + shared.gradio['row_split'] = gr.Checkbox(label="row_split", value=shared.args.row_split, info='Split the model by rows across GPUs. This may improve multi-gpu performance.') + shared.gradio['no_kv_offload'] = gr.Checkbox(label="no_kv_offload", value=shared.args.no_kv_offload, info='Do not offload the K, Q, V to the GPU. This saves VRAM but reduces the performance.') + shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap) + shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) + shared.gradio['numa'] = gr.Checkbox(label="numa", value=shared.args.numa, info='NUMA support can help on some systems with non-uniform memory access.') + shared.gradio['use_eager_attention'] = gr.Checkbox(label="use_eager_attention", value=shared.args.use_eager_attention, info='Set attn_implementation= eager while loading the model.') + shared.gradio['bf16'] = gr.Checkbox(label="bf16", value=shared.args.bf16) + shared.gradio['no_flash_attn'] = gr.Checkbox(label="no_flash_attn", value=shared.args.no_flash_attn) + shared.gradio['no_xformers'] = gr.Checkbox(label="no_xformers", value=shared.args.no_xformers) + shared.gradio['no_sdpa'] = gr.Checkbox(label="no_sdpa", value=shared.args.no_sdpa) + shared.gradio['cfg_cache'] = gr.Checkbox(label="cfg-cache", value=shared.args.cfg_cache, info='Necessary to use CFG with this loader.') + shared.gradio['no_use_fast'] = gr.Checkbox(label="no_use_fast", value=shared.args.no_use_fast, info='Set use_fast=False while loading the tokenizer.') + with gr.Column(): with gr.Tab("Download"): shared.gradio['custom_model_menu'] = gr.Textbox(label="Download model or LoRA", info="Enter the Hugging Face username/model path, for instance: facebook/galactica-125m. To specify a branch, add it at the end after a \":\" character like this: facebook/galactica-125m:main. To download a single file, enter its name in the second box.", interactive=not mu) From a2ab42d39099d89543a8e5c5753350e51905fa36 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 08:00:38 -0700 Subject: [PATCH 52/90] UI: Remove the exllamav2 info message --- modules/loaders.py | 1 - modules/ui_model_menu.py | 1 - 2 files changed, 2 deletions(-) diff --git a/modules/loaders.py b/modules/loaders.py index b29679bd..4b76549b 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -84,7 +84,6 @@ loaders_and_params = OrderedDict({ 'no_flash_attn', 'no_xformers', 'no_sdpa', - 'exllamav2_info', 'model_draft', 'draft_max', 'ctx_size_draft', diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 28b7222d..33e152a0 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -68,7 +68,6 @@ def create_ui(): shared.gradio['enable_tp'] = gr.Checkbox(label="enable_tp", value=shared.args.enable_tp, info='Enable Tensor Parallelism (TP).') shared.gradio['cpp_runner'] = gr.Checkbox(label="cpp-runner", value=shared.args.cpp_runner, info='Enable inference with ModelRunnerCpp, which is faster than the default ModelRunner.') shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Set trust_remote_code=True while loading the tokenizer/model. To enable this option, start the web UI with the --trust-remote-code flag.', interactive=shared.args.trust_remote_code) - shared.gradio['exllamav2_info'] = gr.Markdown("ExLlamav2_HF is recommended over ExLlamav2 for better integration with extensions and more consistent sampling behavior across loaders.") shared.gradio['tensorrt_llm_info'] = gr.Markdown('* TensorRT-LLM has to be installed manually in a separate Python 3.10 environment at the moment. For a guide, consult the description of [this PR](https://github.com/oobabooga/text-generation-webui/pull/5715). \n\n* `ctx_size` is only used when `cpp-runner` is checked.\n\n* `cpp_runner` does not support streaming at the moment.') # Speculative decoding From 13a434f3518e381d04acd869ed3c0ba3d3823d34 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 08:06:07 -0700 Subject: [PATCH 53/90] Bump exllamav3 --- requirements/full/requirements.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 2 +- requirements/full/requirements_apple_silicon.txt | 2 +- requirements/full/requirements_noavx2.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index 3b50c674..ac89f45b 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -32,8 +32,8 @@ tiktoken # CUDA wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index ba23ea9c..6abdb1a4 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -31,5 +31,5 @@ tiktoken # Mac wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7-py3-none-any.whl +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index c245ab74..682c6a47 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -32,5 +32,5 @@ tiktoken https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7-py3-none-any.whl +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index d8bbf6d1..1e185079 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -32,8 +32,8 @@ tiktoken # CUDA wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a7/exllamav3-0.0.1a7+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" From ed6e16191da79523c5cabfd927130b307b3b54b9 Mon Sep 17 00:00:00 2001 From: Scott Z Date: Thu, 8 May 2025 11:21:52 -0400 Subject: [PATCH 54/90] Docker fix for NVIDIA (#6964) --- docker/nvidia/Dockerfile | 2 +- docker/nvidia/docker-compose.yml | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/docker/nvidia/Dockerfile b/docker/nvidia/Dockerfile index 900a4329..82594a26 100644 --- a/docker/nvidia/Dockerfile +++ b/docker/nvidia/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /home/app/ RUN git clone https://github.com/oobabooga/text-generation-webui.git WORKDIR /home/app/text-generation-webui RUN GPU_CHOICE=A LAUNCH_AFTER_INSTALL=FALSE INSTALL_EXTENSIONS=TRUE ./start_linux.sh --verbose -COPY CMD_FLAGS.txt /home/app/text-generation-webui/ +COPY /user_data/CMD_FLAGS.txt /home/app/text-generation-webui/user_data EXPOSE ${CONTAINER_PORT:-7860} ${CONTAINER_API_PORT:-5000} ${CONTAINER_API_STREAM_PORT:-5005} WORKDIR /home/app/text-generation-webui # set umask to ensure group read / write at runtime diff --git a/docker/nvidia/docker-compose.yml b/docker/nvidia/docker-compose.yml index 0392078e..23d5cacc 100644 --- a/docker/nvidia/docker-compose.yml +++ b/docker/nvidia/docker-compose.yml @@ -31,17 +31,7 @@ services: stdin_open: true tty: true volumes: - - ./cache:/home/app/text-generation-webui/cache - - ./characters:/home/app/text-generation-webui/characters - - ./extensions:/home/app/text-generation-webui/extensions - - ./loras:/home/app/text-generation-webui/loras - - ./logs:/home/app/text-generation-webui/logs - - ./models:/home/app/text-generation-webui/models - - ./presets:/home/app/text-generation-webui/presets - - ./prompts:/home/app/text-generation-webui/prompts - - ./softprompts:/home/app/text-generation-webui/softprompts - - ./training:/home/app/text-generation-webui/training - - ./cloudflared:/etc/cloudflared + - ./user_data:/home/app/text-generation-webui/user_data deploy: resources: reservations: From fa960496d554ece24c06088607692fa7b874ff5b Mon Sep 17 00:00:00 2001 From: Jonas Date: Thu, 8 May 2025 17:30:27 +0200 Subject: [PATCH 55/90] Tools support for OpenAI compatible API (#6827) --- extensions/openai/completions.py | 73 +++++++++++++++++++++---- extensions/openai/typing.py | 47 +++++++++++++++- extensions/openai/utils.py | 94 ++++++++++++++++++++++++++++++++ modules/chat.py | 12 ++-- 4 files changed, 209 insertions(+), 17 deletions(-) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index a7d8b4e4..ed0bcc40 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -1,11 +1,14 @@ import copy import time +import json from collections import deque import tiktoken from extensions.openai.errors import InvalidRequestError -from extensions.openai.utils import debug_msg +from extensions.openai.utils import debug_msg, getToolCallId, parseToolCall +from extensions.openai.typing import ToolDefinition +from pydantic import ValidationError from modules import shared from modules.chat import ( generate_chat_prompt, @@ -99,19 +102,24 @@ def convert_history(history): user_input = content user_input_last = True if current_message: - chat_dialogue.append([current_message, '']) + chat_dialogue.append([current_message, '', '']) current_message = "" current_message = content elif role == "assistant": + if "tool_calls" in entry and isinstance(entry["tool_calls"], list) and len(entry["tool_calls"]) > 0 and content.strip() == "": + continue # skip tool calls current_reply = content user_input_last = False if current_message: - chat_dialogue.append([current_message, current_reply]) + chat_dialogue.append([current_message, current_reply, '']) current_message = "" current_reply = "" else: - chat_dialogue.append(['', current_reply]) + chat_dialogue.append(['', current_reply, '']) + elif role == "tool": + user_input_last = False + chat_dialogue.append(['', '', content]) elif role == "system": system_message += f"\n{content}" if system_message else content @@ -131,6 +139,10 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p if 'messages' not in body: raise InvalidRequestError(message="messages is required", param='messages') + tools = None + if 'tools' in body and body['tools'] is not None and isinstance(body['tools'], list) and len(body['tools']) > 0: + tools = validateTools(body['tools']) # raises InvalidRequestError if validation fails + messages = body['messages'] for m in messages: if 'role' not in m: @@ -188,6 +200,7 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p 'custom_system_message': custom_system_message, 'chat_template_str': chat_template_str, 'chat-instruct_command': chat_instruct_command, + 'tools': tools, 'history': history, 'stream': stream }) @@ -200,7 +213,7 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p requested_model = generate_params.pop('model') logprob_proc = generate_params.pop('logprob_proc', None) - def chat_streaming_chunk(content): + def chat_streaming_chunk(content, chunk_tool_calls=None): # begin streaming chunk = { "id": cmpl_id, @@ -210,7 +223,7 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p resp_list: [{ "index": 0, "finish_reason": None, - "delta": {'role': 'assistant', 'content': content}, + "delta": {'role': 'assistant', 'content': content, 'tool_calls': chunk_tool_calls}, }], } @@ -219,6 +232,7 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} # else: # chunk[resp_list][0]["logprobs"] = None + return chunk # generate reply ####################################### @@ -227,8 +241,6 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p yield {'prompt': prompt} return - debug_msg({'prompt': prompt, 'generate_params': generate_params}) - if stream: yield chat_streaming_chunk('') @@ -238,8 +250,23 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p answer = '' seen_content = '' + tool_calls = [] + end_last_tool_call = 0 + supported_tools = [x["function"]["name"] for x in tools] if tools is not None else None + for a in generator: answer = a['internal'][-1][1] + + if supported_tools is not None: + tool_call = parseToolCall(answer[end_last_tool_call:], supported_tools) if len(answer) > 0 else [] + if len(tool_call) > 0: + for tc in tool_call: + tc["id"] = getToolCallId() + tc["index"] = str(len(tool_calls)) + tc["function"]["arguments"] = json.dumps(tc["function"]["arguments"]) + tool_calls.append(tc) + end_last_tool_call = len(answer) + if stream: len_seen = len(seen_content) new_content = answer[len_seen:] @@ -247,18 +274,25 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p if not new_content or chr(0xfffd) in new_content: # partial unicode character, don't send it yet. continue - seen_content = answer chunk = chat_streaming_chunk(new_content) + + seen_content = answer yield chunk + # stop generation if tool_calls were generated previously + if len(tool_calls) > 0: + break + token_count = len(encode(prompt)[0]) completion_token_count = len(encode(answer)[0]) stop_reason = "stop" + if len(tool_calls) > 0: + stop_reason = "tool_calls" if token_count + completion_token_count >= generate_params['truncation_length'] or completion_token_count >= generate_params['max_new_tokens']: stop_reason = "length" if stream: - chunk = chat_streaming_chunk('') + chunk = chat_streaming_chunk('', tool_calls) chunk[resp_list][0]['finish_reason'] = stop_reason chunk['usage'] = { "prompt_tokens": token_count, @@ -276,7 +310,8 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p resp_list: [{ "index": 0, "finish_reason": stop_reason, - "message": {"role": "assistant", "content": answer} + "message": {"role": "assistant", "content": answer}, + "tool_calls": tool_calls }], "usage": { "prompt_tokens": token_count, @@ -465,3 +500,19 @@ def completions(body: dict, is_legacy: bool = False) -> dict: def stream_completions(body: dict, is_legacy: bool = False): for resp in completions_common(body, is_legacy, stream=True): yield resp + + +def validateTools(tools: list[dict]): + # Validate each tool definition in the JSON array + valid_tools = None + for idx in range(len(tools)): + tool = tools[idx] + try: + tool_definition = ToolDefinition(**tool) + if valid_tools is None: + valid_tools = [] + valid_tools.append(tool) + except ValidationError: + raise InvalidRequestError(message=f"Invalid tool specification at index {idx}.", param='tools') + + return valid_tools diff --git a/extensions/openai/typing.py b/extensions/openai/typing.py index b1979cbc..b28ebb4e 100644 --- a/extensions/openai/typing.py +++ b/extensions/openai/typing.py @@ -1,8 +1,8 @@ import json import time -from typing import Dict, List +from typing import Dict, List, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, validator class GenerationOptions(BaseModel): @@ -54,6 +54,48 @@ class GenerationOptions(BaseModel): grammar_string: str = "" +class ToolDefinition(BaseModel): + function: 'ToolFunction' + type: str + + +class ToolFunction(BaseModel): + description: str + name: str + parameters: 'ToolParameters' + + +class ToolParameters(BaseModel): + properties: Optional[Dict[str, 'ToolProperty']] = None + required: Optional[list[str]] = None + type: str + description: Optional[str] = None + + +class ToolProperty(BaseModel): + description: Optional[str] = None + type: Optional[str] = None # we are faced with definitions like anyOf, e.g. {'type': 'function', 'function': {'name': 'git_create_branch', 'description': 'Creates a new branch from an optional base branch', 'parameters': {'type': 'object', 'properties': {'repo_path': {'title': 'Repo Path', 'type': 'string'}, 'branch_name': {'title': 'Branch Name', 'type': 'string'}, 'base_branch': {'anyOf': [{'type': 'string'}, {'type': 'null'}], 'default': None, 'title': 'Base Branch'}}, 'required': ['repo_path', 'branch_name'], 'title': 'GitCreateBranch'}}} + + +class FunctionCall(BaseModel): + name: str + arguments: Optional[str] = None + parameters: Optional[str] = None + + @validator('arguments', allow_reuse=True) + def checkPropertyArgsOrParams(cls, v, values, **kwargs): + if not v and not values.get('parameters'): + raise ValueError("At least one of 'arguments' or 'parameters' must be provided as property in FunctionCall type") + return v + + +class ToolCall(BaseModel): + id: str + index: int + type: str + function: FunctionCall + + class CompletionRequestParams(BaseModel): model: str | None = Field(default=None, description="Unused parameter. To change the model, use the /v1/internal/model/load endpoint.") prompt: str | List[str] @@ -92,6 +134,7 @@ class ChatCompletionRequestParams(BaseModel): frequency_penalty: float | None = 0 function_call: str | dict | None = Field(default=None, description="Unused parameter.") functions: List[dict] | None = Field(default=None, description="Unused parameter.") + tools: List[dict] | None = Field(default=None, description="Tools signatures passed via MCP.") logit_bias: dict | None = None max_tokens: int | None = None n: int | None = Field(default=1, description="Unused parameter.") diff --git a/extensions/openai/utils.py b/extensions/openai/utils.py index 2b414769..8cb856ff 100644 --- a/extensions/openai/utils.py +++ b/extensions/openai/utils.py @@ -1,6 +1,9 @@ import base64 import os import time +import json +import random +import re import traceback from typing import Callable, Optional @@ -52,3 +55,94 @@ def _start_cloudflared(port: int, tunnel_id: str, max_attempts: int = 3, on_star time.sleep(3) raise Exception('Could not start cloudflared.') + + +def getToolCallId() -> str: + letter_bytes = "abcdefghijklmnopqrstuvwxyz0123456789" + b = [random.choice(letter_bytes) for _ in range(8)] + return "call_" + "".join(b).lower() + + +def checkAndSanitizeToolCallCandidate(candidate_dict: dict, tool_names: list[str]): + # check if property 'function' exists and is a dictionary, otherwise adapt dict + if 'function' not in candidate_dict and 'name' in candidate_dict and isinstance(candidate_dict['name'], str): + candidate_dict = {"type": "function", "function": candidate_dict} + if 'function' in candidate_dict and isinstance(candidate_dict['function'], str): + candidate_dict['name'] = candidate_dict['function'] + del candidate_dict['function'] + candidate_dict = {"type": "function", "function": candidate_dict} + if 'function' in candidate_dict and isinstance(candidate_dict['function'], dict): + # check if 'name' exists within 'function' and is part of known tools + if 'name' in candidate_dict['function'] and candidate_dict['function']['name'] in tool_names: + candidate_dict["type"] = "function" # ensure required property 'type' exists and has the right value + # map property 'parameters' used by some older models to 'arguments' + if "arguments" not in candidate_dict["function"] and "parameters" in candidate_dict["function"]: + candidate_dict["function"]["arguments"] = candidate_dict["function"]["parameters"] + del candidate_dict["function"]["parameters"] + return candidate_dict + return None + + +def parseToolCall(answer: str, tool_names: list[str]): + matches = [] + + # abort on very short answers to save computation cycles + if len(answer) < 10: + return matches + + # Define the regex pattern to find the JSON content wrapped in , , , and other tags observed from various models + patterns = [ r"(```[^\n]*)\n(.*?)```", r"<([^>]+)>(.*?)" ] + + for pattern in patterns: + for match in re.finditer(pattern, answer, re.DOTALL): + # print(match.group(2)) + if match.group(2) is None: + continue + # remove backtick wraps if present + candidate = re.sub(r"^```(json|xml|python[^\n]*)\n", "", match.group(2).strip()) + candidate = re.sub(r"```$", "", candidate.strip()) + # unwrap inner tags + candidate = re.sub(pattern, r"\2", candidate.strip(), flags=re.DOTALL) + # llm might have generated multiple json objects separated by linebreaks, check for this pattern and try parsing each object individually + if re.search(r"\}\s*\n\s*\{", candidate) is not None: + candidate = re.sub(r"\}\s*\n\s*\{", "},\n{", candidate) + if not candidate.strip().startswith("["): + candidate = "[" + candidate + "]" + + candidates = [] + try: + # parse the candidate JSON into a dictionary + candidates = json.loads(candidate) + if not isinstance(candidates, list): + candidates = [candidates] + except json.JSONDecodeError: + # Ignore invalid JSON silently + continue + + for candidate_dict in candidates: + checked_candidate = checkAndSanitizeToolCallCandidate(candidate_dict, tool_names) + if checked_candidate is not None: + matches.append(checked_candidate) + + # last resort if nothing has been mapped: LLM might have produced plain json tool call without xml-like tags + if len(matches) == 0: + try: + candidate = answer + # llm might have generated multiple json objects separated by linebreaks, check for this pattern and try parsing each object individually + if re.search(r"\}\s*\n\s*\{", candidate) is not None: + candidate = re.sub(r"\}\s*\n\s*\{", "},\n{", candidate) + if not candidate.strip().startswith("["): + candidate = "[" + candidate + "]" + # parse the candidate JSON into a dictionary + candidates = json.loads(candidate) + if not isinstance(candidates, list): + candidates = [candidates] + for candidate_dict in candidates: + checked_candidate = checkAndSanitizeToolCallCandidate(candidate_dict, tool_names) + if checked_candidate is not None: + matches.append(checked_candidate) + except json.JSONDecodeError: + # Ignore invalid JSON silently + pass + + return matches diff --git a/modules/chat.py b/modules/chat.py index feac6bdd..b524b1b9 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -145,7 +145,7 @@ def generate_chat_prompt(user_input, state, **kwargs): instruct_renderer = partial( instruction_template.render, builtin_tools=None, - tools=None, + tools=state['tools'] if 'tools' in state else None, tools_in_user_message=False, add_generation_prompt=False ) @@ -171,9 +171,13 @@ def generate_chat_prompt(user_input, state, **kwargs): messages.append({"role": "system", "content": context}) insert_pos = len(messages) - for user_msg, assistant_msg in reversed(history): - user_msg = user_msg.strip() - assistant_msg = assistant_msg.strip() + for entry in reversed(history): + user_msg = entry[0].strip() + assistant_msg = entry[1].strip() + tool_msg = entry[2].strip() if len(entry) > 2 else '' + + if tool_msg: + messages.insert(insert_pos, {"role": "tool", "content": tool_msg}) if assistant_msg: messages.insert(insert_pos, {"role": "assistant", "content": assistant_msg}) From a1b3307b6636b13373e8f399690cb3b782854d2c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 08:58:14 -0700 Subject: [PATCH 56/90] Bump llama.cpp --- requirements/full/requirements.txt | 4 ++-- requirements/full/requirements_amd.txt | 4 ++-- requirements/full/requirements_amd_noavx2.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 4 ++-- requirements/full/requirements_apple_silicon.txt | 6 +++--- requirements/full/requirements_cpu_only.txt | 4 ++-- requirements/full/requirements_cpu_only_noavx2.txt | 4 ++-- requirements/full/requirements_noavx2.txt | 4 ++-- requirements/portable/requirements.txt | 4 ++-- requirements/portable/requirements_apple_intel.txt | 4 ++-- requirements/portable/requirements_apple_silicon.txt | 6 +++--- requirements/portable/requirements_cpu_only.txt | 4 ++-- requirements/portable/requirements_cpu_only_noavx2.txt | 4 ++-- requirements/portable/requirements_noavx2.txt | 4 ++-- requirements/portable/requirements_vulkan.txt | 4 ++-- requirements/portable/requirements_vulkan_noavx2.txt | 4 ++-- 16 files changed, 34 insertions(+), 34 deletions(-) diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index ac89f45b..3a059c91 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt index 431cd740..ebc33216 100644 --- a/requirements/full/requirements_amd.txt +++ b/requirements/full/requirements_amd.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt index 0c581f86..8ec6898f 100644 --- a/requirements/full/requirements_amd_noavx2.txt +++ b/requirements/full/requirements_amd_noavx2.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index 6abdb1a4..afc869c8 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index 682c6a47..8d7d29b7 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -29,8 +29,8 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt index ac277d61..d69aae18 100644 --- a/requirements/full/requirements_cpu_only.txt +++ b/requirements/full/requirements_cpu_only.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt index cc412d33..540c9ac8 100644 --- a/requirements/full/requirements_cpu_only_noavx2.txt +++ b/requirements/full/requirements_cpu_only_noavx2.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index 1e185079..3bb5a74a 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt index 1240d335..95319d75 100644 --- a/requirements/portable/requirements.txt +++ b/requirements/portable/requirements.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt index 6b165b7c..4b49b4e1 100644 --- a/requirements/portable/requirements_apple_intel.txt +++ b/requirements/portable/requirements_apple_intel.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt index 1b2b5cf2..a6ebda30 100644 --- a/requirements/portable/requirements_apple_silicon.txt +++ b/requirements/portable/requirements_apple_silicon.txt @@ -15,6 +15,6 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt index 2793d743..bb5ba8ad 100644 --- a/requirements/portable/requirements_cpu_only.txt +++ b/requirements/portable/requirements_cpu_only.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt index 6d7316a6..3d17dd49 100644 --- a/requirements/portable/requirements_cpu_only_noavx2.txt +++ b/requirements/portable/requirements_cpu_only_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt index e56eba08..ff9fa04c 100644 --- a/requirements/portable/requirements_noavx2.txt +++ b/requirements/portable/requirements_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt index a7f8c703..e17f8ce7 100644 --- a/requirements/portable/requirements_vulkan.txt +++ b/requirements/portable/requirements_vulkan.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_vulkan_noavx2.txt b/requirements/portable/requirements_vulkan_noavx2.txt index 5b427fd2..dd01b3a8 100644 --- a/requirements/portable/requirements_vulkan_noavx2.txt +++ b/requirements/portable/requirements_vulkan_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.9.0/llama_cpp_binaries-0.9.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" From 1c7209a725c8811f2d4d2325007b2e871c5af020 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 09:46:43 -0700 Subject: [PATCH 57/90] Save the chat history periodically during streaming --- modules/chat.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/chat.py b/modules/chat.py index b524b1b9..403d05e1 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -5,6 +5,7 @@ import html import json import pprint import re +import time from datetime import datetime from functools import partial from pathlib import Path @@ -485,10 +486,16 @@ def generate_chat_reply_wrapper(text, state, regenerate=False, _continue=False): send_dummy_reply(state['start_with'], state) history = state['history'] + last_save_time = time.monotonic() + save_interval = 8 for i, history in enumerate(generate_chat_reply(text, state, regenerate, _continue, loading_message=True, for_ui=True)): yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu']), history - if i == 0: + + current_time = time.monotonic() + # Save on first iteration or if save_interval seconds have passed + if i == 0 or (current_time - last_save_time) >= save_interval: save_history(history, state['unique_id'], state['character_menu'], state['mode']) + last_save_time = current_time save_history(history, state['unique_id'], state['character_menu'], state['mode']) From 3bc2ec2b119c058446f9e9600213c75302a4ac4f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 10:34:09 -0700 Subject: [PATCH 58/90] Fix #6965 --- one_click.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/one_click.py b/one_click.py index cb16b813..482a6aa9 100644 --- a/one_click.py +++ b/one_click.py @@ -126,7 +126,7 @@ def check_env(): sys.exit(1) # Ensure this is a new environment and not the base environment - if os.environ["CONDA_DEFAULT_ENV"] == "base": + if os.environ.get("CONDA_DEFAULT_ENV", "") == "base": print("Create an environment for this project and activate it. Exiting...") sys.exit(1) From 9ea2a69210ab5658ba8daf6d7d604589de5fc741 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 10:41:25 -0700 Subject: [PATCH 59/90] llama.cpp: Add --no-webui to the llama-server command --- modules/llama_cpp_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index d8d2f61b..1046969a 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -261,6 +261,7 @@ class LlamaServer: "--gpu-layers", str(shared.args.gpu_layers), "--batch-size", str(shared.args.batch_size), "--port", str(self.port), + "--no-webui", ] if shared.args.flash_attn: From bf7e4a4597b6492b4c440d32a8afbda59d4ef035 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 16:12:07 -0700 Subject: [PATCH 60/90] Docs: Add a tool/function calling example (from https://github.com/oobabooga/text-generation-webui/pull/6827#issuecomment-2854716960) --- docs/12 - OpenAI API.md | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/docs/12 - OpenAI API.md b/docs/12 - OpenAI API.md index 364c6b09..db9befed 100644 --- a/docs/12 - OpenAI API.md +++ b/docs/12 - OpenAI API.md @@ -257,6 +257,85 @@ headers = { in any of the examples above. +#### Tool/Function Calling Example + +You need to use a model with tools support. The prompt will be automatically formatted using the model's Jinja2 template. + +Request: + +``` +curl http://127.0.0.1:5000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant." + }, + { + "role": "user", + "content": "What time is it currently in New York City?" + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "Get current time in a specific timezones", + "parameters": { + "type": "object", + "required": ["timezone"], + "properties": { + "timezone": { + "type": "string", + "description": "IANA timezone name (e.g., America/New_York, Europe/London). Use Europe/Berlin as local timezone if no timezone provided by the user." + } + } + } + } + } + ] + }' +``` + +Sample response: + +``` +{ + "id": "chatcmpl-1746532051477984256", + "object": "chat.completion", + "created": 1746532051, + "model": "qwen2.5-coder-14b-instruct-q4_k_m.gguf", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "```xml\n\n{\n \"name\": \"get_current_time\",\n \"arguments\": {\n \"timezone\": \"America/New_York\"\n }\n}\n\n```" + }, + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_current_time", + "arguments": "{\"timezone\": \"America/New_York\"}" + }, + "id": "call_52ij07mh", + "index": "0" + } + ] + } + ], + "usage": { + "prompt_tokens": 224, + "completion_tokens": 38, + "total_tokens": 262 + } +} +``` + ### Environment variables The following environment variables can be used (they take precedence over everything else): From f8ef6e09af5d2e28cf67d1eea165591e156ac9d2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 18:19:04 -0700 Subject: [PATCH 61/90] UI: Make ctx-size a slider --- modules/ui_model_menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 33e152a0..d4d9b8b1 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -51,8 +51,8 @@ def create_ui(): with gr.Row(): with gr.Column(): shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') + shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=256, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') - shared.gradio['ctx_size'] = gr.Number(label='ctx-size', precision=0, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model. Common values: 2048, 4096, 8192, 16384, 32768, 65536, 131072.') shared.gradio['cache_type'] = gr.Dropdown(label="cache_type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) From 512bc2d0e02bef2434370c2317bcf56e50f0513f Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 8 May 2025 23:43:55 -0700 Subject: [PATCH 62/90] UI: Update some labels --- modules/ui_model_menu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index d4d9b8b1..1e27255b 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -53,12 +53,12 @@ def create_ui(): shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=256, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') - shared.gradio['cache_type'] = gr.Dropdown(label="cache_type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') + shared.gradio['cache_type'] = gr.Dropdown(label="cache-type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) with gr.Column(): - shared.gradio['flash_attn'] = gr.Checkbox(label="flash_attn", value=shared.args.flash_attn, info='Use flash-attention.') - shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming_llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') + shared.gradio['flash_attn'] = gr.Checkbox(label="flash-attn", value=shared.args.flash_attn, info='Use flash-attention.') + shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming-llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) shared.gradio['load_in_4bit'] = gr.Checkbox(label="load-in-4bit", value=shared.args.load_in_4bit) shared.gradio['torch_compile'] = gr.Checkbox(label="torch-compile", value=shared.args.torch_compile, info='Compile the model with torch.compile for improved performance.') From 2bde625d5716355b30fdd414c9b104812b101ed1 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 9 May 2025 00:19:25 -0700 Subject: [PATCH 63/90] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6cc84c50..0833f9b0 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Its goal is to become the [AUTOMATIC1111/stable-diffusion-webui](https://github. - Free-form text generation in the Default/Notebook tabs without being limited to chat turns. You can send formatted conversations from the Chat tab to these. - Multiple sampling parameters and generation options for sophisticated text generation control. - Switch between different models easily in the UI without restarting, with fine control over settings. -- OpenAI-compatible API with Chat and Completions endpoints – see [examples](https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API#examples). +- OpenAI-compatible API with Chat and Completions endpoints, including tool-calling support – see [examples](https://github.com/oobabooga/text-generation-webui/wiki/12-%E2%80%90-OpenAI-API#examples). - 100% offline and private, with zero telemetry, external resources, or remote update requests. - Extension support, with numerous built-in and user-contributed extensions available. See the [wiki](https://github.com/oobabooga/text-generation-webui/wiki/07-%E2%80%90-Extensions) and [extensions directory](https://github.com/oobabooga/text-generation-webui-extensions) for details. From 8984e95c671c262b1667805895d317a9ffe9cd0a Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 9 May 2025 07:21:05 -0700 Subject: [PATCH 64/90] UI: More friendly message when no model is loaded --- modules/logits.py | 7 ++++--- modules/text_generation.py | 5 +++-- modules/utils.py | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/logits.py b/modules/logits.py index 32aef7ae..56a20572 100644 --- a/modules/logits.py +++ b/modules/logits.py @@ -7,6 +7,7 @@ from modules import models, shared from modules.logging_colors import logger from modules.models import load_model from modules.text_generation import generate_reply +from modules.utils import check_model_loaded global_scores = None @@ -33,9 +34,9 @@ def get_next_logits(*args, **kwargs): def _get_next_logits(prompt, state, use_samplers, previous, top_logits=25, return_dict=False): - if shared.model is None: - logger.error("No model is loaded! Select one in the Model tab.") - return 'Error: No model is loaded1 Select one in the Model tab.', previous + model_is_loaded, error_message = check_model_loaded() + if not model_is_loaded: + return error_message, previous # llama.cpp case if shared.model.__class__.__name__ == 'LlamaServer': diff --git a/modules/text_generation.py b/modules/text_generation.py index 7e48a2f6..c0c0350d 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -14,6 +14,7 @@ from modules.callbacks import Iteratorize from modules.extensions import apply_extensions from modules.html_generator import generate_basic_html from modules.logging_colors import logger +from modules.utils import check_model_loaded def generate_reply(*args, **kwargs): @@ -34,8 +35,8 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap # Find the appropriate generation function generate_func = apply_extensions('custom_generate_reply') if generate_func is None: - if shared.model_name == 'None' or shared.model is None: - logger.error("No model is loaded! Select one in the Model tab.") + model_is_loaded, error_message = check_model_loaded() + if not model_is_loaded: yield '' return diff --git a/modules/utils.py b/modules/utils.py index 77324139..0e390d08 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -72,6 +72,20 @@ def natural_keys(text): return [atoi(c) for c in re.split(r'(\d+)', text)] +def check_model_loaded(): + if shared.model_name == 'None' or shared.model is None: + if len(get_available_models()) <= 1: + error_msg = "No model is loaded.\n\nTo get started:\n1) Place a GGUF file in your user_data/models folder\n2) Go to the Model tab and select it" + logger.error(error_msg) + return False, error_msg + else: + error_msg = "No model is loaded. Please select one in the Model tab." + logger.error(error_msg) + return False, error_msg + + return True, None + + def get_available_models(): # Get all GGUF files gguf_files = get_available_ggufs() From 4920981b140862f3b085f614b83269e6ac228605 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 9 May 2025 20:35:38 -0700 Subject: [PATCH 65/90] UI: Remove the typing cursor --- modules/chat.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index 403d05e1..b83c4bfe 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -399,16 +399,13 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess # Extract the reply if state['mode'] in ['chat', 'chat-instruct']: - visible_reply = re.sub("(||{{user}})", state['name1'], reply + '▍') + visible_reply = re.sub("(||{{user}})", state['name1'], reply) else: - visible_reply = reply + '▍' + visible_reply = reply visible_reply = html.escape(visible_reply) if shared.stop_everything: - if output['visible'][-1][1].endswith('▍'): - output['visible'][-1][1] = output['visible'][-1][1][:-1] - output['visible'][-1][1] = apply_extensions('output', output['visible'][-1][1], state, is_chat=True) yield output return @@ -424,9 +421,6 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess if is_stream: yield output - if output['visible'][-1][1].endswith('▍'): - output['visible'][-1][1] = output['visible'][-1][1][:-1] - output['visible'][-1][1] = apply_extensions('output', output['visible'][-1][1], state, is_chat=True) yield output From 47d47585095da3a76988eabe52765a332a668d55 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 10 May 2025 17:46:00 -0700 Subject: [PATCH 66/90] Fix #6970 --- modules/shared.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index f2698bd2..4e0a20db 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -128,9 +128,9 @@ group.add_argument('--extra-flags', type=str, default=None, help='Extra flags to group.add_argument('--streaming-llm', action='store_true', help='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') # Cache -group = parser.add_argument_group('Context and cache management') +group = parser.add_argument_group('Context and cache') group.add_argument('--ctx-size', '--n_ctx', '--max_seq_len', type=int, default=8192, metavar='N', help='Context size in tokens.') -group.add_argument('--cache_type', type=str, default='fp16', help='KV cache type; valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8 (can specify k_bits and v_bits separately, e.g. q4_q8).') +group.add_argument('--cache-type', '--cache_type', type=str, default='fp16', metavar='N', help='KV cache type; valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8 (can specify k_bits and v_bits separately, e.g. q4_q8).') # Speculative decoding group = parser.add_argument_group('Speculative decoding') @@ -159,10 +159,6 @@ group.add_argument('--hqq-backend', type=str, default='PYTORCH_COMPILE', help='B group = parser.add_argument_group('TensorRT-LLM') group.add_argument('--cpp-runner', action='store_true', help='Use the ModelRunnerCpp runner, which is faster than the default ModelRunner but doesn\'t support streaming yet.') -# Cache -group = parser.add_argument_group('Cache') -group.add_argument('--cache_type', type=str, default='fp16', help='KV cache type; valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4.') - # DeepSpeed group = parser.add_argument_group('DeepSpeed') group.add_argument('--deepspeed', action='store_true', help='Enable the use of DeepSpeed ZeRO-3 for inference via the Transformers integration.') From 006a866079d4a719f4405efdbbe18c03e106c541 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 10 May 2025 17:55:48 -0700 Subject: [PATCH 67/90] Fix API failing to cancel streams (attempt), closes #6966 --- extensions/openai/script.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index a995da9d..66f38501 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -118,6 +118,7 @@ async def openai_completions(request: Request, request_data: CompletionRequest): for resp in response: disconnected = await request.is_disconnected() if disconnected: + stop_everything_event() break yield {"data": json.dumps(resp)} @@ -141,6 +142,7 @@ async def openai_chat_completions(request: Request, request_data: ChatCompletion for resp in response: disconnected = await request.is_disconnected() if disconnected: + stop_everything_event() break yield {"data": json.dumps(resp)} From 0c5fa3728e8f0505692966f7a296e6561566c7bd Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 10 May 2025 19:12:40 -0700 Subject: [PATCH 68/90] Revert "Fix API failing to cancel streams (attempt), closes #6966" This reverts commit 006a866079d4a719f4405efdbbe18c03e106c541. --- extensions/openai/script.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 66f38501..a995da9d 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -118,7 +118,6 @@ async def openai_completions(request: Request, request_data: CompletionRequest): for resp in response: disconnected = await request.is_disconnected() if disconnected: - stop_everything_event() break yield {"data": json.dumps(resp)} @@ -142,7 +141,6 @@ async def openai_chat_completions(request: Request, request_data: ChatCompletion for resp in response: disconnected = await request.is_disconnected() if disconnected: - stop_everything_event() break yield {"data": json.dumps(resp)} From e7ac06c1694024594450437f3b899e32ab2ce6e4 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 10 May 2025 19:20:04 -0700 Subject: [PATCH 69/90] New attempt --- modules/llama_cpp_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 1046969a..615f29ad 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -146,8 +146,9 @@ class LlamaServer: pprint.PrettyPrinter(indent=4, sort_dicts=False).pprint(printable_payload) print() - # Make a direct request with streaming enabled using a context manager - with self.session.post(url, json=payload, stream=True) as response: + # Make a request with streaming enabled + response = self.session.post(url, json=payload, stream=True) + try: response.raise_for_status() # Raise an exception for HTTP errors full_text = "" @@ -185,6 +186,9 @@ class LlamaServer: print(f"Problematic line: {line}") continue + finally: + response.close() + def generate(self, prompt, state): output = "" for output in self.generate_with_streaming(prompt, state): From 62c774bf24d35a1ebdcdb9927f8a6c6ae3949c82 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 13 May 2025 06:42:25 -0700 Subject: [PATCH 70/90] Revert "New attempt" This reverts commit e7ac06c1694024594450437f3b899e32ab2ce6e4. --- modules/llama_cpp_server.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 615f29ad..1046969a 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -146,9 +146,8 @@ class LlamaServer: pprint.PrettyPrinter(indent=4, sort_dicts=False).pprint(printable_payload) print() - # Make a request with streaming enabled - response = self.session.post(url, json=payload, stream=True) - try: + # Make a direct request with streaming enabled using a context manager + with self.session.post(url, json=payload, stream=True) as response: response.raise_for_status() # Raise an exception for HTTP errors full_text = "" @@ -186,9 +185,6 @@ class LlamaServer: print(f"Problematic line: {line}") continue - finally: - response.close() - def generate(self, prompt, state): output = "" for output in self.generate_with_streaming(prompt, state): From c375b6941395454bef52d9ac0e102c0de3f4d3ee Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 13 May 2025 11:23:33 -0700 Subject: [PATCH 71/90] API: Fix llama.cpp generating after disconnect, improve disconnect detection, fix deadlock on simultaneous requests --- extensions/openai/script.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index a995da9d..2b4f274f 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -14,6 +14,7 @@ from fastapi.requests import Request from fastapi.responses import JSONResponse from pydub import AudioSegment from sse_starlette import EventSourceResponse +from starlette.concurrency import iterate_in_threadpool import extensions.openai.completions as OAIcompletions import extensions.openai.images as OAIimages @@ -115,7 +116,7 @@ async def openai_completions(request: Request, request_data: CompletionRequest): async def generator(): async with streaming_semaphore: response = OAIcompletions.stream_completions(to_dict(request_data), is_legacy=is_legacy) - for resp in response: + async for resp in iterate_in_threadpool(response): disconnected = await request.is_disconnected() if disconnected: break @@ -125,7 +126,12 @@ async def openai_completions(request: Request, request_data: CompletionRequest): return EventSourceResponse(generator()) # SSE streaming else: - response = OAIcompletions.completions(to_dict(request_data), is_legacy=is_legacy) + response = await asyncio.to_thread( + OAIcompletions.completions, + to_dict(request_data), + is_legacy=is_legacy + ) + return JSONResponse(response) @@ -138,7 +144,7 @@ async def openai_chat_completions(request: Request, request_data: ChatCompletion async def generator(): async with streaming_semaphore: response = OAIcompletions.stream_chat_completions(to_dict(request_data), is_legacy=is_legacy) - for resp in response: + async for resp in iterate_in_threadpool(response): disconnected = await request.is_disconnected() if disconnected: break @@ -148,7 +154,12 @@ async def openai_chat_completions(request: Request, request_data: ChatCompletion return EventSourceResponse(generator()) # SSE streaming else: - response = OAIcompletions.chat_completions(to_dict(request_data), is_legacy=is_legacy) + response = await asyncio.to_thread( + OAIcompletions.chat_completions, + to_dict(request_data), + is_legacy=is_legacy + ) + return JSONResponse(response) From 3fa1a899aea3ff2700a20a8bc2da17202d3065e5 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 13 May 2025 12:07:59 -0700 Subject: [PATCH 72/90] UI: Fix gpu-layers being ignored (closes #6973) --- modules/loaders.py | 2 +- modules/models_settings.py | 2 +- modules/ui.py | 2 +- modules/ui_model_menu.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/loaders.py b/modules/loaders.py index 4b76549b..583b65c2 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -5,7 +5,7 @@ import gradio as gr loaders_and_params = OrderedDict({ 'llama.cpp': [ - 'n_gpu_layers', + 'gpu_layers', 'threads', 'threads_batch', 'batch_size', diff --git a/modules/models_settings.py b/modules/models_settings.py index ae589bb3..4418e3fb 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -67,7 +67,7 @@ def get_model_metadata(model): elif k.endswith('rope.scaling.factor'): model_settings['compress_pos_emb'] = metadata[k] elif k.endswith('block_count'): - model_settings['n_gpu_layers'] = metadata[k] + 1 + model_settings['gpu_layers'] = metadata[k] + 1 if 'tokenizer.chat_template' in metadata: template = metadata['tokenizer.chat_template'] diff --git a/modules/ui.py b/modules/ui.py index b3d4bccf..eeb6ce92 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -105,7 +105,7 @@ def list_model_elements(): 'filter_by_loader', 'loader', 'cpu_memory', - 'n_gpu_layers', + 'gpu_layers', 'threads', 'threads_batch', 'batch_size', diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 1e27255b..b63a127c 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -50,7 +50,7 @@ def create_ui(): gr.Markdown("## Main options") with gr.Row(): with gr.Column(): - shared.gradio['n_gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') + shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=256, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') shared.gradio['cache_type'] = gr.Dropdown(label="cache-type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') From 2826c60044a05f316510ef93546b5dbff59b3864 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 13 May 2025 14:45:46 -0700 Subject: [PATCH 73/90] Use logger for "Output generated in ..." messages --- modules/text_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/text_generation.py b/modules/text_generation.py index c0c0350d..00b9275a 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -470,7 +470,7 @@ def generate_reply_HF(question, original_question, state, stopping_strings=None, t1 = time.time() original_tokens = len(original_input_ids[0]) new_tokens = len(output) - (original_tokens if not shared.is_seq2seq else 0) - print(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {seed})') + logger.info(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {seed})') return @@ -499,7 +499,7 @@ def generate_reply_custom(question, original_question, state, stopping_strings=N t1 = time.time() original_tokens = len(encode(original_question)[0]) new_tokens = len(encode(original_question + reply)[0]) - original_tokens - print(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {state["seed"]})') + logger.info(f'Output generated in {(t1-t0):.2f} seconds ({new_tokens/(t1-t0):.2f} tokens/s, {new_tokens} tokens, context {original_tokens}, seed {state["seed"]})') return From 035cd3e2a906a6094d0f1f298df49c0152f1a2ee Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 13 May 2025 20:09:22 -0700 Subject: [PATCH 74/90] UI: Hide the extension install menu in portable builds --- modules/ui_session.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/ui_session.py b/modules/ui_session.py index 7cf9f6e6..a4eba667 100644 --- a/modules/ui_session.py +++ b/modules/ui_session.py @@ -23,11 +23,15 @@ def create_ui(): shared.gradio['bool_menu'] = gr.CheckboxGroup(choices=get_boolean_arguments(), value=get_boolean_arguments(active=True), label="Boolean command-line flags", elem_classes='checkboxgroup-table') with gr.Column(): - extension_name = gr.Textbox(lines=1, label='Install or update an extension', info='Enter the GitHub URL below and press Enter. For a list of extensions, see: https://github.com/oobabooga/text-generation-webui-extensions ⚠️ WARNING ⚠️ : extensions can execute arbitrary code. Make sure to inspect their source code before activating them.', interactive=not mu) - extension_status = gr.Markdown() + if not shared.args.portable: + extension_name = gr.Textbox(lines=1, label='Install or update an extension', info='Enter the GitHub URL below and press Enter. For a list of extensions, see: https://github.com/oobabooga/text-generation-webui-extensions ⚠️ WARNING ⚠️ : extensions can execute arbitrary code. Make sure to inspect their source code before activating them.', interactive=not mu) + extension_status = gr.Markdown() + else: + pass shared.gradio['theme_state'] = gr.Textbox(visible=False, value='dark' if shared.settings['dark_theme'] else 'light') - extension_name.submit(clone_or_pull_repository, extension_name, extension_status, show_progress=False) + if not shared.args.portable: + extension_name.submit(clone_or_pull_repository, extension_name, extension_status, show_progress=False) # Reset interface event shared.gradio['reset_interface'].click( From c4a715fd1e86e52e3350f8126847524b488a04e2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 13 May 2025 20:14:09 -0700 Subject: [PATCH 75/90] UI: Move the LoRA menu under "Other options" --- modules/ui_model_menu.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index b63a127c..81ad1a53 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -26,25 +26,12 @@ def create_ui(): with gr.Row(): with gr.Column(): with gr.Row(): - with gr.Column(): - with gr.Row(): - shared.gradio['model_menu'] = gr.Dropdown(choices=utils.get_available_models(), value=lambda: shared.model_name, label='Model', elem_classes='slim-dropdown', interactive=not mu) - ui.create_refresh_button(shared.gradio['model_menu'], lambda: None, lambda: {'choices': utils.get_available_models()}, 'refresh-button', interactive=not mu) - shared.gradio['load_model'] = gr.Button("Load", elem_classes='refresh-button', interactive=not mu) - shared.gradio['unload_model'] = gr.Button("Unload", elem_classes='refresh-button', interactive=not mu) - shared.gradio['save_model_settings'] = gr.Button("Save settings", elem_classes='refresh-button', interactive=not mu) + shared.gradio['model_menu'] = gr.Dropdown(choices=utils.get_available_models(), value=lambda: shared.model_name, label='Model', elem_classes='slim-dropdown', interactive=not mu) + ui.create_refresh_button(shared.gradio['model_menu'], lambda: None, lambda: {'choices': utils.get_available_models()}, 'refresh-button', interactive=not mu) + shared.gradio['load_model'] = gr.Button("Load", elem_classes='refresh-button', interactive=not mu) + shared.gradio['unload_model'] = gr.Button("Unload", elem_classes='refresh-button', interactive=not mu) + shared.gradio['save_model_settings'] = gr.Button("Save settings", elem_classes='refresh-button', interactive=not mu) - with gr.Column(): - if shared.args.portable: - pass - else: - with gr.Row(): - shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)', elem_classes='slim-dropdown', interactive=not mu) - ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) - shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) - - with gr.Row(): - with gr.Column(): shared.gradio['loader'] = gr.Dropdown(label="Model loader", choices=loaders.loaders_and_params.keys() if not shared.args.portable else ['llama.cpp'], value=None) with gr.Blocks(): gr.Markdown("## Main options") @@ -113,6 +100,11 @@ def create_ui(): shared.gradio['no_sdpa'] = gr.Checkbox(label="no_sdpa", value=shared.args.no_sdpa) shared.gradio['cfg_cache'] = gr.Checkbox(label="cfg-cache", value=shared.args.cfg_cache, info='Necessary to use CFG with this loader.') shared.gradio['no_use_fast'] = gr.Checkbox(label="no_use_fast", value=shared.args.no_use_fast, info='Set use_fast=False while loading the tokenizer.') + with gr.Row(): + shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)', elem_classes='slim-dropdown', interactive=not mu) + ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) + shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) + with gr.Column(): with gr.Tab("Download"): From 5534d01da0913d315709a6adacd075639a6cffec Mon Sep 17 00:00:00 2001 From: oobabooga Date: Fri, 16 May 2025 00:07:37 -0300 Subject: [PATCH 76/90] Estimate the VRAM for GGUF models + autoset `gpu-layers` (#6980) --- css/main.css | 14 +++- modules/llama_cpp_server.py | 3 + modules/models.py | 1 - modules/models_settings.py | 151 +++++++++++++++++++++++++++++++++++- modules/ui_model_menu.py | 16 +++- server.py | 12 +++ 6 files changed, 193 insertions(+), 4 deletions(-) diff --git a/css/main.css b/css/main.css index 30089aca..0902b184 100644 --- a/css/main.css +++ b/css/main.css @@ -569,7 +569,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { } .dark .message-body :not(pre) > code { - background-color: rgb(255 255 255 / 12.5%); + background-color: rgb(255 255 255 / 10%); } #chat-input { @@ -1386,3 +1386,15 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* { strong { font-weight: bold; } + +.min.svelte-1ybaih5 { + min-height: 0; +} + +#vram-info .value { + color: #008d00; +} + +.dark #vram-info .value { + color: #07ff07; +} diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 1046969a..3fc7a0cc 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -282,8 +282,10 @@ class LlamaServer: cmd.append("--no-kv-offload") if shared.args.row_split: cmd += ["--split-mode", "row"] + cache_type = "fp16" if shared.args.cache_type != "fp16" and shared.args.cache_type in llamacpp_valid_cache_types: cmd += ["--cache-type-k", shared.args.cache_type, "--cache-type-v", shared.args.cache_type] + cache_type = shared.args.cache_type if shared.args.compress_pos_emb != 1: cmd += ["--rope-freq-scale", str(1.0 / shared.args.compress_pos_emb)] if shared.args.rope_freq_base > 0: @@ -343,6 +345,7 @@ class LlamaServer: print(' '.join(str(item) for item in cmd[1:])) print() + logger.info(f"Using gpu_layers={shared.args.gpu_layers} | ctx_size={shared.args.ctx_size} | cache_type={cache_type}") # Start the server with pipes for output self.process = subprocess.Popen( cmd, diff --git a/modules/models.py b/modules/models.py index d0b0402a..9ecee803 100644 --- a/modules/models.py +++ b/modules/models.py @@ -71,7 +71,6 @@ def llama_cpp_server_loader(model_name): else: model_file = sorted(Path(f'{shared.args.model_dir}/{model_name}').glob('*.gguf'))[0] - logger.info(f"llama.cpp weights detected: \"{model_file}\"") try: model = LlamaServer(model_file) return model, model diff --git a/modules/models_settings.py b/modules/models_settings.py index 4418e3fb..a8e17594 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -1,7 +1,11 @@ +import functools import json import re +import subprocess +from math import exp from pathlib import Path +import gradio as gr import yaml from modules import chat, loaders, metadata_gguf, shared, ui @@ -216,7 +220,17 @@ def apply_model_settings_to_state(model, state): for k in model_settings: if k in state: - state[k] = model_settings[k] + if k == 'gpu_layers': + available_vram = get_nvidia_free_vram() + n_layers = model_settings[k] + if available_vram > 0: + tolerance = 906 + while n_layers > 0 and estimate_vram(model, n_layers, state['ctx_size'], state['cache_type']) > available_vram - tolerance: + n_layers -= 1 + + state[k] = gr.update(value=n_layers, maximum=model_settings[k]) + else: + state[k] = model_settings[k] return state @@ -277,3 +291,138 @@ def save_instruction_template(model, template): yield (f"Instruction template for `{model}` unset in `{p}`, as the value for template was `{template}`.") else: yield (f"Instruction template for `{model}` saved to `{p}` as `{template}`.") + + +@functools.lru_cache(maxsize=None) +def get_gguf_metadata_cached(model_file): + return metadata_gguf.load_metadata(model_file) + + +def get_model_size_mb(model_file: Path) -> float: + filename = model_file.name + + # Check for multipart pattern + match = re.match(r'(.+)-\d+-of-\d+\.gguf$', filename) + + if match: + # It's a multipart file, find all matching parts + base_pattern = match.group(1) + part_files = sorted(model_file.parent.glob(f'{base_pattern}-*-of-*.gguf')) + total_size = sum(p.stat().st_size for p in part_files) + else: + # Single part + total_size = model_file.stat().st_size + + return total_size / (1024 ** 2) # Return size in MB + + +def estimate_vram(gguf_file, gpu_layers, ctx_size, cache_type): + model_file = Path(f'{shared.args.model_dir}/{gguf_file}') + metadata = get_gguf_metadata_cached(model_file) + size_in_mb = get_model_size_mb(model_file) + + # Extract values from metadata + n_layers = None + n_kv_heads = None + embedding_dim = None + context_length = None + feed_forward_dim = None + + for key, value in metadata.items(): + if key.endswith('.block_count'): + n_layers = value + elif key.endswith('.attention.head_count_kv'): + n_kv_heads = value + elif key.endswith('.embedding_length'): + embedding_dim = value + elif key.endswith('.context_length'): + context_length = value + elif key.endswith('.feed_forward_length'): + feed_forward_dim = value + + if gpu_layers > n_layers: + gpu_layers = n_layers + + # Convert cache_type to numeric + if cache_type == 'q4_0': + cache_type = 4 + elif cache_type == 'q8_0': + cache_type = 8 + else: + cache_type = 16 + + # Derived features + size_per_layer = size_in_mb / max(n_layers, 1e-6) + context_per_layer = context_length / max(n_layers, 1e-6) + ffn_per_embedding = feed_forward_dim / max(embedding_dim, 1e-6) + kv_cache_factor = n_kv_heads * cache_type * ctx_size + + # Helper function for smaller + def smaller(x, y): + return 1 if x < y else 0 + + # Calculate VRAM using the model + # Details: https://oobabooga.github.io/blog/posts/gguf-vram-formula/ + vram = ( + (size_per_layer - 21.19195204848197) + * exp(0.0001047328491557063 * size_in_mb * smaller(ffn_per_embedding, 2.671096993407845)) + + 0.0006621544775632052 * context_per_layer + + 3.34664386576376e-05 * kv_cache_factor + ) * (1.363306170123392 + gpu_layers) + 1255.163594536052 + + return vram + + +def get_nvidia_free_vram(): + """ + Calculates the total free VRAM across all NVIDIA GPUs by parsing nvidia-smi output. + + Returns: + int: The total free VRAM in MiB summed across all detected NVIDIA GPUs. + Returns -1 if nvidia-smi command fails (not found, error, etc.). + Returns 0 if nvidia-smi succeeds but no GPU memory info found. + """ + try: + # Execute nvidia-smi command + result = subprocess.run( + ['nvidia-smi'], + capture_output=True, + text=True, + check=False + ) + + # Check if nvidia-smi returned an error + if result.returncode != 0: + return -1 + + # Parse the output for memory usage patterns + output = result.stdout + + # Find memory usage like "XXXXMiB / YYYYMiB" + # Captures used and total memory for each GPU + matches = re.findall(r"(\d+)\s*MiB\s*/\s*(\d+)\s*MiB", output) + + if not matches: + # No GPUs found in expected format + return 0 + + total_free_vram_mib = 0 + for used_mem_str, total_mem_str in matches: + try: + used_mib = int(used_mem_str) + total_mib = int(total_mem_str) + total_free_vram_mib += (total_mib - used_mib) + except ValueError: + # Skip malformed entries + pass + + return total_free_vram_mib + + except FileNotFoundError: + raise + # nvidia-smi not found (likely no NVIDIA drivers installed) + return -1 + except Exception: + raise + # Handle any other unexpected exceptions + return -1 diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 81ad1a53..2353f39c 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -11,6 +11,7 @@ from modules.LoRA import add_lora_to_model from modules.models import load_model, unload_model from modules.models_settings import ( apply_model_settings_to_state, + estimate_vram, get_model_metadata, save_instruction_template, save_model_settings, @@ -44,6 +45,7 @@ def create_ui(): shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) with gr.Column(): + shared.gradio['vram_info'] = gr.HTML(value=lambda: estimate_vram_wrapper(shared.args.model, shared.args.gpu_layers, shared.args.ctx_size, shared.args.cache_type)) shared.gradio['flash_attn'] = gr.Checkbox(label="flash-attn", value=shared.args.flash_attn, info='Use flash-attention.') shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming-llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) @@ -105,7 +107,6 @@ def create_ui(): ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) - with gr.Column(): with gr.Tab("Download"): shared.gradio['custom_model_menu'] = gr.Textbox(label="Download model or LoRA", info="Enter the Hugging Face username/model path, for instance: facebook/galactica-125m. To specify a branch, add it at the end after a \":\" character like this: facebook/galactica-125m:main. To download a single file, enter its name in the second box.", interactive=not mu) @@ -148,6 +149,11 @@ def create_event_handlers(): ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( save_model_settings, gradio('model_menu', 'interface_state'), gradio('model_status'), show_progress=False) + shared.gradio['model_menu'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) + shared.gradio['gpu_layers'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) + shared.gradio['ctx_size'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) + shared.gradio['cache_type'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) + if not shared.args.portable: shared.gradio['lora_menu_apply'].click(load_lora_wrapper, gradio('lora_menu'), gradio('model_status'), show_progress=False) @@ -275,6 +281,14 @@ def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), retur yield traceback.format_exc().replace('\n', '\n\n') +def estimate_vram_wrapper(model, gpu_layers, ctx_size, cache_type): + if model in ["None", None]: + return "
Estimated VRAM to load the model:" + + result = estimate_vram(model, gpu_layers, ctx_size, cache_type) + return f"
Estimated VRAM to load the model: {result:.0f} MiB" + + def update_truncation_length(current_length, state): if 'loader' in state: if state['loader'].lower().startswith('exllama') or state['loader'] == 'llama.cpp': diff --git a/server.py b/server.py index b0b9e633..c35d65a8 100644 --- a/server.py +++ b/server.py @@ -49,8 +49,10 @@ from modules.extensions import apply_extensions from modules.LoRA import add_lora_to_model from modules.models import load_model, unload_model_if_idle from modules.models_settings import ( + estimate_vram, get_fallback_settings, get_model_metadata, + get_nvidia_free_vram, update_model_parameters ) from modules.shared import do_cmd_flags_warnings @@ -248,6 +250,16 @@ if __name__ == "__main__": model_settings = get_model_metadata(model_name) update_model_parameters(model_settings, initial=True) # hijack the command-line arguments + if 'gpu_layers' not in shared.provided_arguments: + available_vram = get_nvidia_free_vram() + if available_vram > 0: + n_layers = model_settings['gpu_layers'] + tolerance = 906 + while n_layers > 0 and estimate_vram(model_name, n_layers, shared.args.ctx_size, shared.args.cache_type) > available_vram - tolerance: + n_layers -= 1 + + shared.args.gpu_layers = n_layers + # Load the model shared.model, shared.tokenizer = load_model(model_name) if shared.args.lora: From 041248cc9f321aa6ff2e706083cdb776e3bf8d21 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 15 May 2025 20:10:02 -0700 Subject: [PATCH 77/90] Update llama.cpp --- requirements/full/requirements.txt | 4 ++-- requirements/full/requirements_amd.txt | 4 ++-- requirements/full/requirements_amd_noavx2.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 4 ++-- requirements/full/requirements_apple_silicon.txt | 6 +++--- requirements/full/requirements_cpu_only.txt | 4 ++-- requirements/full/requirements_cpu_only_noavx2.txt | 4 ++-- requirements/full/requirements_noavx2.txt | 4 ++-- requirements/portable/requirements.txt | 4 ++-- requirements/portable/requirements_apple_intel.txt | 4 ++-- requirements/portable/requirements_apple_silicon.txt | 6 +++--- requirements/portable/requirements_cpu_only.txt | 4 ++-- requirements/portable/requirements_cpu_only_noavx2.txt | 4 ++-- requirements/portable/requirements_noavx2.txt | 4 ++-- requirements/portable/requirements_vulkan.txt | 4 ++-- requirements/portable/requirements_vulkan_noavx2.txt | 4 ++-- 16 files changed, 34 insertions(+), 34 deletions(-) diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index 3a059c91..45bb5c85 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt index ebc33216..4e011989 100644 --- a/requirements/full/requirements_amd.txt +++ b/requirements/full/requirements_amd.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt index 8ec6898f..a3bd1350 100644 --- a/requirements/full/requirements_amd_noavx2.txt +++ b/requirements/full/requirements_amd_noavx2.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index afc869c8..a52f2d64 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -29,7 +29,7 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index 8d7d29b7..929b1d86 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -29,8 +29,8 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt index d69aae18..bd7c4a4f 100644 --- a/requirements/full/requirements_cpu_only.txt +++ b/requirements/full/requirements_cpu_only.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt index 540c9ac8..b5aa1cf7 100644 --- a/requirements/full/requirements_cpu_only_noavx2.txt +++ b/requirements/full/requirements_cpu_only_noavx2.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index 3bb5a74a..bc320c27 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt index 95319d75..79959398 100644 --- a/requirements/portable/requirements.txt +++ b/requirements/portable/requirements.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt index 4b49b4e1..ca16e4c7 100644 --- a/requirements/portable/requirements_apple_intel.txt +++ b/requirements/portable/requirements_apple_intel.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt index a6ebda30..18e1c506 100644 --- a/requirements/portable/requirements_apple_silicon.txt +++ b/requirements/portable/requirements_apple_silicon.txt @@ -15,6 +15,6 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt index bb5ba8ad..693f4712 100644 --- a/requirements/portable/requirements_cpu_only.txt +++ b/requirements/portable/requirements_cpu_only.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt index 3d17dd49..8635d11e 100644 --- a/requirements/portable/requirements_cpu_only_noavx2.txt +++ b/requirements/portable/requirements_cpu_only_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt index ff9fa04c..e844596e 100644 --- a/requirements/portable/requirements_noavx2.txt +++ b/requirements/portable/requirements_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt index e17f8ce7..9b7435d1 100644 --- a/requirements/portable/requirements_vulkan.txt +++ b/requirements/portable/requirements_vulkan.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" diff --git a/requirements/portable/requirements_vulkan_noavx2.txt b/requirements/portable/requirements_vulkan_noavx2.txt index dd01b3a8..513b7a15 100644 --- a/requirements/portable/requirements_vulkan_noavx2.txt +++ b/requirements/portable/requirements_vulkan_noavx2.txt @@ -15,5 +15,5 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.11.0/llama_cpp_binaries-0.11.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" From 8cb73b78e14154040ac4a2f7dd33dc7d46121108 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 15 May 2025 20:10:34 -0700 Subject: [PATCH 78/90] Update ExLlamaV3 --- requirements/full/requirements.txt | 4 ++-- requirements/full/requirements_apple_intel.txt | 2 +- requirements/full/requirements_apple_silicon.txt | 2 +- requirements/full/requirements_noavx2.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt index 45bb5c85..af5f7d8a 100644 --- a/requirements/full/requirements.txt +++ b/requirements/full/requirements.txt @@ -32,8 +32,8 @@ tiktoken # CUDA wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a9/exllamav3-0.0.1a9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a9/exllamav3-0.0.1a9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt index a52f2d64..363365bf 100644 --- a/requirements/full/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -31,5 +31,5 @@ tiktoken # Mac wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a9/exllamav3-0.0.1a9-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt index 929b1d86..2843fed2 100644 --- a/requirements/full/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -32,5 +32,5 @@ tiktoken https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8-py3-none-any.whl +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a9/exllamav3-0.0.1a9-py3-none-any.whl https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt index bc320c27..89947cbe 100644 --- a/requirements/full/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -32,8 +32,8 @@ tiktoken # CUDA wheels https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.12.0/llama_cpp_binaries-0.12.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a8/exllamav3-0.0.1a8+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a9/exllamav3-0.0.1a9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a9/exllamav3-0.0.1a9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/turboderp-org/exllamav2/releases/download/v0.2.9/exllamav2-0.2.9-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64" From fd612979330ee0009ccbb14ac5bff894b675bb82 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 15 May 2025 21:19:19 -0700 Subject: [PATCH 79/90] Lint --- extensions/openai/completions.py | 8 ++++---- extensions/openai/utils.py | 6 +++--- extensions/superboogav2/chromadb.py | 3 ++- modules/tensorrt_llm.py | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index ed0bcc40..5181b18b 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -1,14 +1,14 @@ import copy -import time import json +import time from collections import deque import tiktoken +from pydantic import ValidationError from extensions.openai.errors import InvalidRequestError -from extensions.openai.utils import debug_msg, getToolCallId, parseToolCall from extensions.openai.typing import ToolDefinition -from pydantic import ValidationError +from extensions.openai.utils import debug_msg, getToolCallId, parseToolCall from modules import shared from modules.chat import ( generate_chat_prompt, @@ -141,7 +141,7 @@ def chat_completions_common(body: dict, is_legacy: bool = False, stream=False, p tools = None if 'tools' in body and body['tools'] is not None and isinstance(body['tools'], list) and len(body['tools']) > 0: - tools = validateTools(body['tools']) # raises InvalidRequestError if validation fails + tools = validateTools(body['tools']) # raises InvalidRequestError if validation fails messages = body['messages'] for m in messages: diff --git a/extensions/openai/utils.py b/extensions/openai/utils.py index 8cb856ff..9a1de2e7 100644 --- a/extensions/openai/utils.py +++ b/extensions/openai/utils.py @@ -1,9 +1,9 @@ import base64 -import os -import time import json +import os import random import re +import time import traceback from typing import Callable, Optional @@ -91,7 +91,7 @@ def parseToolCall(answer: str, tool_names: list[str]): return matches # Define the regex pattern to find the JSON content wrapped in , , , and other tags observed from various models - patterns = [ r"(```[^\n]*)\n(.*?)```", r"<([^>]+)>(.*?)" ] + patterns = [r"(```[^\n]*)\n(.*?)```", r"<([^>]+)>(.*?)"] for pattern in patterns: for match in re.finditer(pattern, answer, re.DOTALL): diff --git a/extensions/superboogav2/chromadb.py b/extensions/superboogav2/chromadb.py index f4f77821..9344e25c 100644 --- a/extensions/superboogav2/chromadb.py +++ b/extensions/superboogav2/chromadb.py @@ -1,10 +1,11 @@ import math import random import threading -import torch + import chromadb import numpy as np import posthog +import torch from chromadb.config import Settings from chromadb.utils import embedding_functions diff --git a/modules/tensorrt_llm.py b/modules/tensorrt_llm.py index 73178c39..0527d493 100644 --- a/modules/tensorrt_llm.py +++ b/modules/tensorrt_llm.py @@ -1,15 +1,15 @@ from pathlib import Path -import torch - import tensorrt_llm +import torch +from tensorrt_llm.runtime import ModelRunner, ModelRunnerCpp + from modules import shared from modules.logging_colors import logger from modules.text_generation import ( get_max_prompt_length, get_reply_from_output_ids ) -from tensorrt_llm.runtime import ModelRunner, ModelRunnerCpp class TensorRTLLMModel: From cbf4daf1c8d149206da80892ced0220cf858ebb7 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 15 May 2025 21:21:54 -0700 Subject: [PATCH 80/90] Hide the LoRA menu in portable mode --- modules/ui_model_menu.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 2353f39c..a1911124 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -102,10 +102,11 @@ def create_ui(): shared.gradio['no_sdpa'] = gr.Checkbox(label="no_sdpa", value=shared.args.no_sdpa) shared.gradio['cfg_cache'] = gr.Checkbox(label="cfg-cache", value=shared.args.cfg_cache, info='Necessary to use CFG with this loader.') shared.gradio['no_use_fast'] = gr.Checkbox(label="no_use_fast", value=shared.args.no_use_fast, info='Set use_fast=False while loading the tokenizer.') - with gr.Row(): - shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)', elem_classes='slim-dropdown', interactive=not mu) - ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) - shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) + if not shared.args.portable: + with gr.Row(): + shared.gradio['lora_menu'] = gr.Dropdown(multiselect=True, choices=utils.get_available_loras(), value=shared.lora_names, label='LoRA(s)', elem_classes='slim-dropdown', interactive=not mu) + ui.create_refresh_button(shared.gradio['lora_menu'], lambda: None, lambda: {'choices': utils.get_available_loras(), 'value': shared.lora_names}, 'refresh-button', interactive=not mu) + shared.gradio['lora_menu_apply'] = gr.Button(value='Apply LoRAs', elem_classes='refresh-button', interactive=not mu) with gr.Column(): with gr.Tab("Download"): From 93e1850a2c1eef8fe914bd020dde3e94d6b54f6c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 15 May 2025 21:42:15 -0700 Subject: [PATCH 81/90] Only show the VRAM info for llama.cpp --- modules/loaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/loaders.py b/modules/loaders.py index 583b65c2..79a7a4a3 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -28,6 +28,7 @@ loaders_and_params = OrderedDict({ 'device_draft', 'ctx_size_draft', 'speculative_decoding_accordion', + 'vram_info', ], 'Transformers': [ 'gpu_split', From 4925c307cfc97c1ca549b71db6f1aaaf82fd4fb2 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 09:07:38 -0700 Subject: [PATCH 82/90] Auto-adjust GPU layers on context size and cache type changes + many fixes --- modules/models_settings.py | 78 +++++++++++++++++++++++++++++++------- modules/ui_model_menu.py | 46 ++++++++++++++-------- server.py | 23 ++++++----- 3 files changed, 109 insertions(+), 38 deletions(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index a8e17594..6ea6660c 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -213,24 +213,26 @@ def apply_model_settings_to_state(model, state): model_settings = get_model_metadata(model) if 'loader' in model_settings: loader = model_settings.pop('loader') - - # If the user is using an alternative loader for the same model type, let them keep using it if not (loader == 'ExLlamav2_HF' and state['loader'] in ['ExLlamav2']): state['loader'] = loader for k in model_settings: - if k in state: - if k == 'gpu_layers': - available_vram = get_nvidia_free_vram() - n_layers = model_settings[k] - if available_vram > 0: - tolerance = 906 - while n_layers > 0 and estimate_vram(model, n_layers, state['ctx_size'], state['cache_type']) > available_vram - tolerance: - n_layers -= 1 + if k in state and k != 'gpu_layers': # Skip gpu_layers, handle separately + state[k] = model_settings[k] - state[k] = gr.update(value=n_layers, maximum=model_settings[k]) - else: - state[k] = model_settings[k] + # Handle GPU layers and VRAM update for llama.cpp + if state['loader'] == 'llama.cpp' and 'gpu_layers' in model_settings: + vram_info, gpu_layers_update = update_gpu_layers_and_vram( + state['loader'], + model, + model_settings['gpu_layers'], + state['ctx_size'], + state['cache_type'], + auto_adjust=True + ) + + state['gpu_layers'] = gpu_layers_update + state['vram_info'] = vram_info return state @@ -426,3 +428,53 @@ def get_nvidia_free_vram(): raise # Handle any other unexpected exceptions return -1 + + +def update_gpu_layers_and_vram(loader, model, gpu_layers, ctx_size, cache_type, auto_adjust=False, for_ui=True): + """ + Unified function to handle GPU layers and VRAM updates. + + Args: + for_ui: If True, returns Gradio updates. If False, returns raw values. + + Returns: + - If for_ui=True: (vram_info_update, gpu_layers_update) or just vram_info_update + - If for_ui=False: (vram_usage, adjusted_layers) or just vram_usage + """ + if loader != 'llama.cpp' or model in ["None", None]: + vram_info = "
Estimated VRAM to load the model:" + if for_ui: + return (vram_info, gr.update()) if auto_adjust else vram_info + else: + return (0, gpu_layers) if auto_adjust else 0 + + current_layers = gpu_layers + max_layers = gpu_layers + + if auto_adjust: + # Get max layers from model metadata + model_settings = get_model_metadata(model) + max_layers = model_settings.get('gpu_layers', gpu_layers) + + # Auto-adjust based on available VRAM + available_vram = get_nvidia_free_vram() + if available_vram > 0: + tolerance = 906 + current_layers = max_layers + while current_layers > 0 and estimate_vram(model, current_layers, ctx_size, cache_type) > available_vram - tolerance: + current_layers -= 1 + + # Calculate VRAM with current layers + vram_usage = estimate_vram(model, current_layers, ctx_size, cache_type) + + if for_ui: + vram_info = f"
Estimated VRAM to load the model: {vram_usage:.0f} MiB" + if auto_adjust: + return vram_info, gr.update(value=current_layers, maximum=max_layers) + else: + return vram_info + else: + if auto_adjust: + return vram_usage, current_layers + else: + return vram_usage diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index a1911124..b6febb50 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -11,10 +11,10 @@ from modules.LoRA import add_lora_to_model from modules.models import load_model, unload_model from modules.models_settings import ( apply_model_settings_to_state, - estimate_vram, get_model_metadata, save_instruction_template, save_model_settings, + update_gpu_layers_and_vram, update_model_parameters ) from modules.utils import gradio @@ -45,7 +45,7 @@ def create_ui(): shared.gradio['hqq_backend'] = gr.Dropdown(label="hqq_backend", choices=["PYTORCH", "PYTORCH_COMPILE", "ATEN"], value=shared.args.hqq_backend) with gr.Column(): - shared.gradio['vram_info'] = gr.HTML(value=lambda: estimate_vram_wrapper(shared.args.model, shared.args.gpu_layers, shared.args.ctx_size, shared.args.cache_type)) + shared.gradio['vram_info'] = gr.HTML(value=get_initial_vram_info()) shared.gradio['flash_attn'] = gr.Checkbox(label="flash-attn", value=shared.args.flash_attn, info='Use flash-attention.') shared.gradio['streaming_llm'] = gr.Checkbox(label="streaming-llm", value=shared.args.streaming_llm, info='Activate StreamingLLM to avoid re-evaluating the entire prompt when old messages are removed.') shared.gradio['load_in_8bit'] = gr.Checkbox(label="load-in-8bit", value=shared.args.load_in_8bit) @@ -150,10 +150,18 @@ def create_event_handlers(): ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( save_model_settings, gradio('model_menu', 'interface_state'), gradio('model_status'), show_progress=False) - shared.gradio['model_menu'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) - shared.gradio['gpu_layers'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) - shared.gradio['ctx_size'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) - shared.gradio['cache_type'].change(estimate_vram_wrapper, gradio('model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info'), show_progress=False) + # For ctx_size and cache_type - auto-adjust GPU layers + for param in ['ctx_size', 'cache_type']: + shared.gradio[param].change( + partial(update_gpu_layers_and_vram, auto_adjust=True), + gradio('loader', 'model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), + gradio('vram_info', 'gpu_layers'), show_progress=False) + + # For manual gpu_layers changes - only update VRAM + shared.gradio['gpu_layers'].change( + partial(update_gpu_layers_and_vram, auto_adjust=False), + gradio('loader', 'model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), + gradio('vram_info'), show_progress=False) if not shared.args.portable: shared.gradio['lora_menu_apply'].click(load_lora_wrapper, gradio('lora_menu'), gradio('model_status'), show_progress=False) @@ -282,14 +290,6 @@ def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), retur yield traceback.format_exc().replace('\n', '\n\n') -def estimate_vram_wrapper(model, gpu_layers, ctx_size, cache_type): - if model in ["None", None]: - return "
Estimated VRAM to load the model:" - - result = estimate_vram(model, gpu_layers, ctx_size, cache_type) - return f"
Estimated VRAM to load the model: {result:.0f} MiB" - - def update_truncation_length(current_length, state): if 'loader' in state: if state['loader'].lower().startswith('exllama') or state['loader'] == 'llama.cpp': @@ -298,10 +298,26 @@ def update_truncation_length(current_length, state): return current_length +def get_initial_vram_info(): + if shared.model_name != 'None' and shared.args.loader == 'llama.cpp': + return update_gpu_layers_and_vram( + shared.args.loader, + shared.model_name, + shared.args.gpu_layers, + shared.args.ctx_size, + shared.args.cache_type, + auto_adjust=False, + for_ui=True + ) + + return "
Estimated VRAM to load the model:" + + def handle_load_model_event_initial(model, state): state = apply_model_settings_to_state(model, state) output = ui.apply_interface_values(state) - update_model_parameters(state) + update_model_parameters(state) # This updates the command-line flags + return output + [state] diff --git a/server.py b/server.py index c35d65a8..c22ed1f1 100644 --- a/server.py +++ b/server.py @@ -49,10 +49,9 @@ from modules.extensions import apply_extensions from modules.LoRA import add_lora_to_model from modules.models import load_model, unload_model_if_idle from modules.models_settings import ( - estimate_vram, get_fallback_settings, get_model_metadata, - get_nvidia_free_vram, + update_gpu_layers_and_vram, update_model_parameters ) from modules.shared import do_cmd_flags_warnings @@ -250,15 +249,19 @@ if __name__ == "__main__": model_settings = get_model_metadata(model_name) update_model_parameters(model_settings, initial=True) # hijack the command-line arguments - if 'gpu_layers' not in shared.provided_arguments: - available_vram = get_nvidia_free_vram() - if available_vram > 0: - n_layers = model_settings['gpu_layers'] - tolerance = 906 - while n_layers > 0 and estimate_vram(model_name, n_layers, shared.args.ctx_size, shared.args.cache_type) > available_vram - tolerance: - n_layers -= 1 + # Auto-adjust GPU layers if not provided by user and it's a llama.cpp model + if 'gpu_layers' not in shared.provided_arguments and shared.args.loader == 'llama.cpp' and 'gpu_layers' in model_settings: + vram_usage, adjusted_layers = update_gpu_layers_and_vram( + shared.args.loader, + model_name, + model_settings['gpu_layers'], + shared.args.ctx_size, + shared.args.cache_type, + auto_adjust=True, + for_ui=False + ) - shared.args.gpu_layers = n_layers + shared.args.gpu_layers = adjusted_layers # Load the model shared.model, shared.tokenizer = load_model(model_name) From ee7b3028acaa38399272e78bb05272a420e72f05 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 09:12:36 -0700 Subject: [PATCH 83/90] Always cache GGUF metadata calls --- modules/models_settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index 6ea6660c..8ecd2267 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -58,7 +58,7 @@ def get_model_metadata(model): else: model_file = list(path.glob('*.gguf'))[0] - metadata = metadata_gguf.load_metadata(model_file) + metadata = load_gguf_metadata_with_cache(model_file) for k in metadata: if k.endswith('context_length'): @@ -295,8 +295,8 @@ def save_instruction_template(model, template): yield (f"Instruction template for `{model}` saved to `{p}` as `{template}`.") -@functools.lru_cache(maxsize=None) -def get_gguf_metadata_cached(model_file): +@functools.lru_cache(maxsize=1) +def load_gguf_metadata_with_cache(model_file): return metadata_gguf.load_metadata(model_file) @@ -320,7 +320,7 @@ def get_model_size_mb(model_file: Path) -> float: def estimate_vram(gguf_file, gpu_layers, ctx_size, cache_type): model_file = Path(f'{shared.args.model_dir}/{gguf_file}') - metadata = get_gguf_metadata_cached(model_file) + metadata = load_gguf_metadata_with_cache(model_file) size_in_mb = get_model_size_mb(model_file) # Extract values from metadata From 9ec9b1bf837a995af8f203c8d05897510ab77c3d Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 09:56:23 -0700 Subject: [PATCH 84/90] Auto-adjust GPU layers after model unload to utilize freed VRAM --- modules/ui_model_menu.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index b6febb50..39c57bf3 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -145,7 +145,9 @@ def create_event_handlers(): partial(load_model_wrapper, autoload=True), gradio('model_menu', 'loader'), gradio('model_status'), show_progress=True).success( handle_load_model_event_final, gradio('truncation_length', 'loader', 'interface_state'), gradio('truncation_length', 'filter_by_loader'), show_progress=False) - shared.gradio['unload_model'].click(handle_unload_model_click, None, gradio('model_status'), show_progress=False) + shared.gradio['unload_model'].click(handle_unload_model_click, None, gradio('model_status'), show_progress=False).then( + partial(update_gpu_layers_and_vram, auto_adjust=True), gradio('loader', 'model_menu', 'gpu_layers', 'ctx_size', 'cache_type'), gradio('vram_info', 'gpu_layers'), show_progress=False) + shared.gradio['save_model_settings'].click( ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( save_model_settings, gradio('model_menu', 'interface_state'), gradio('model_status'), show_progress=False) From 253e85a519219385668aabeabc82633c8e734ff9 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 10:02:30 -0700 Subject: [PATCH 85/90] Only compute VRAM/GPU layers for llama.cpp models --- modules/models_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index 8ecd2267..0eb179d7 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -441,7 +441,7 @@ def update_gpu_layers_and_vram(loader, model, gpu_layers, ctx_size, cache_type, - If for_ui=True: (vram_info_update, gpu_layers_update) or just vram_info_update - If for_ui=False: (vram_usage, adjusted_layers) or just vram_usage """ - if loader != 'llama.cpp' or model in ["None", None]: + if loader != 'llama.cpp' or model in ["None", None] or not model.endswith(".gguf"): vram_info = "
Estimated VRAM to load the model:" if for_ui: return (vram_info, gr.update()) if auto_adjust else vram_info From 38c50087feb11af41fed7a944ab0d7ef45a3bc44 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 11:55:06 -0700 Subject: [PATCH 86/90] Prevent a crash on systems without an NVIDIA GPU --- modules/models_settings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index 0eb179d7..3fdf3c84 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -421,11 +421,9 @@ def get_nvidia_free_vram(): return total_free_vram_mib except FileNotFoundError: - raise # nvidia-smi not found (likely no NVIDIA drivers installed) return -1 except Exception: - raise # Handle any other unexpected exceptions return -1 From fc483650b5e8c4933ac20b647cb822cf45856596 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 11:58:17 -0700 Subject: [PATCH 87/90] Set the maximum gpu_layers value automatically when the model is loaded with --model --- modules/ui_model_menu.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index 39c57bf3..cd101c4a 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -38,7 +38,7 @@ def create_ui(): gr.Markdown("## Main options") with gr.Row(): with gr.Column(): - shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=256, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') + shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=get_initial_gpu_layers_max(), value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=256, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') shared.gradio['cache_type'] = gr.Dropdown(label="cache-type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') @@ -315,6 +315,14 @@ def get_initial_vram_info(): return "
Estimated VRAM to load the model:" +def get_initial_gpu_layers_max(): + if shared.model_name != 'None' and shared.args.loader == 'llama.cpp': + model_settings = get_model_metadata(shared.model_name) + return model_settings.get('gpu_layers', 256) + + return 256 + + def handle_load_model_event_initial(model, state): state = apply_model_settings_to_state(model, state) output = ui.apply_interface_values(state) From adb975a380b219bbe14bbd7a19c83eaebc15cd55 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 12:52:43 -0700 Subject: [PATCH 88/90] Prevent fractional gpu-layers in the UI --- modules/ui_model_menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py index cd101c4a..59bb6759 100644 --- a/modules/ui_model_menu.py +++ b/modules/ui_model_menu.py @@ -38,7 +38,7 @@ def create_ui(): gr.Markdown("## Main options") with gr.Row(): with gr.Column(): - shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=get_initial_gpu_layers_max(), value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') + shared.gradio['gpu_layers'] = gr.Slider(label="gpu-layers", minimum=0, maximum=get_initial_gpu_layers_max(), step=1, value=shared.args.gpu_layers, info='Must be greater than 0 for the GPU to be used. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['ctx_size'] = gr.Slider(label='ctx-size', minimum=256, maximum=131072, step=256, value=shared.args.ctx_size, info='Context length. ⚠️ Lower this value if you can\'t load the model.') shared.gradio['gpu_split'] = gr.Textbox(label='gpu-split', info='Comma-separated list of VRAM (in GB) to use per GPU. Example: 20,7,7') shared.gradio['cache_type'] = gr.Dropdown(label="cache-type", choices=['fp16', 'q8_0', 'q4_0', 'fp8', 'q8', 'q7', 'q6', 'q5', 'q4', 'q3', 'q2'], value=shared.args.cache_type, allow_custom_value=True, info='Valid options: llama.cpp - fp16, q8_0, q4_0; ExLlamaV2 - fp16, fp8, q8, q6, q4; ExLlamaV3 - fp16, q2 to q8. For ExLlamaV3, you can type custom combinations for separate k/v bits (e.g. q4_q8).') From 470c822f44dce2269dfaa8e3b37989195982b975 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 12:54:39 -0700 Subject: [PATCH 89/90] API: Hide the uvicorn access logs from the terminal --- extensions/openai/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 2b4f274f..2c98ee78 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -447,7 +447,7 @@ def run_server(): # Start server logging.getLogger("uvicorn.error").propagate = False - uvicorn.run(app, host=server_addrs, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile) + uvicorn.run(app, host=server_addrs, port=port, ssl_certfile=ssl_certfile, ssl_keyfile=ssl_keyfile, access_log=False) def setup(): From e4d3f4449d75ea1b1f7f3438dbed8c910a970cec Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 16 May 2025 13:02:27 -0700 Subject: [PATCH 90/90] API: Fix a regression --- modules/llama_cpp_server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py index 3fc7a0cc..d695c74e 100644 --- a/modules/llama_cpp_server.py +++ b/modules/llama_cpp_server.py @@ -146,8 +146,9 @@ class LlamaServer: pprint.PrettyPrinter(indent=4, sort_dicts=False).pprint(printable_payload) print() - # Make a direct request with streaming enabled using a context manager - with self.session.post(url, json=payload, stream=True) as response: + # Make the generation request + response = self.session.post(url, json=payload, stream=True) + try: response.raise_for_status() # Raise an exception for HTTP errors full_text = "" @@ -184,6 +185,8 @@ class LlamaServer: print(f"JSON decode error: {e}") print(f"Problematic line: {line}") continue + finally: + response.close() def generate(self, prompt, state): output = ""