" + data.html + "
";
+ }
+
+ const queryScope = target_element;
+
// Track open blocks
const openBlocks = new Set();
- document.querySelectorAll(".thinking-block").forEach(block => {
+ queryScope.querySelectorAll(".thinking-block").forEach(block => {
const blockId = block.getAttribute("data-block-id");
// If block exists and is open, add to open set
if (blockId && block.hasAttribute("open")) {
@@ -242,7 +255,7 @@ function handleMorphdomUpdate(text) {
// Store scroll positions for any open blocks
const scrollPositions = {};
- document.querySelectorAll(".thinking-block[open]").forEach(block => {
+ queryScope.querySelectorAll(".thinking-block[open]").forEach(block => {
const content = block.querySelector(".thinking-content");
const blockId = block.getAttribute("data-block-id");
if (content && blockId) {
@@ -255,8 +268,8 @@ function handleMorphdomUpdate(text) {
});
morphdom(
- document.getElementById("chat").parentNode,
- "" + text + "
",
+ target_element,
+ target_html,
{
onBeforeElUpdated: function(fromEl, toEl) {
// Preserve code highlighting
@@ -307,7 +320,7 @@ function handleMorphdomUpdate(text) {
);
// Add toggle listeners for new blocks
- document.querySelectorAll(".thinking-block").forEach(block => {
+ queryScope.querySelectorAll(".thinking-block").forEach(block => {
if (!block._hasToggleListener) {
block.addEventListener("toggle", function(e) {
if (this.open) {
diff --git a/modules/chat.py b/modules/chat.py
index 1222d2bb..f1ea16f1 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -825,7 +825,9 @@ def generate_chat_reply_wrapper(text, state, regenerate=False, _continue=False):
last_save_time = time.monotonic()
save_interval = 8
for i, history in enumerate(generate_chat_reply(text, state, regenerate, _continue, loading_message=True, for_ui=True)):
- yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu']), history
+ yield chat_html_wrapper(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu'], last_message_only=(i > 0)), history
+ if i == 0:
+ time.sleep(0.125) # We need this to make sure the first update goes through
current_time = time.monotonic()
# Save on first iteration or if save_interval seconds have passed
diff --git a/modules/html_generator.py b/modules/html_generator.py
index 03b5d485..f90e3b04 100644
--- a/modules/html_generator.py
+++ b/modules/html_generator.py
@@ -462,64 +462,69 @@ def actions_html(history, i, role, info_message=""):
f'{version_nav_html}')
-def generate_instruct_html(history):
- output = f'"
+
return output
-def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=False):
- output = f'
'
+def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=False, last_message_only=False):
+ if not last_message_only:
+ output = f'"
+ # Determine range
+ start_idx = len(history['visible']) - 1 if last_message_only else 0
+ end_idx = len(history['visible'])
+
+ for i in range(start_idx, end_idx):
+ row_visible = history['visible'][i]
+ row_internal = history['internal'][i]
+
+ # Convert content
+ if last_message_only:
+ converted_visible = [None, convert_to_markdown_wrapped(row_visible[1], message_id=i, use_cache=i != len(history['visible']) - 1)]
+ else:
+ converted_visible = [convert_to_markdown_wrapped(entry, message_id=i, use_cache=i != len(history['visible']) - 1) for entry in row_visible]
+
+ # Generate messages
+ if not last_message_only and converted_visible[0]:
+ output += create_message("user", converted_visible[0], row_internal[0])
+
+ output += create_message("assistant", converted_visible[1], row_internal[1])
+
+ if not last_message_only:
+ output += "
'
+ else:
+ output = ""
# We use ?character and ?time.time() to force the browser to reset caches
img_bot = (
@@ -527,110 +532,117 @@ def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=
if Path("user_data/cache/pfp_character_thumb.png").exists() else ''
)
- img_me = (
- f' if reset_cache else )
'
- if Path("user_data/cache/pfp_me.png").exists() else ''
- )
+ def create_message(role, content, raw_content):
+ """Inner function for CAI-style messages."""
+ circle_class = "circle-you" if role == "user" else "circle-bot"
+ name = name1 if role == "user" else name2
- for i in range(len(history['visible'])):
- row_visible = history['visible'][i]
- row_internal = history['internal'][i]
- converted_visible = [convert_to_markdown_wrapped(entry, message_id=i, use_cache=i != len(history['visible']) - 1) for entry in row_visible]
+ # Get role-specific data
+ timestamp = format_message_timestamp(history, role, i, tooltip_include_timestamp=False)
+ attachments = format_message_attachments(history, role, i)
- # Get timestamps
- user_timestamp = format_message_timestamp(history, "user", i, tooltip_include_timestamp=False)
- assistant_timestamp = format_message_timestamp(history, "assistant", i, tooltip_include_timestamp=False)
+ # Get appropriate image
+ if role == "user":
+ img = (f''
+ if Path("user_data/cache/pfp_me.png").exists() else '')
+ else:
+ img = img_bot
- # Get attachments
- user_attachments = format_message_attachments(history, "user", i)
- assistant_attachments = format_message_attachments(history, "assistant", i)
-
- if converted_visible[0]: # Don't display empty user messages
- output += (
- f''
- )
-
- output += (
+ return (
f''
)
- output += "
'
- if Path("user_data/cache/pfp_me.png").exists() else ''
- )
+ def create_message(role, content, raw_content):
+ """Inner function for CAI-style messages."""
+ circle_class = "circle-you" if role == "user" else "circle-bot"
+ name = name1 if role == "user" else name2
- for i in range(len(history['visible'])):
- row_visible = history['visible'][i]
- row_internal = history['internal'][i]
- converted_visible = [convert_to_markdown_wrapped(entry, message_id=i, use_cache=i != len(history['visible']) - 1) for entry in row_visible]
+ # Get role-specific data
+ timestamp = format_message_timestamp(history, role, i, tooltip_include_timestamp=False)
+ attachments = format_message_attachments(history, role, i)
- # Get timestamps
- user_timestamp = format_message_timestamp(history, "user", i, tooltip_include_timestamp=False)
- assistant_timestamp = format_message_timestamp(history, "assistant", i, tooltip_include_timestamp=False)
+ # Get appropriate image
+ if role == "user":
+ img = (f''
+ if Path("user_data/cache/pfp_me.png").exists() else '')
+ else:
+ img = img_bot
- # Get attachments
- user_attachments = format_message_attachments(history, "user", i)
- assistant_attachments = format_message_attachments(history, "assistant", i)
-
- if converted_visible[0]: # Don't display empty user messages
- output += (
- f''
- )
-
- output += (
+ return (
f''
)
- output += ""
+
return output
@@ -644,15 +656,15 @@ def time_greeting():
return "Good evening!"
-def chat_html_wrapper(history, name1, name2, mode, style, character, reset_cache=False):
+def chat_html_wrapper(history, name1, name2, mode, style, character, reset_cache=False, last_message_only=False):
if len(history['visible']) == 0:
greeting = f"
Date: Mon, 2 Jun 2025 09:57:55 -0300
Subject: [PATCH 20/82] Simplify the one-click installer (#7039)
---
one_click.py | 253 +++++++++++++++++++++++++--------------------------
1 file changed, 123 insertions(+), 130 deletions(-)
diff --git a/one_click.py b/one_click.py
index 482a6aa9..cccb0dc9 100644
--- a/one_click.py
+++ b/one_click.py
@@ -70,12 +70,8 @@ def is_installed():
def cpu_has_avx2():
try:
import cpuinfo
-
info = cpuinfo.get_cpu_info()
- if 'avx2' in info['flags']:
- return True
- else:
- return False
+ return 'avx2' in info['flags']
except:
return True
@@ -83,30 +79,112 @@ def cpu_has_avx2():
def cpu_has_amx():
try:
import cpuinfo
-
info = cpuinfo.get_cpu_info()
- if 'amx' in info['flags']:
- return True
- else:
- return False
+ return 'amx' in info['flags']
except:
return True
-def torch_version():
- site_packages_path = None
- for sitedir in site.getsitepackages():
- if "site-packages" in sitedir and conda_env_path in sitedir:
- site_packages_path = sitedir
- break
+def load_state():
+ """Load installer state from JSON file"""
+ if os.path.exists(state_file):
+ try:
+ with open(state_file, 'r') as f:
+ return json.load(f)
+ except:
+ return {}
+ return {}
- if site_packages_path:
- torch_version_file = open(os.path.join(site_packages_path, 'torch', 'version.py')).read().splitlines()
- torver = [line for line in torch_version_file if line.startswith('__version__')][0].split('__version__ = ')[1].strip("'")
+
+def save_state(state):
+ """Save installer state to JSON file"""
+ with open(state_file, 'w') as f:
+ json.dump(state, f)
+
+
+def get_gpu_choice():
+ """Get GPU choice from state file or ask user"""
+ state = load_state()
+ gpu_choice = state.get('gpu_choice')
+
+ if not gpu_choice:
+ if "GPU_CHOICE" in os.environ:
+ choice = os.environ["GPU_CHOICE"].upper()
+ print_big_message(f"Selected GPU choice \"{choice}\" based on the GPU_CHOICE environment variable.")
+ else:
+ choice = get_user_choice(
+ "What is your GPU?",
+ {
+ 'A': 'NVIDIA - CUDA 12.4',
+ 'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4',
+ 'C': 'Apple M Series',
+ 'D': 'Intel Arc (beta)',
+ 'N': 'CPU mode'
+ },
+ )
+
+ # Convert choice to GPU name
+ gpu_choice = {"A": "NVIDIA", "B": "AMD", "C": "APPLE", "D": "INTEL", "N": "NONE"}[choice]
+
+ # Save choice to state
+ state['gpu_choice'] = gpu_choice
+ save_state(state)
+
+ return gpu_choice
+
+
+def get_pytorch_install_command(gpu_choice):
+ """Get PyTorch installation command based on GPU choice"""
+ base_cmd = f"python -m pip install torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION} "
+
+ if gpu_choice == "NVIDIA":
+ return base_cmd + "--index-url https://download.pytorch.org/whl/cu124"
+ elif gpu_choice == "AMD":
+ return base_cmd + "--index-url https://download.pytorch.org/whl/rocm6.2.4"
+ elif gpu_choice in ["APPLE", "NONE"]:
+ return base_cmd + "--index-url https://download.pytorch.org/whl/cpu"
+ elif gpu_choice == "INTEL":
+ if is_linux():
+ return "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
+ else:
+ return "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
else:
- from torch import __version__ as torver
+ return base_cmd
- return torver
+
+def get_pytorch_update_command(gpu_choice):
+ """Get PyTorch update command based on GPU choice"""
+ base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION}"
+
+ if gpu_choice == "NVIDIA":
+ return f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124"
+ elif gpu_choice == "AMD":
+ return f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4"
+ elif gpu_choice in ["APPLE", "NONE"]:
+ return f"{base_cmd} --index-url https://download.pytorch.org/whl/cpu"
+ elif gpu_choice == "INTEL":
+ intel_extension = "intel-extension-for-pytorch==2.1.10+xpu" if is_linux() else "intel-extension-for-pytorch==2.1.10"
+ return f"{base_cmd} {intel_extension} --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
+ else:
+ return base_cmd
+
+
+def get_requirements_file(gpu_choice):
+ """Get requirements file path based on GPU choice"""
+ requirements_base = os.path.join("requirements", "full")
+
+ if gpu_choice == "AMD":
+ file_name = f"requirements_amd{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ elif gpu_choice == "APPLE":
+ file_name = f"requirements_apple_{'intel' if is_x86_64() else 'silicon'}.txt"
+ elif gpu_choice in ["INTEL", "NONE"]:
+ file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ elif gpu_choice == "NVIDIA":
+ file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ else:
+ raise ValueError(f"Unknown GPU choice: {gpu_choice}")
+
+ return os.path.join(requirements_base, file_name)
def get_current_commit():
@@ -209,28 +287,8 @@ def get_user_choice(question, options_dict):
def update_pytorch_and_python():
print_big_message("Checking for PyTorch updates.")
-
- # Update the Python version. Left here for future reference in case this becomes necessary.
- # print_big_message("Checking for PyTorch and Python updates.")
- # current_python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
- # if current_python_version != PYTHON_VERSION:
- # run_cmd(f"conda install -y python={PYTHON_VERSION}", assert_success=True, environment=True)
-
- torver = torch_version()
- base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION}"
-
- if "+cu" in torver:
- install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124"
- elif "+rocm" in torver:
- install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4"
- elif "+cpu" in torver:
- install_cmd = f"{base_cmd} --index-url https://download.pytorch.org/whl/cpu"
- elif "+cxx11" in torver:
- intel_extension = "intel-extension-for-pytorch==2.1.10+xpu" if is_linux() else "intel-extension-for-pytorch==2.1.10"
- install_cmd = f"{base_cmd} {intel_extension} --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
- else:
- install_cmd = base_cmd
-
+ gpu_choice = get_gpu_choice()
+ install_cmd = get_pytorch_update_command(gpu_choice)
run_cmd(install_cmd, assert_success=True, environment=True)
@@ -256,43 +314,11 @@ def install_webui():
if os.path.isfile(state_file):
os.remove(state_file)
- # Ask the user for the GPU vendor
- if "GPU_CHOICE" in os.environ:
- choice = os.environ["GPU_CHOICE"].upper()
- print_big_message(f"Selected GPU choice \"{choice}\" based on the GPU_CHOICE environment variable.")
-
- # Warn about changed meanings and handle old choices
- if choice == "B":
- print_big_message("Warning: GPU_CHOICE='B' now means 'AMD' in the new version.")
- elif choice == "C":
- print_big_message("Warning: GPU_CHOICE='C' now means 'Apple M Series' in the new version.")
- elif choice == "D":
- print_big_message("Warning: GPU_CHOICE='D' now means 'Intel Arc' in the new version.")
- else:
- choice = get_user_choice(
- "What is your GPU?",
- {
- 'A': 'NVIDIA - CUDA 12.4',
- 'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4',
- 'C': 'Apple M Series',
- 'D': 'Intel Arc (beta)',
- 'N': 'CPU mode'
- },
- )
-
- # Convert choices to GPU names for compatibility
- gpu_choice_to_name = {
- "A": "NVIDIA",
- "B": "AMD",
- "C": "APPLE",
- "D": "INTEL",
- "N": "NONE"
- }
-
- selected_gpu = gpu_choice_to_name[choice]
+ # Get GPU choice and save it to state
+ gpu_choice = get_gpu_choice()
# Write a flag to CMD_FLAGS.txt for CPU mode
- if selected_gpu == "NONE":
+ if gpu_choice == "NONE":
cmd_flags_path = os.path.join(script_dir, "user_data", "CMD_FLAGS.txt")
with open(cmd_flags_path, 'r+') as cmd_flags_file:
if "--cpu" not in cmd_flags_file.read():
@@ -300,34 +326,20 @@ def install_webui():
cmd_flags_file.write("\n--cpu\n")
# Handle CUDA version display
- elif any((is_windows(), is_linux())) and selected_gpu == "NVIDIA":
+ elif any((is_windows(), is_linux())) and gpu_choice == "NVIDIA":
print("CUDA: 12.4")
# No PyTorch for AMD on Windows (?)
- elif is_windows() and selected_gpu == "AMD":
+ elif is_windows() and gpu_choice == "AMD":
print("PyTorch setup on Windows is not implemented yet. Exiting...")
sys.exit(1)
- # Find the Pytorch installation command
- install_pytorch = f"python -m pip install torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION} "
-
- if selected_gpu == "NVIDIA":
- install_pytorch += "--index-url https://download.pytorch.org/whl/cu124"
- elif selected_gpu == "AMD":
- install_pytorch += "--index-url https://download.pytorch.org/whl/rocm6.2.4"
- elif selected_gpu in ["APPLE", "NONE"]:
- install_pytorch += "--index-url https://download.pytorch.org/whl/cpu"
- elif selected_gpu == "INTEL":
- if is_linux():
- install_pytorch = "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
- else:
- install_pytorch = "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
-
# Install Git and then Pytorch
print_big_message("Installing PyTorch.")
+ install_pytorch = get_pytorch_install_command(gpu_choice)
run_cmd(f"conda install -y ninja git && {install_pytorch} && python -m pip install py-cpuinfo==9.0.0", assert_success=True, environment=True)
- if selected_gpu == "INTEL":
+ if gpu_choice == "INTEL":
# Install oneAPI dependencies via conda
print_big_message("Installing Intel oneAPI runtime libraries.")
run_cmd("conda install -y -c https://software.repos.intel.com/python/conda/ -c conda-forge dpcpp-cpp-rt=2024.0 mkl-dpcpp=2024.0", environment=True)
@@ -349,31 +361,15 @@ def update_requirements(initial_installation=False, pull=True):
assert_success=True
)
- torver = torch_version()
- requirements_base = os.path.join("requirements", "full")
-
- if "+rocm" in torver:
- file_name = f"requirements_amd{'_noavx2' if not cpu_has_avx2() else ''}.txt"
- elif "+cpu" in torver or "+cxx11" in torver:
- file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt"
- elif is_macos():
- file_name = f"requirements_apple_{'intel' if is_x86_64() else 'silicon'}.txt"
- else:
- file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt"
-
- requirements_file = os.path.join(requirements_base, file_name)
-
- # Load state from JSON file
current_commit = get_current_commit()
- wheels_changed = False
- if os.path.exists(state_file):
- with open(state_file, 'r') as f:
- last_state = json.load(f)
-
- if 'wheels_changed' in last_state or last_state.get('last_installed_commit') != current_commit:
+ wheels_changed = not os.path.exists(state_file)
+ if not wheels_changed:
+ state = load_state()
+ if 'wheels_changed' in state or state.get('last_installed_commit') != current_commit:
wheels_changed = True
- else:
- wheels_changed = True
+
+ gpu_choice = get_gpu_choice()
+ requirements_file = get_requirements_file(gpu_choice)
if pull:
# Read .whl lines before pulling
@@ -409,19 +405,17 @@ def update_requirements(initial_installation=False, pull=True):
print_big_message(f"File '{file}' was updated during 'git pull'. Please run the script again.")
# Save state before exiting
- current_state = {}
+ state = load_state()
if wheels_changed:
- current_state['wheels_changed'] = True
-
- with open(state_file, 'w') as f:
- json.dump(current_state, f)
-
+ state['wheels_changed'] = True
+ save_state(state)
sys.exit(1)
# Save current state
- current_state = {'last_installed_commit': current_commit}
- with open(state_file, 'w') as f:
- json.dump(current_state, f)
+ state = load_state()
+ state['last_installed_commit'] = current_commit
+ state.pop('wheels_changed', None) # Remove wheels_changed flag
+ save_state(state)
if os.environ.get("INSTALL_EXTENSIONS", "").lower() in ("yes", "y", "true", "1", "t", "on"):
install_extensions_requirements()
@@ -432,11 +426,10 @@ def update_requirements(initial_installation=False, pull=True):
# Update PyTorch
if not initial_installation:
update_pytorch_and_python()
- torver = torch_version()
clean_outdated_pytorch_cuda_dependencies()
print_big_message(f"Installing webui requirements from file: {requirements_file}")
- print(f"TORCH: {torver}\n")
+ print(f"GPU Choice: {gpu_choice}\n")
# Prepare the requirements file
textgen_requirements = open(requirements_file).read().splitlines()
From b30a73016d626e985e248de15fa65e5a531c8bd2 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 2 Jun 2025 07:49:22 -0700
Subject: [PATCH 21/82] Remove the "Is typing..." yield by default
---
modules/chat.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index f1ea16f1..3c4c3636 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -656,7 +656,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
update_message_metadata(output['metadata'], "user", row_idx, timestamp=get_current_timestamp())
# *Is typing...*
- if loading_message:
+ if loading_message and shared.processing_message:
yield {
'visible': output['visible'][:-1] + [[output['visible'][-1][0], shared.processing_message]],
'internal': output['internal'],
@@ -680,7 +680,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
})
output['metadata'][key]["current_version_index"] = len(output['metadata'][key]["versions"]) - 1
- if loading_message:
+ if loading_message and shared.processing_message:
yield {
'visible': output['visible'][:-1] + [[visible_text, shared.processing_message]],
'internal': output['internal'][:-1] + [[text, '']],
From b38ec0ec385d44d49d3fe7adf2ad77ae62302214 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 2 Jun 2025 11:33:17 -0700
Subject: [PATCH 22/82] Update llama.cpp
---
requirements/full/requirements.txt | 4 ++--
requirements/full/requirements_amd.txt | 4 ++--
requirements/full/requirements_amd_noavx2.txt | 4 ++--
requirements/full/requirements_apple_intel.txt | 4 ++--
requirements/full/requirements_apple_silicon.txt | 6 +++---
requirements/full/requirements_cpu_only.txt | 4 ++--
requirements/full/requirements_cpu_only_noavx2.txt | 4 ++--
requirements/full/requirements_noavx2.txt | 4 ++--
requirements/portable/requirements.txt | 4 ++--
requirements/portable/requirements_apple_intel.txt | 4 ++--
requirements/portable/requirements_apple_silicon.txt | 6 +++---
requirements/portable/requirements_cpu_only.txt | 4 ++--
requirements/portable/requirements_cpu_only_noavx2.txt | 4 ++--
requirements/portable/requirements_noavx2.txt | 4 ++--
requirements/portable/requirements_vulkan.txt | 4 ++--
requirements/portable/requirements_vulkan_noavx2.txt | 4 ++--
16 files changed, 34 insertions(+), 34 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index 04d97220..277f8249 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -34,8 +34,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt
index f807199d..dbf35c34 100644
--- a/requirements/full/requirements_amd.txt
+++ b/requirements/full/requirements_amd.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt
index 4fb70eb1..2e5eb6c9 100644
--- a/requirements/full/requirements_amd_noavx2.txt
+++ b/requirements/full/requirements_amd_noavx2.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt
index a311ab9b..9a19ab29 100644
--- a/requirements/full/requirements_apple_intel.txt
+++ b/requirements/full/requirements_apple_intel.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3-py3-none-any.whl
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl
diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt
index 30e8409a..973d9bfb 100644
--- a/requirements/full/requirements_apple_silicon.txt
+++ b/requirements/full/requirements_apple_silicon.txt
@@ -33,8 +33,8 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3-py3-none-any.whl
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl
diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt
index 70949949..4a48a51f 100644
--- a/requirements/full/requirements_cpu_only.txt
+++ b/requirements/full/requirements_cpu_only.txt
@@ -33,5 +33,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt
index 318bb93a..76bde864 100644
--- a/requirements/full/requirements_cpu_only_noavx2.txt
+++ b/requirements/full/requirements_cpu_only_noavx2.txt
@@ -33,5 +33,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt
index 7c3635cc..6cd0fa65 100644
--- a/requirements/full/requirements_noavx2.txt
+++ b/requirements/full/requirements_noavx2.txt
@@ -34,8 +34,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt
index bde310e1..60ce941e 100644
--- a/requirements/portable/requirements.txt
+++ b/requirements/portable/requirements.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt
index 521edc0c..b1649bc9 100644
--- a/requirements/portable/requirements_apple_intel.txt
+++ b/requirements/portable/requirements_apple_intel.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt
index ef7946ff..571eba52 100644
--- a/requirements/portable/requirements_apple_silicon.txt
+++ b/requirements/portable/requirements_apple_silicon.txt
@@ -19,6 +19,6 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0"
diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt
index a3ad743e..88170cf3 100644
--- a/requirements/portable/requirements_cpu_only.txt
+++ b/requirements/portable/requirements_cpu_only.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt
index eec052d3..e96cef49 100644
--- a/requirements/portable/requirements_cpu_only_noavx2.txt
+++ b/requirements/portable/requirements_cpu_only_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt
index c9898a05..78f94aa5 100644
--- a/requirements/portable/requirements_noavx2.txt
+++ b/requirements/portable/requirements_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt
index 0de9c7cb..3e41427d 100644
--- a/requirements/portable/requirements_vulkan.txt
+++ b/requirements/portable/requirements_vulkan.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan_noavx2.txt b/requirements/portable/requirements_vulkan_noavx2.txt
index 2bfb4d51..022ebb61 100644
--- a/requirements/portable/requirements_vulkan_noavx2.txt
+++ b/requirements/portable/requirements_vulkan_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.14.0/llama_cpp_binaries-0.14.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
From 93b3752cdf9f43dd391462168e2e14dd2ab75643 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 09:40:30 -0700
Subject: [PATCH 23/82] Revert "Remove the "Is typing..." yield by default"
This reverts commit b30a73016d626e985e248de15fa65e5a531c8bd2.
---
modules/chat.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index 3c4c3636..f1ea16f1 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -656,7 +656,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
update_message_metadata(output['metadata'], "user", row_idx, timestamp=get_current_timestamp())
# *Is typing...*
- if loading_message and shared.processing_message:
+ if loading_message:
yield {
'visible': output['visible'][:-1] + [[output['visible'][-1][0], shared.processing_message]],
'internal': output['internal'],
@@ -680,7 +680,7 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
})
output['metadata'][key]["current_version_index"] = len(output['metadata'][key]["versions"]) - 1
- if loading_message and shared.processing_message:
+ if loading_message:
yield {
'visible': output['visible'][:-1] + [[visible_text, shared.processing_message]],
'internal': output['internal'][:-1] + [[text, '']],
From 9bd7359ffab5e434b7cdfdb43ee91cb3ad397c0d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:47:14 -0700
Subject: [PATCH 24/82] Scroll the textarea into view when editing a message
---
js/global_scope_js.js | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/js/global_scope_js.js b/js/global_scope_js.js
index d5140c93..801f1574 100644
--- a/js/global_scope_js.js
+++ b/js/global_scope_js.js
@@ -95,6 +95,12 @@ function startEditing(messageElement, messageBody, isUserMessage) {
editingInterface.textarea.focus();
editingInterface.textarea.setSelectionRange(rawText.length, rawText.length);
+ // Scroll the textarea into view
+ editingInterface.textarea.scrollIntoView({
+ behavior: "smooth",
+ block: "center"
+ });
+
// Setup event handlers
setupEditingHandlers(editingInterface.textarea, messageElement, originalHTML, messageBody, isUserMessage);
}
From 66a75c899a4b0786cd8744886a189864923287b5 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:59:43 -0700
Subject: [PATCH 25/82] Improve the scrollbars in code blocks
---
js/main.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/js/main.js b/js/main.js
index 8090937f..2e72d82e 100644
--- a/js/main.js
+++ b/js/main.js
@@ -229,6 +229,7 @@ function doSyntaxHighlighting() {
codeBlocks.forEach((codeBlock) => {
hljs.highlightElement(codeBlock);
codeBlock.setAttribute("data-highlighted", "true");
+ codeBlock.classList.add("pretty_scrollbar");
});
renderMathInElement(messageBody, {
From 3d676cd50f8661ca96a20a452611422acb47177c Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Wed, 4 Jun 2025 11:02:04 -0700
Subject: [PATCH 26/82] Optimize syntax highlighting
---
js/main.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/js/main.js b/js/main.js
index 2e72d82e..9a620fa9 100644
--- a/js/main.js
+++ b/js/main.js
@@ -217,7 +217,7 @@ function isElementVisibleOnScreen(element) {
}
function doSyntaxHighlighting() {
- const messageBodies = document.querySelectorAll(".message-body");
+ const messageBodies = document.getElementById("chat").querySelectorAll(".message-body");
if (messageBodies.length > 0) {
observer.disconnect();
From 3829507d0fd66eccc532b5d8d0e3d77c38143d0c Mon Sep 17 00:00:00 2001
From: Hanusz Leszek
Date: Wed, 4 Jun 2025 20:13:36 +0200
Subject: [PATCH 27/82] Stop model during graceful shutdown (#7042)
---
server.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/server.py b/server.py
index c22ed1f1..99d2e171 100644
--- a/server.py
+++ b/server.py
@@ -60,6 +60,14 @@ from modules.utils import gradio
def signal_handler(sig, frame):
logger.info("Received Ctrl+C. Shutting down Text generation web UI gracefully.")
+
+ # Explicitly stop LlamaServer to avoid __del__ cleanup issues during shutdown
+ if shared.model and shared.model.__class__.__name__ == 'LlamaServer':
+ try:
+ shared.model.stop()
+ except:
+ pass
+
sys.exit(0)
From 977ec801b7682c3239fe3e6fdfcb8b90c1e802f3 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 5 Jun 2025 06:33:45 -0700
Subject: [PATCH 28/82] Improve table colors in instruct mode
---
css/html_instruct_style.css | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/css/html_instruct_style.css b/css/html_instruct_style.css
index 6ad250aa..9831ee8f 100644
--- a/css/html_instruct_style.css
+++ b/css/html_instruct_style.css
@@ -17,6 +17,14 @@
color: #d1d5db !important;
}
+.chat .message-body :is(th, td) {
+ border-color: #40404096 !important;
+}
+
+.dark .chat .message-body :is(th, td) {
+ border-color: #ffffff75 !important;
+}
+
.chat .message-body :is(p, ul, ol) {
margin: 1.25em 0 !important;
}
From d47c8eb956a72ebc7c1f582718758697aef62118 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 5 Jun 2025 06:56:24 -0700
Subject: [PATCH 29/82] Remove quotes from LLM-generated websearch query
(closes #7045).
Fix by @Quiet-Joker
---
modules/chat.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/modules/chat.py b/modules/chat.py
index f1ea16f1..14f2a4f7 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -604,7 +604,12 @@ def generate_search_query(user_message, state):
query = ""
for reply in generate_reply(formatted_prompt, search_state, stopping_strings=[], is_chat=True):
- query = reply.strip()
+ query = reply
+
+ # Strip and remove surrounding quotes if present
+ query = query.strip()
+ if len(query) >= 2 and query.startswith('"') and query.endswith('"'):
+ query = query[1:-1]
return query
From 45f823ddf625cb914acb6620b0551bf6734b2838 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:23:34 -0700
Subject: [PATCH 30/82] Print \n after the llama.cpp progress bar reaches 1.0
---
modules/llama_cpp_server.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index aa712541..d8fcb89d 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -417,14 +417,23 @@ def filter_stderr_with_progress(process_stderr):
progress_match = progress_pattern.search(line)
if progress_match:
+ progress_value = float(progress_match.group(1))
+
if last_was_progress:
# Overwrite the previous progress line using carriage return
sys.stderr.write(f'\r{line}')
else:
# First progress line - print normally
sys.stderr.write(line)
+
+ # Check if progress is 100% (1.0)
+ if progress_value >= 1.0:
+ sys.stderr.write('\n') # Add newline for completed progress
+ last_was_progress = False
+ else:
+ last_was_progress = True
+
sys.stderr.flush()
- last_was_progress = True
elif not line.startswith(('srv ', 'slot ')) and 'log_server_r: request: GET /health' not in line:
if last_was_progress:
# Finish the progress line with a newline, then print the new line
From f8f23b5489e70467ffc46a1b6ae2a406b3fcf9ff Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:25:13 -0700
Subject: [PATCH 31/82] Simplify the llama.cpp stderr filter code
---
modules/llama_cpp_server.py | 49 ++++++++++++++++---------------------
1 file changed, 21 insertions(+), 28 deletions(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index d8fcb89d..f0a72de8 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -408,41 +408,34 @@ class LlamaServer:
def filter_stderr_with_progress(process_stderr):
- progress_pattern = re.compile(r'slot update_slots: id.*progress = (\d+\.\d+)')
+ """
+ Reads stderr lines from a process, filters out noise, and displays progress updates
+ inline (overwriting the same line) until completion.
+ """
+ progress_re = re.compile(r'slot update_slots: id.*progress = (\d+\.\d+)')
last_was_progress = False
try:
- for line in iter(process_stderr.readline, ''):
- line = line.rstrip('\n\r') # Remove existing newlines
- progress_match = progress_pattern.search(line)
+ for raw in iter(process_stderr.readline, ''):
+ line = raw.rstrip('\r\n')
+ match = progress_re.search(line)
- if progress_match:
- progress_value = float(progress_match.group(1))
+ if match:
+ progress = float(match.group(1))
+ # choose carriage return for in-progress or newline at completion
+ end_char = '\r' if progress < 1.0 else '\n'
+ print(line, end=end_char, file=sys.stderr, flush=True)
+ last_was_progress = (progress < 1.0)
+ # skip noise lines
+ elif not (line.startswith(('srv ', 'slot ')) or 'log_server_r: request: GET /health' in line):
+ # if we were in progress, finish that line first
if last_was_progress:
- # Overwrite the previous progress line using carriage return
- sys.stderr.write(f'\r{line}')
- else:
- # First progress line - print normally
- sys.stderr.write(line)
+ print(file=sys.stderr)
- # Check if progress is 100% (1.0)
- if progress_value >= 1.0:
- sys.stderr.write('\n') # Add newline for completed progress
- last_was_progress = False
- else:
- last_was_progress = True
-
- sys.stderr.flush()
- elif not line.startswith(('srv ', 'slot ')) and 'log_server_r: request: GET /health' not in line:
- if last_was_progress:
- # Finish the progress line with a newline, then print the new line
- sys.stderr.write(f'\n{line}\n')
- else:
- # Normal line - print with newline
- sys.stderr.write(f'{line}\n')
- sys.stderr.flush()
+ print(line, file=sys.stderr, flush=True)
last_was_progress = False
- # For filtered lines, don't change last_was_progress state
+
except (ValueError, IOError):
+ # silently ignore broken output or IO errors
pass
From 379dd01ca7770484bbd0798252665935a0bbfa05 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:32:07 -0700
Subject: [PATCH 32/82] Filter out failed web search downloads from attachments
---
modules/web_search.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/modules/web_search.py b/modules/web_search.py
index 1f670349..a1e47253 100644
--- a/modules/web_search.py
+++ b/modules/web_search.py
@@ -107,6 +107,13 @@ def add_web_search_attachments(history, row_idx, user_message, search_query, sta
logger.warning("No search results found")
return
+ # Filter out failed downloads before adding attachments
+ successful_results = [result for result in search_results if result['content'] and result['content'].strip()]
+
+ if not successful_results:
+ logger.warning("No successful downloads to add as attachments")
+ return
+
# Add search results as attachments
key = f"user_{row_idx}"
if key not in history['metadata']:
@@ -114,7 +121,7 @@ def add_web_search_attachments(history, row_idx, user_message, search_query, sta
if "attachments" not in history['metadata'][key]:
history['metadata'][key]["attachments"] = []
- for result in search_results:
+ for result in successful_results:
attachment = {
"name": result['title'],
"type": "text/html",
@@ -123,7 +130,7 @@ def add_web_search_attachments(history, row_idx, user_message, search_query, sta
}
history['metadata'][key]["attachments"].append(attachment)
- logger.info(f"Added {len(search_results)} web search results as attachments")
+ logger.info(f"Added {len(successful_results)} successful web search results as attachments")
except Exception as e:
logger.error(f"Error in web search: {e}")
From 2d263f227d090d2d5c0b152e045ab5cc3eae3073 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Fri, 6 Jun 2025 22:38:20 -0700
Subject: [PATCH 33/82] Fix the chat input reappearing when the page is
reloaded
---
modules/ui.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/modules/ui.py b/modules/ui.py
index 14a09d2b..89bbbdb6 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -270,6 +270,9 @@ def gather_interface_values(*args):
if not shared.args.multi_user:
shared.persistent_interface_state = output
+ # Remove the chat input, as it gets cleared after this function call
+ shared.persistent_interface_state.pop('textbox')
+
# Prevent history loss if backend is restarted but UI is not refreshed
if output['history'] is None and output['unique_id'] is not None:
output['history'] = load_history(output['unique_id'], output['character_menu'], output['mode'])
From db847eed4c0693c735e364fcf48e79e4f001bb77 Mon Sep 17 00:00:00 2001
From: rakha abadi susilo
Date: Sun, 8 Jun 2025 07:44:15 +0700
Subject: [PATCH 34/82] Add RTX 50XX Nvidia blackwell support (ExLlamaV2/V3 and
Transformers) (#7011)
---------
Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com>
---
one_click.py | 21 ++++++---
requirements/full/requirements_cuda128.txt | 46 +++++++++++++++++++
.../full/requirements_cuda128_noavx2.txt | 46 +++++++++++++++++++
3 files changed, 106 insertions(+), 7 deletions(-)
create mode 100644 requirements/full/requirements_cuda128.txt
create mode 100644 requirements/full/requirements_cuda128_noavx2.txt
diff --git a/one_click.py b/one_click.py
index cccb0dc9..94f2aab0 100644
--- a/one_click.py
+++ b/one_click.py
@@ -17,8 +17,6 @@ import sys
# Define the required versions
TORCH_VERSION = "2.6.0"
-TORCHVISION_VERSION = "0.21.0"
-TORCHAUDIO_VERSION = "2.6.0"
PYTHON_VERSION = "3.11"
LIBSTDCXX_VERSION_LINUX = "12.1.0"
@@ -119,12 +117,13 @@ def get_gpu_choice():
'B': 'AMD - Linux/macOS only, requires ROCm 6.2.4',
'C': 'Apple M Series',
'D': 'Intel Arc (beta)',
+ 'E': 'NVIDIA - CUDA 12.8',
'N': 'CPU mode'
},
)
# Convert choice to GPU name
- gpu_choice = {"A": "NVIDIA", "B": "AMD", "C": "APPLE", "D": "INTEL", "N": "NONE"}[choice]
+ gpu_choice = {"A": "NVIDIA", "B": "AMD", "C": "APPLE", "D": "INTEL", "E": "NVIDIA_CUDA128", "N": "NONE"}[choice]
# Save choice to state
state['gpu_choice'] = gpu_choice
@@ -135,29 +134,33 @@ def get_gpu_choice():
def get_pytorch_install_command(gpu_choice):
"""Get PyTorch installation command based on GPU choice"""
- base_cmd = f"python -m pip install torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION} "
+ base_cmd = f"python -m pip install torch=={TORCH_VERSION} "
if gpu_choice == "NVIDIA":
return base_cmd + "--index-url https://download.pytorch.org/whl/cu124"
+ elif gpu_choice == "NVIDIA_CUDA128":
+ return "python -m pip install torch==2.7.1 --index-url https://download.pytorch.org/whl/cu128"
elif gpu_choice == "AMD":
return base_cmd + "--index-url https://download.pytorch.org/whl/rocm6.2.4"
elif gpu_choice in ["APPLE", "NONE"]:
return base_cmd + "--index-url https://download.pytorch.org/whl/cpu"
elif gpu_choice == "INTEL":
if is_linux():
- return "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
+ return "python -m pip install torch==2.1.0a0 intel-extension-for-pytorch==2.1.10+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
else:
- return "python -m pip install torch==2.1.0a0 torchvision==0.16.0a0 torchaudio==2.1.0a0 intel-extension-for-pytorch==2.1.10 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
+ return "python -m pip install torch==2.1.0a0 intel-extension-for-pytorch==2.1.10 --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/"
else:
return base_cmd
def get_pytorch_update_command(gpu_choice):
"""Get PyTorch update command based on GPU choice"""
- base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} torchvision=={TORCHVISION_VERSION} torchaudio=={TORCHAUDIO_VERSION}"
+ base_cmd = f"python -m pip install --upgrade torch=={TORCH_VERSION} "
if gpu_choice == "NVIDIA":
return f"{base_cmd} --index-url https://download.pytorch.org/whl/cu124"
+ elif gpu_choice == "NVIDIA_CUDA128":
+ return "python -m pip install --upgrade torch==2.7.1 --index-url https://download.pytorch.org/whl/cu128"
elif gpu_choice == "AMD":
return f"{base_cmd} --index-url https://download.pytorch.org/whl/rocm6.2.4"
elif gpu_choice in ["APPLE", "NONE"]:
@@ -181,6 +184,8 @@ def get_requirements_file(gpu_choice):
file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt"
elif gpu_choice == "NVIDIA":
file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt"
+ elif gpu_choice == "NVIDIA_CUDA128":
+ file_name = f"requirements_cuda128{'_noavx2' if not cpu_has_avx2() else ''}.txt"
else:
raise ValueError(f"Unknown GPU choice: {gpu_choice}")
@@ -328,6 +333,8 @@ def install_webui():
# Handle CUDA version display
elif any((is_windows(), is_linux())) and gpu_choice == "NVIDIA":
print("CUDA: 12.4")
+ elif any((is_windows(), is_linux())) and gpu_choice == "NVIDIA_CUDA128":
+ print("CUDA: 12.8")
# No PyTorch for AMD on Windows (?)
elif is_windows() and gpu_choice == "AMD":
diff --git a/requirements/full/requirements_cuda128.txt b/requirements/full/requirements_cuda128.txt
new file mode 100644
index 00000000..9fc99606
--- /dev/null
+++ b/requirements/full/requirements_cuda128.txt
@@ -0,0 +1,46 @@
+accelerate==1.5.*
+beautifulsoup4==4.13.4
+bitsandbytes==0.45.*
+colorama
+datasets
+duckduckgo_search==8.0.2
+einops
+fastapi==0.112.4
+gradio==4.37.*
+jinja2==3.1.6
+markdown
+numpy==2.2.*
+pandas
+peft==0.15.*
+Pillow>=9.5.0
+psutil
+pydantic==2.8.2
+PyPDF2==3.0.1
+python-docx==1.1.2
+pyyaml
+requests
+rich
+safetensors==0.5.*
+scipy
+sentencepiece
+tensorboard
+transformers==4.52.*
+triton-windows; platform_system == "Windows"
+tqdm
+wandb
+
+# API
+flask_cloudflared==0.0.14
+sse-starlette==1.6.5
+tiktoken
+
+# CUDA wheels
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64"
+https://github.com/kingbri1/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/kingbri1/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
diff --git a/requirements/full/requirements_cuda128_noavx2.txt b/requirements/full/requirements_cuda128_noavx2.txt
new file mode 100644
index 00000000..ff34673a
--- /dev/null
+++ b/requirements/full/requirements_cuda128_noavx2.txt
@@ -0,0 +1,46 @@
+accelerate==1.5.*
+beautifulsoup4==4.13.4
+bitsandbytes==0.45.*
+colorama
+datasets
+duckduckgo_search==8.0.2
+einops
+fastapi==0.112.4
+gradio==4.37.*
+jinja2==3.1.6
+markdown
+numpy==2.2.*
+pandas
+peft==0.15.*
+Pillow>=9.5.0
+psutil
+pydantic==2.8.2
+PyPDF2==3.0.1
+python-docx==1.1.2
+pyyaml
+requests
+rich
+safetensors==0.5.*
+scipy
+sentencepiece
+tensorboard
+transformers==4.52.*
+triton-windows; platform_system == "Windows"
+tqdm
+wandb
+
+# API
+flask_cloudflared==0.0.14
+sse-starlette==1.6.5
+tiktoken
+
+# CUDA wheels
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system == "Linux" and platform_machine != "x86_64"
+https://github.com/kingbri1/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/kingbri1/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu128torch2.7.0cxx11abiFALSE-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
From 35ed55d18f0d7acfe55af10d2086185f7141cc46 Mon Sep 17 00:00:00 2001
From: oobabooga
Date: Sat, 7 Jun 2025 22:46:52 -0300
Subject: [PATCH 35/82] UI persistence (#7050)
---
modules/shared.py | 2 +
modules/ui.py | 106 ++++++++++++++++++++++++++++++++++++++++++
modules/ui_session.py | 17 +------
server.py | 3 ++
4 files changed, 112 insertions(+), 16 deletions(-)
diff --git a/modules/shared.py b/modules/shared.py
index f712f7f8..2e500779 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -35,6 +35,8 @@ settings = {
'mode': 'instruct',
'chat_style': 'cai-chat',
'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
+ 'enable_web_search': False,
+ 'web_search_pages': 3,
'prompt-default': 'QA',
'prompt-notebook': 'QA',
'character': 'Assistant',
diff --git a/modules/ui.py b/modules/ui.py
index 89bbbdb6..db3adf0f 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -1,4 +1,5 @@
import copy
+import threading
from pathlib import Path
import gradio as gr
@@ -7,6 +8,16 @@ import yaml
import extensions
from modules import shared
from modules.chat import load_history
+from modules.utils import gradio
+
+# Global state for auto-saving UI settings with debouncing
+_auto_save_timer = None
+_auto_save_lock = threading.Lock()
+_last_interface_state = None
+_last_preset = None
+_last_extensions = None
+_last_show_controls = None
+_last_theme_state = None
with open(Path(__file__).resolve().parent / '../css/NotoSans/stylesheet.css', 'r') as f:
css = f.read()
@@ -334,6 +345,101 @@ def save_settings(state, preset, extensions_list, show_controls, theme_state):
return yaml.dump(output, sort_keys=False, width=float("inf"), allow_unicode=True)
+def store_current_state_and_debounce(interface_state, preset, extensions, show_controls, theme_state):
+ """Store current state and trigger debounced save"""
+ global _auto_save_timer, _last_interface_state, _last_preset, _last_extensions, _last_show_controls, _last_theme_state
+
+ if shared.args.multi_user:
+ return
+
+ # Store the current state in global variables
+ _last_interface_state = interface_state
+ _last_preset = preset
+ _last_extensions = extensions
+ _last_show_controls = show_controls
+ _last_theme_state = theme_state
+
+ # Reset the debounce timer
+ with _auto_save_lock:
+ if _auto_save_timer is not None:
+ _auto_save_timer.cancel()
+
+ _auto_save_timer = threading.Timer(2.0, _perform_debounced_save)
+ _auto_save_timer.start()
+
+
+def _perform_debounced_save():
+ """Actually perform the save using the stored state"""
+ global _auto_save_timer
+
+ try:
+ if _last_interface_state is not None:
+ contents = save_settings(_last_interface_state, _last_preset, _last_extensions, _last_show_controls, _last_theme_state)
+ settings_path = Path('user_data') / 'settings.yaml'
+ settings_path.parent.mkdir(exist_ok=True)
+ with open(settings_path, 'w', encoding='utf-8') as f:
+ f.write(contents)
+ except Exception as e:
+ print(f"Auto-save failed: {e}")
+ finally:
+ with _auto_save_lock:
+ _auto_save_timer = None
+
+
+def setup_auto_save():
+ """Attach auto-save to key UI elements"""
+ if shared.args.multi_user:
+ return
+
+ change_elements = [
+ # Chat tab (ui_chat.py)
+ 'start_with',
+ 'enable_web_search',
+ 'web_search_pages',
+ 'mode',
+ 'chat_style',
+ 'chat-instruct_command',
+ 'character_menu',
+ 'name1',
+ 'user_bio',
+ 'custom_system_message',
+ 'chat_template_str',
+
+ # Parameters tab (ui_parameters.py)
+ 'preset_menu',
+ 'max_new_tokens',
+ 'prompt_lookup_num_tokens',
+ 'max_tokens_second',
+ 'auto_max_new_tokens',
+ 'ban_eos_token',
+ 'add_bos_token',
+ 'enable_thinking',
+ 'skip_special_tokens',
+ 'stream',
+ 'static_cache',
+ 'seed',
+ 'custom_stopping_strings',
+ 'custom_token_bans',
+ 'negative_prompt',
+
+ # Default tab (ui_default.py)
+ 'prompt_menu-default',
+
+ # Notebook tab (ui_notebook.py)
+ 'prompt_menu-notebook',
+
+ # Session tab (ui_session.py)
+ 'show_controls',
+ 'theme_state',
+ ]
+
+ for element_name in change_elements:
+ if element_name in shared.gradio:
+ shared.gradio[element_name].change(
+ gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ store_current_state_and_debounce, gradio('interface_state', 'preset_menu', 'extensions_menu', 'show_controls', 'theme_state'), None, show_progress=False)
+
+
def create_refresh_button(refresh_component, refresh_method, refreshed_args, elem_class, interactive=True):
"""
Copied from https://github.com/AUTOMATIC1111/stable-diffusion-webui
diff --git a/modules/ui_session.py b/modules/ui_session.py
index a4eba667..4ed740cd 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -10,11 +10,10 @@ def create_ui():
with gr.Tab("Session", elem_id="session-tab"):
with gr.Row():
with gr.Column():
- shared.gradio['reset_interface'] = gr.Button("Apply flags/extensions and restart", interactive=not mu)
with gr.Row():
shared.gradio['toggle_dark_mode'] = gr.Button('Toggle 💡')
- shared.gradio['save_settings'] = gr.Button('Save UI defaults to user_data/settings.yaml', interactive=not mu)
+ shared.gradio['reset_interface'] = gr.Button("Apply flags/extensions and restart", interactive=not mu)
with gr.Row():
with gr.Column():
shared.gradio['extensions_menu'] = gr.CheckboxGroup(choices=utils.get_available_extensions(), value=shared.args.extensions, label="Available extensions", info='Note that some of these extensions may require manually installing Python requirements through the command: pip install -r extensions/extension_name/requirements.txt', elem_classes='checkboxgroup-table')
@@ -42,20 +41,6 @@ def create_ui():
lambda x: 'dark' if x == 'light' else 'light', gradio('theme_state'), gradio('theme_state')).then(
None, None, None, js=f'() => {{{ui.dark_theme_js}; toggleDarkMode()}}')
- shared.gradio['save_settings'].click(
- ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
- handle_save_settings, gradio('interface_state', 'preset_menu', 'extensions_menu', 'show_controls', 'theme_state'), gradio('save_contents', 'save_filename', 'save_root', 'file_saver'), show_progress=False)
-
-
-def handle_save_settings(state, preset, extensions, show_controls, theme):
- contents = ui.save_settings(state, preset, extensions, show_controls, theme)
- return [
- contents,
- "settings.yaml",
- "user_data/",
- gr.update(visible=True)
- ]
-
def set_interface_arguments(extensions, bool_active):
shared.args.extensions = extensions
diff --git a/server.py b/server.py
index 99d2e171..3be4c27c 100644
--- a/server.py
+++ b/server.py
@@ -150,6 +150,9 @@ def create_interface():
ui_parameters.create_event_handlers()
ui_model_menu.create_event_handlers()
+ # UI persistence events
+ ui.setup_auto_save()
+
# Interface launch events
shared.gradio['interface'].load(
None,
From 6436bf1920b128b40ed2abb35da947af6844cca3 Mon Sep 17 00:00:00 2001
From: oobabooga
Date: Sun, 8 Jun 2025 01:58:02 -0300
Subject: [PATCH 36/82] More UI persistence: presets and characters (#7051)
---
modules/chat.py | 61 ++++++++++++++++++++++++++
modules/html_generator.py | 16 ++++---
modules/presets.py | 72 +++++--------------------------
modules/shared.py | 61 +++++++++++++++++++++++---
modules/ui.py | 48 +++++++++++++++++++--
modules/ui_chat.py | 20 ++++++---
modules/ui_default.py | 4 +-
modules/ui_notebook.py | 4 +-
modules/ui_parameters.py | 90 ++++++++++++++++++++-------------------
server.py | 20 +++++----
10 files changed, 260 insertions(+), 136 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index 14f2a4f7..f740db55 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -1220,6 +1220,45 @@ def load_character(character, name1, name2):
return name1, name2, picture, greeting, context
+def reset_character_for_ui(state):
+ """Reset character fields to the currently loaded character's saved values"""
+ if state['character_menu'] and state['character_menu'] != 'None':
+ try:
+ name1, name2, picture, greeting, context = load_character(state['character_menu'], state['name1'], state['name2'])
+
+ state['name2'] = name2
+ state['greeting'] = greeting
+ state['context'] = context
+ state['character_picture'] = picture # This triggers cache update via generate_pfp_cache
+
+ return state, name2, context, greeting, picture
+
+ except Exception as e:
+ logger.error(f"Failed to reset character '{state['character_menu']}': {e}")
+ return clear_character_for_ui(state)
+ else:
+ return clear_character_for_ui(state)
+
+
+def clear_character_for_ui(state):
+ """Clear all character fields and picture cache"""
+ state['name2'] = shared.settings['name2']
+ state['context'] = shared.settings['context']
+ state['greeting'] = shared.settings['greeting']
+ state['character_picture'] = None
+
+ # Clear the cache files
+ cache_folder = Path(shared.args.disk_cache_dir)
+ for cache_file in ['pfp_character.png', 'pfp_character_thumb.png']:
+ cache_path = Path(f'{cache_folder}/{cache_file}')
+ if cache_path.exists():
+ cache_path.unlink()
+
+ logger.info("Cleared character fields and picture cache")
+
+ return state, state['name2'], state['context'], state['greeting'], None
+
+
def load_instruction_template(template):
if template == 'None':
return ''
@@ -1710,6 +1749,28 @@ def handle_character_menu_change(state):
]
+def handle_character_picture_change(picture):
+ """Update or clear cache when character picture changes"""
+ cache_folder = Path(shared.args.disk_cache_dir)
+ if not cache_folder.exists():
+ cache_folder.mkdir()
+
+ if picture is not None:
+ # Save to cache
+ picture.save(Path(f'{cache_folder}/pfp_character.png'), format='PNG')
+ thumb = make_thumbnail(picture)
+ thumb.save(Path(f'{cache_folder}/pfp_character_thumb.png'), format='PNG')
+ logger.info("Updated character picture cache")
+ else:
+ # Remove cache files when picture is cleared
+ for cache_file in ['pfp_character.png', 'pfp_character_thumb.png']:
+ cache_path = Path(f'{cache_folder}/{cache_file}')
+ if cache_path.exists():
+ cache_path.unlink()
+
+ logger.info("Cleared character picture cache")
+
+
def handle_mode_change(state):
history = load_latest_history(state)
histories = find_all_histories_with_first_prompts(state)
diff --git a/modules/html_generator.py b/modules/html_generator.py
index f90e3b04..eac7d91a 100644
--- a/modules/html_generator.py
+++ b/modules/html_generator.py
@@ -520,17 +520,23 @@ def generate_instruct_html(history, last_message_only=False):
return output
+def get_character_image_with_cache_buster():
+ """Get character image URL with cache busting based on file modification time"""
+ cache_path = Path("user_data/cache/pfp_character_thumb.png")
+ if cache_path.exists():
+ mtime = int(cache_path.stat().st_mtime)
+ return f'
'
+
+ return ''
+
+
def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=False, last_message_only=False):
if not last_message_only:
output = f'
{time_greeting()} How can I help you today?
"
result = f'{greeting}
'
elif mode == 'instruct':
- result = generate_instruct_html(history)
+ result = generate_instruct_html(history, last_message_only=last_message_only)
elif style == 'wpp':
- result = generate_chat_html(history, name1, name2)
+ result = generate_chat_html(history, name1, name2, last_message_only=last_message_only)
else:
- result = generate_cai_chat_html(history, name1, name2, style, character, reset_cache)
+ result = generate_cai_chat_html(history, name1, name2, style, character, reset_cache=reset_cache, last_message_only=last_message_only)
- return {'html': result}
+ return {'html': result, 'last_message_only': last_message_only}
diff --git a/modules/shared.py b/modules/shared.py
index d2305f30..f712f7f8 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -21,7 +21,7 @@ lora_names = []
# Generation variables
stop_everything = False
generation_lock = None
-processing_message = '*Is typing...*'
+processing_message = ''
# UI variables
gradio = {}
@@ -47,7 +47,6 @@ settings = {
'max_new_tokens_max': 4096,
'prompt_lookup_num_tokens': 0,
'max_tokens_second': 0,
- 'max_updates_second': 12,
'auto_max_new_tokens': True,
'ban_eos_token': False,
'add_bos_token': True,
diff --git a/modules/text_generation.py b/modules/text_generation.py
index 1fd6d810..0d499d50 100644
--- a/modules/text_generation.py
+++ b/modules/text_generation.py
@@ -65,41 +65,39 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap
all_stop_strings += st
shared.stop_everything = False
- last_update = -1
reply = ''
is_stream = state['stream']
if len(all_stop_strings) > 0 and not state['stream']:
state = copy.deepcopy(state)
state['stream'] = True
- min_update_interval = 0
- if state.get('max_updates_second', 0) > 0:
- min_update_interval = 1 / state['max_updates_second']
-
# Generate
+ last_update = -1
+ latency_threshold = 1 / 1000
for reply in generate_func(question, original_question, state, stopping_strings, is_chat=is_chat):
+ cur_time = time.monotonic()
reply, stop_found = apply_stopping_strings(reply, all_stop_strings)
if escape_html:
reply = html.escape(reply)
if is_stream:
- cur_time = time.time()
-
# Limit number of tokens/second to make text readable in real time
if state['max_tokens_second'] > 0:
diff = 1 / state['max_tokens_second'] - (cur_time - last_update)
if diff > 0:
time.sleep(diff)
- last_update = time.time()
+ last_update = time.monotonic()
yield reply
# Limit updates to avoid lag in the Gradio UI
# API updates are not limited
else:
- if cur_time - last_update > min_update_interval:
- last_update = cur_time
+ # If 'generate_func' takes less than 0.001 seconds to yield the next token
+ # (equivalent to more than 1000 tok/s), assume that the UI is lagging behind and skip yielding
+ if (cur_time - last_update) > latency_threshold:
yield reply
+ last_update = time.monotonic()
if stop_found or (state['max_tokens_second'] > 0 and shared.stop_everything):
break
diff --git a/modules/ui.py b/modules/ui.py
index 9f4d67cb..14a09d2b 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -6,6 +6,7 @@ import yaml
import extensions
from modules import shared
+from modules.chat import load_history
with open(Path(__file__).resolve().parent / '../css/NotoSans/stylesheet.css', 'r') as f:
css = f.read()
@@ -194,7 +195,6 @@ def list_interface_input_elements():
'max_new_tokens',
'prompt_lookup_num_tokens',
'max_tokens_second',
- 'max_updates_second',
'do_sample',
'dynamic_temperature',
'temperature_last',
@@ -270,6 +270,10 @@ def gather_interface_values(*args):
if not shared.args.multi_user:
shared.persistent_interface_state = output
+ # Prevent history loss if backend is restarted but UI is not refreshed
+ if output['history'] is None and output['unique_id'] is not None:
+ output['history'] = load_history(output['unique_id'], output['character_menu'], output['mode'])
+
return output
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 822b77b8..0d5a2c18 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -18,7 +18,7 @@ def create_ui():
mu = shared.args.multi_user
shared.gradio['Chat input'] = gr.State()
- shared.gradio['history'] = gr.JSON(visible=False)
+ shared.gradio['history'] = gr.State({'internal': [], 'visible': [], 'metadata': {}})
with gr.Tab('Chat', id='Chat', elem_id='chat-tab'):
with gr.Row(elem_id='past-chats-row', elem_classes=['pretty_scrollbar']):
@@ -195,7 +195,7 @@ def create_event_handlers():
shared.reload_inputs = gradio(reload_arr)
# Morph HTML updates instead of updating everything
- shared.gradio['display'].change(None, gradio('display'), None, js="(data) => handleMorphdomUpdate(data.html)")
+ shared.gradio['display'].change(None, gradio('display'), None, js="(data) => handleMorphdomUpdate(data)")
shared.gradio['Generate'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index 733d0901..84f9fbfc 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -71,8 +71,6 @@ def create_ui(default_preset):
shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.')
shared.gradio['prompt_lookup_num_tokens'] = gr.Slider(value=shared.settings['prompt_lookup_num_tokens'], minimum=0, maximum=10, step=1, label='prompt_lookup_num_tokens', info='Activates Prompt Lookup Decoding.')
shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.')
- shared.gradio['max_updates_second'] = gr.Slider(value=shared.settings['max_updates_second'], minimum=0, maximum=24, step=1, label='Maximum UI updates/second', info='Set this if you experience lag in the UI during streaming.')
-
with gr.Column():
with gr.Row():
with gr.Column():
diff --git a/user_data/settings-template.yaml b/user_data/settings-template.yaml
index ce0f77e1..db481e84 100644
--- a/user_data/settings-template.yaml
+++ b/user_data/settings-template.yaml
@@ -18,7 +18,6 @@ max_new_tokens_min: 1
max_new_tokens_max: 4096
prompt_lookup_num_tokens: 0
max_tokens_second: 0
-max_updates_second: 12
auto_max_new_tokens: true
ban_eos_token: false
add_bos_token: true
From 7278548cd18a9ba05062eb2db59d7f2965d8a9f6 Mon Sep 17 00:00:00 2001
From: oobabooga '
+
+ return ''
+
+
def generate_cai_chat_html(history, name1, name2, style, character, reset_cache=False, last_message_only=False):
if not last_message_only:
output = f''
else:
output = ""
- # We use ?character and ?time.time() to force the browser to reset caches
- img_bot = (
- f'
'
- if Path("user_data/cache/pfp_character_thumb.png").exists() else ''
- )
+ img_bot = get_character_image_with_cache_buster()
def create_message(role, content, raw_content):
"""Inner function for CAI-style messages."""
diff --git a/modules/presets.py b/modules/presets.py
index cf706605..3eb1f5fc 100644
--- a/modules/presets.py
+++ b/modules/presets.py
@@ -1,6 +1,5 @@
import functools
import pprint
-import random
from pathlib import Path
import yaml
@@ -93,68 +92,17 @@ def load_preset_for_ui(name, state):
return state, *[generate_params[k] for k in presets_params()]
-def random_preset(state):
- params_and_values = {
- 'remove_tail_tokens': {
- 'top_p': [0.5, 0.8, 0.9, 0.95, 0.99],
- 'min_p': [0.5, 0.2, 0.1, 0.05, 0.01],
- 'top_k': [3, 5, 10, 20, 30, 40],
- 'typical_p': [0.2, 0.575, 0.95],
- 'tfs': [0.5, 0.8, 0.9, 0.95, 0.99],
- 'top_a': [0.5, 0.2, 0.1, 0.05, 0.01],
- 'epsilon_cutoff': [1, 3, 5, 7, 9],
- 'eta_cutoff': [3, 6, 9, 12, 15, 18],
- },
- 'flatten_distribution': {
- 'temperature': [0.1, 0.5, 0.7, 0.8, 1, 1.2, 1.5, 2.0, 5.0],
- 'dynamic_temperature': [
- [0.1, 1],
- [0.1, 1.5],
- [0.1, 2],
- [0.1, 5],
- [0.5, 1],
- [0.5, 1.5],
- [0.5, 2],
- [0.5, 5],
- [0.8, 1],
- [0.8, 1.5],
- [0.8, 2],
- [0.8, 5],
- [1, 1.5],
- [1, 2],
- [1, 5]
- ],
- 'smoothing_factor': [0.2, 0.3, 0.6, 1.2],
- },
- 'repetition': {
- 'repetition_penalty': [1, 1.05, 1.1, 1.15, 1.20, 1.25],
- 'presence_penalty': [0, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2.0],
- 'frequency_penalty': [0, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2.0],
- },
- 'other': {
- 'temperature_last': [True, False],
- }
- }
-
- generate_params = default_preset()
- for cat in params_and_values:
- choices = list(params_and_values[cat].keys())
- if shared.args.loader is not None:
- choices = [x for x in choices if loader_contains(x)]
-
- if len(choices) > 0:
- choice = random.choice(choices)
- value = random.choice(params_and_values[cat][choice])
- if choice == 'dynamic_temperature':
- generate_params['dynamic_temperature'] = True
- generate_params['dynatemp_low'] = value[0]
- generate_params['dynatemp_high'] = value[1]
- else:
- generate_params[choice] = value
-
+def reset_preset_for_ui(name, state):
+ """Reset current preset to its saved values from file"""
+ generate_params = load_preset(name, verbose=True)
+ state.update(generate_params)
+ return state, *[generate_params[k] for k in presets_params()]
+
+
+def neutralize_samplers_for_ui(state):
+ """Set all samplers to their default/neutral values"""
+ generate_params = default_preset()
state.update(generate_params)
- logger.info("GENERATED_PRESET=")
- pprint.PrettyPrinter(indent=4, width=1, sort_dicts=False).pprint(remove_defaults(state))
return state, *[generate_params[k] for k in presets_params()]
diff --git a/modules/shared.py b/modules/shared.py
index 2e500779..f4f8e180 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -39,10 +39,6 @@ settings = {
'web_search_pages': 3,
'prompt-default': 'QA',
'prompt-notebook': 'QA',
- 'character': 'Assistant',
- 'name1': 'You',
- 'user_bio': '',
- 'custom_system_message': '',
'preset': 'min_p',
'max_new_tokens': 512,
'max_new_tokens_min': 1,
@@ -63,8 +59,64 @@ settings = {
'negative_prompt': '',
'dark_theme': True,
'default_extensions': [],
+
+ # Character settings
+ 'character': 'Assistant',
+ 'name1': 'You',
+ 'name2': 'AI',
+ 'user_bio': '',
+ 'context': 'The following is a conversation with an AI Large Language Model. The AI has been trained to answer questions, provide recommendations, and help with decision making. The AI follows user requests. The AI thinks outside the box.',
+ 'greeting': 'How can I help you today?',
+ 'custom_system_message': '',
'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}",
'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
+
+ # Generation parameters - Curve shape
+ 'temperature': 1.0,
+ 'dynatemp_low': 1.0,
+ 'dynatemp_high': 1.0,
+ 'dynatemp_exponent': 1.0,
+ 'smoothing_factor': 0.0,
+ 'smoothing_curve': 1.0,
+
+ # Generation parameters - Curve cutoff
+ 'min_p': 0.0,
+ 'top_p': 1.0,
+ 'top_k': 0,
+ 'typical_p': 1.0,
+ 'xtc_threshold': 0.1,
+ 'xtc_probability': 0.0,
+ 'epsilon_cutoff': 0.0,
+ 'eta_cutoff': 0.0,
+ 'tfs': 1.0,
+ 'top_a': 0.0,
+ 'top_n_sigma': 0.0,
+
+ # Generation parameters - Repetition suppression
+ 'dry_multiplier': 0.0,
+ 'dry_allowed_length': 2,
+ 'dry_base': 1.75,
+ 'repetition_penalty': 1.0,
+ 'frequency_penalty': 0.0,
+ 'presence_penalty': 0.0,
+ 'encoder_repetition_penalty': 1.0,
+ 'no_repeat_ngram_size': 0,
+ 'repetition_penalty_range': 1024,
+
+ # Generation parameters - Alternative sampling methods
+ 'penalty_alpha': 0.0,
+ 'guidance_scale': 1.0,
+ 'mirostat_mode': 0,
+ 'mirostat_tau': 5.0,
+ 'mirostat_eta': 0.1,
+
+ # Generation parameters - Other options
+ 'do_sample': True,
+ 'dynamic_temperature': False,
+ 'temperature_last': False,
+ 'sampler_priority': 'repetition_penalty\npresence_penalty\nfrequency_penalty\ndry\ntop_n_sigma\ntemperature\ndynamic_temperature\nquadratic_sampling\ntop_k\ntop_p\ntypical_p\nepsilon_cutoff\neta_cutoff\ntfs\ntop_a\nmin_p\nmirostat\nxtc\nencoder_repetition_penalty\nno_repeat_ngram',
+ 'dry_sequence_breakers': '"\\n", ":", "\\"", "*"',
+ 'grammar_string': '',
}
default_settings = copy.deepcopy(settings)
@@ -75,7 +127,6 @@ parser = argparse.ArgumentParser(description="Text generation web UI", conflict_
# Basic settings
group = parser.add_argument_group('Basic settings')
group.add_argument('--multi-user', action='store_true', help='Multi-user mode. Chat histories are not saved or automatically loaded. Warning: this is likely not safe for sharing publicly.')
-group.add_argument('--character', type=str, help='The name of the character to load in chat mode by default.')
group.add_argument('--model', type=str, help='Name of the model to load by default.')
group.add_argument('--lora', type=str, nargs='+', help='The list of LoRAs to load. If you want to load more than one LoRA, write the names separated by spaces.')
group.add_argument('--model-dir', type=str, default='user_data/models', help='Path to directory with all the models.')
diff --git a/modules/ui.py b/modules/ui.py
index db3adf0f..8ec4b165 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -310,7 +310,7 @@ def apply_interface_values(state, use_persistent=False):
def save_settings(state, preset, extensions_list, show_controls, theme_state):
output = copy.deepcopy(shared.settings)
- exclude = ['name2', 'greeting', 'context', 'truncation_length', 'instruction_template_str']
+ exclude = []
for k in state:
if k in shared.settings and k not in exclude:
output[k] = state[k]
@@ -323,6 +323,7 @@ def save_settings(state, preset, extensions_list, show_controls, theme_state):
output['seed'] = int(output['seed'])
output['show_controls'] = show_controls
output['dark_theme'] = True if theme_state == 'dark' else False
+ output.pop('instruction_template_str')
# Save extension values in the UI
for extension_name in extensions_list:
@@ -364,7 +365,7 @@ def store_current_state_and_debounce(interface_state, preset, extensions, show_c
if _auto_save_timer is not None:
_auto_save_timer.cancel()
- _auto_save_timer = threading.Timer(2.0, _perform_debounced_save)
+ _auto_save_timer = threading.Timer(1.0, _perform_debounced_save)
_auto_save_timer.start()
@@ -401,15 +402,52 @@ def setup_auto_save():
'chat-instruct_command',
'character_menu',
'name1',
+ 'name2',
+ 'context',
+ 'greeting',
'user_bio',
'custom_system_message',
'chat_template_str',
- # Parameters tab (ui_parameters.py)
+ # Parameters tab (ui_parameters.py) - Generation parameters
'preset_menu',
+ 'temperature',
+ 'dynatemp_low',
+ 'dynatemp_high',
+ 'dynatemp_exponent',
+ 'smoothing_factor',
+ 'smoothing_curve',
+ 'min_p',
+ 'top_p',
+ 'top_k',
+ 'typical_p',
+ 'xtc_threshold',
+ 'xtc_probability',
+ 'epsilon_cutoff',
+ 'eta_cutoff',
+ 'tfs',
+ 'top_a',
+ 'top_n_sigma',
+ 'dry_multiplier',
+ 'dry_allowed_length',
+ 'dry_base',
+ 'repetition_penalty',
+ 'frequency_penalty',
+ 'presence_penalty',
+ 'encoder_repetition_penalty',
+ 'no_repeat_ngram_size',
+ 'repetition_penalty_range',
+ 'penalty_alpha',
+ 'guidance_scale',
+ 'mirostat_mode',
+ 'mirostat_tau',
+ 'mirostat_eta',
'max_new_tokens',
'prompt_lookup_num_tokens',
'max_tokens_second',
+ 'do_sample',
+ 'dynamic_temperature',
+ 'temperature_last',
'auto_max_new_tokens',
'ban_eos_token',
'add_bos_token',
@@ -417,10 +455,14 @@ def setup_auto_save():
'skip_special_tokens',
'stream',
'static_cache',
+ 'truncation_length',
'seed',
+ 'sampler_priority',
'custom_stopping_strings',
'custom_token_bans',
'negative_prompt',
+ 'dry_sequence_breakers',
+ 'grammar_string',
# Default tab (ui_default.py)
'prompt_menu-default',
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 0d5a2c18..712843d3 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -86,7 +86,7 @@ def create_ui():
shared.gradio['web_search_pages'] = gr.Number(value=shared.settings.get('web_search_pages', 3), precision=0, label='Number of pages to download', minimum=1, maximum=10)
with gr.Row():
- shared.gradio['mode'] = gr.Radio(choices=['instruct', 'chat-instruct', 'chat'], value=shared.settings['mode'] if shared.settings['mode'] in ['chat', 'chat-instruct'] else None, label='Mode', info='Defines how the chat prompt is generated. In instruct and chat-instruct modes, the instruction template Parameters > Instruction template is used.', elem_id='chat-mode')
+ shared.gradio['mode'] = gr.Radio(choices=['instruct', 'chat-instruct', 'chat'], value=None, label='Mode', info='Defines how the chat prompt is generated. In instruct and chat-instruct modes, the instruction template Parameters > Instruction template is used.', elem_id='chat-mode')
with gr.Row():
shared.gradio['chat_style'] = gr.Dropdown(choices=utils.get_available_chat_styles(), label='Chat style', value=shared.settings['chat_style'], visible=shared.settings['mode'] != 'instruct')
@@ -118,14 +118,16 @@ def create_chat_settings_ui():
with gr.Column(scale=8):
with gr.Tab("Character"):
with gr.Row():
- shared.gradio['character_menu'] = gr.Dropdown(value=None, choices=utils.get_available_characters(), label='Character', elem_id='character-menu', info='Used in chat and chat-instruct modes.', elem_classes='slim-dropdown')
+ shared.gradio['character_menu'] = gr.Dropdown(value=shared.settings['character'], choices=utils.get_available_characters(), label='Character', elem_id='character-menu', info='Used in chat and chat-instruct modes.', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['character_menu'], lambda: None, lambda: {'choices': utils.get_available_characters()}, 'refresh-button', interactive=not mu)
shared.gradio['save_character'] = gr.Button('💾', elem_classes='refresh-button', elem_id="save-character", interactive=not mu)
shared.gradio['delete_character'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
+ shared.gradio['reset_character'] = gr.Button('↺', elem_classes='refresh-button', interactive=True)
+ shared.gradio['clear_character'] = gr.Button('⚪', elem_classes='refresh-button', interactive=True)
- shared.gradio['name2'] = gr.Textbox(value='', lines=1, label='Character\'s name')
- shared.gradio['context'] = gr.Textbox(value='', lines=10, label='Context', elem_classes=['add_scrollbar'])
- shared.gradio['greeting'] = gr.Textbox(value='', lines=5, label='Greeting', elem_classes=['add_scrollbar'])
+ shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name')
+ shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=10, label='Context', elem_classes=['add_scrollbar'])
+ shared.gradio['greeting'] = gr.Textbox(value=shared.settings['greeting'], lines=5, label='Greeting', elem_classes=['add_scrollbar'])
with gr.Tab("User"):
shared.gradio['name1'] = gr.Textbox(value=shared.settings['name1'], lines=1, label='Name')
@@ -178,7 +180,7 @@ def create_chat_settings_ui():
with gr.Row():
with gr.Column():
shared.gradio['custom_system_message'] = gr.Textbox(value=shared.settings['custom_system_message'], lines=2, label='Custom system message', info='If not empty, will be used instead of the default one.', elem_classes=['add_scrollbar'])
- shared.gradio['instruction_template_str'] = gr.Textbox(value='', label='Instruction template', lines=24, info='This gets autodetected; you usually don\'t need to change it. Used in instruct and chat-instruct modes.', elem_classes=['add_scrollbar', 'monospace'])
+ shared.gradio['instruction_template_str'] = gr.Textbox(value=shared.settings['instruction_template_str'], label='Instruction template', lines=24, info='This gets autodetected; you usually don\'t need to change it. Used in instruct and chat-instruct modes.', elem_classes=['add_scrollbar', 'monospace'])
with gr.Row():
shared.gradio['send_instruction_to_default'] = gr.Button('Send to default', elem_classes=['small-button'])
shared.gradio['send_instruction_to_notebook'] = gr.Button('Send to notebook', elem_classes=['small-button'])
@@ -294,6 +296,8 @@ def create_event_handlers():
chat.handle_character_menu_change, gradio('interface_state'), gradio('history', 'display', 'name1', 'name2', 'character_picture', 'greeting', 'context', 'unique_id'), show_progress=False).then(
None, None, None, js=f'() => {{{ui.update_big_picture_js}; updateBigPicture()}}')
+ shared.gradio['character_picture'].change(chat.handle_character_picture_change, gradio('character_picture'), None, show_progress=False)
+
shared.gradio['mode'].change(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
chat.handle_mode_change, gradio('interface_state'), gradio('history', 'display', 'chat_style', 'chat-instruct_command', 'unique_id'), show_progress=False).then(
@@ -317,6 +321,10 @@ def create_event_handlers():
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
chat.handle_save_template_click, gradio('instruction_template_str'), gradio('save_filename', 'save_root', 'save_contents', 'file_saver'), show_progress=False)
+ shared.gradio['reset_character'].click(
+ ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ chat.reset_character_for_ui, gradio('interface_state'), gradio('interface_state', 'name2', 'context', 'greeting', 'character_picture'), show_progress=False)
+
shared.gradio['delete_template'].click(chat.handle_delete_template_click, gradio('instruction_template'), gradio('delete_filename', 'delete_root', 'file_deleter'), show_progress=False)
shared.gradio['save_chat_history'].click(
lambda x: json.dumps(x, indent=4), gradio('history'), gradio('temporary_text')).then(
diff --git a/modules/ui_default.py b/modules/ui_default.py
index c2946b37..8acc4b10 100644
--- a/modules/ui_default.py
+++ b/modules/ui_default.py
@@ -19,7 +19,7 @@ def create_ui():
with gr.Row():
with gr.Column():
with gr.Row():
- shared.gradio['textbox-default'] = gr.Textbox(value='', lines=27, label='Input', elem_classes=['textbox_default', 'add_scrollbar'])
+ shared.gradio['textbox-default'] = gr.Textbox(value=load_prompt(shared.settings['prompt-default']), lines=27, label='Input', elem_classes=['textbox_default', 'add_scrollbar'])
shared.gradio['token-counter-default'] = gr.HTML(value="0", elem_id="default-token-counter")
with gr.Row():
@@ -28,7 +28,7 @@ def create_ui():
shared.gradio['Generate-default'] = gr.Button('Generate', variant='primary')
with gr.Row():
- shared.gradio['prompt_menu-default'] = gr.Dropdown(choices=utils.get_available_prompts(), value='None', label='Prompt', elem_classes='slim-dropdown')
+ shared.gradio['prompt_menu-default'] = gr.Dropdown(choices=utils.get_available_prompts(), value=shared.settings['prompt-default'], label='Prompt', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['prompt_menu-default'], lambda: None, lambda: {'choices': utils.get_available_prompts()}, 'refresh-button', interactive=not mu)
shared.gradio['save_prompt-default'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_prompt-default'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
diff --git a/modules/ui_notebook.py b/modules/ui_notebook.py
index b234ac57..3f79a93c 100644
--- a/modules/ui_notebook.py
+++ b/modules/ui_notebook.py
@@ -22,7 +22,7 @@ def create_ui():
with gr.Column(scale=4):
with gr.Tab('Raw'):
with gr.Row():
- shared.gradio['textbox-notebook'] = gr.Textbox(value='', lines=27, elem_id='textbox-notebook', elem_classes=['textbox', 'add_scrollbar'])
+ shared.gradio['textbox-notebook'] = gr.Textbox(value=load_prompt(shared.settings['prompt-notebook']), lines=27, elem_id='textbox-notebook', elem_classes=['textbox', 'add_scrollbar'])
shared.gradio['token-counter-notebook'] = gr.HTML(value="0", elem_id="notebook-token-counter")
with gr.Tab('Markdown'):
@@ -56,7 +56,7 @@ def create_ui():
with gr.Column(scale=1):
gr.HTML('')
with gr.Row():
- shared.gradio['prompt_menu-notebook'] = gr.Dropdown(choices=utils.get_available_prompts(), value='None', label='Prompt', elem_classes='slim-dropdown')
+ shared.gradio['prompt_menu-notebook'] = gr.Dropdown(choices=utils.get_available_prompts(), value=shared.settings['prompt-notebook'], label='Prompt', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['prompt_menu-notebook'], lambda: None, lambda: {'choices': utils.get_available_prompts()}, ['refresh-button', 'refresh-button-small'], interactive=not mu)
shared.gradio['save_prompt-notebook'] = gr.Button('💾', elem_classes=['refresh-button', 'refresh-button-small'], interactive=not mu)
shared.gradio['delete_prompt-notebook'] = gr.Button('🗑️', elem_classes=['refresh-button', 'refresh-button-small'], interactive=not mu)
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index 84f9fbfc..ff965d39 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -6,19 +6,19 @@ from modules import loaders, presets, shared, ui, ui_chat, utils
from modules.utils import gradio
-def create_ui(default_preset):
+def create_ui():
mu = shared.args.multi_user
- generate_params = presets.load_preset(default_preset)
with gr.Tab("Parameters", elem_id="parameters"):
with gr.Tab("Generation"):
with gr.Row():
with gr.Column():
with gr.Row():
- shared.gradio['preset_menu'] = gr.Dropdown(choices=utils.get_available_presets(), value=default_preset, label='Preset', elem_classes='slim-dropdown')
+ shared.gradio['preset_menu'] = gr.Dropdown(choices=utils.get_available_presets(), value=shared.settings['preset'], label='Preset', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['preset_menu'], lambda: None, lambda: {'choices': utils.get_available_presets()}, 'refresh-button', interactive=not mu)
shared.gradio['save_preset'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_preset'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['random_preset'] = gr.Button('🎲', elem_classes='refresh-button')
+ shared.gradio['reset_preset'] = gr.Button('Reload preset', elem_classes='refresh-button', interactive=True)
+ shared.gradio['neutralize_samplers'] = gr.Button('Neutralize samplers', elem_classes='refresh-button', interactive=True)
with gr.Column():
shared.gradio['filter_by_loader'] = gr.Dropdown(label="Filter by loader", choices=["All"] + list(loaders.loaders_and_params.keys()) if not shared.args.portable else ['llama.cpp'], value="All", elem_classes='slim-dropdown')
@@ -28,44 +28,44 @@ def create_ui(default_preset):
with gr.Row():
with gr.Column():
gr.Markdown('## Curve shape')
- shared.gradio['temperature'] = gr.Slider(0.01, 5, value=generate_params['temperature'], step=0.01, label='temperature')
- shared.gradio['dynatemp_low'] = gr.Slider(0.01, 5, value=generate_params['dynatemp_low'], step=0.01, label='dynatemp_low', visible=generate_params['dynamic_temperature'])
- shared.gradio['dynatemp_high'] = gr.Slider(0.01, 5, value=generate_params['dynatemp_high'], step=0.01, label='dynatemp_high', visible=generate_params['dynamic_temperature'])
- shared.gradio['dynatemp_exponent'] = gr.Slider(0.01, 5, value=generate_params['dynatemp_exponent'], step=0.01, label='dynatemp_exponent', visible=generate_params['dynamic_temperature'])
- shared.gradio['smoothing_factor'] = gr.Slider(0.0, 10.0, value=generate_params['smoothing_factor'], step=0.01, label='smoothing_factor', info='Activates Quadratic Sampling.')
- shared.gradio['smoothing_curve'] = gr.Slider(1.0, 10.0, value=generate_params['smoothing_curve'], step=0.01, label='smoothing_curve', info='Adjusts the dropoff curve of Quadratic Sampling.')
+ shared.gradio['temperature'] = gr.Slider(0.01, 5, value=shared.settings['temperature'], step=0.01, label='temperature')
+ shared.gradio['dynatemp_low'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_low'], step=0.01, label='dynatemp_low', visible=shared.settings['dynamic_temperature'])
+ shared.gradio['dynatemp_high'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_high'], step=0.01, label='dynatemp_high', visible=shared.settings['dynamic_temperature'])
+ shared.gradio['dynatemp_exponent'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_exponent'], step=0.01, label='dynatemp_exponent', visible=shared.settings['dynamic_temperature'])
+ shared.gradio['smoothing_factor'] = gr.Slider(0.0, 10.0, value=shared.settings['smoothing_factor'], step=0.01, label='smoothing_factor', info='Activates Quadratic Sampling.')
+ shared.gradio['smoothing_curve'] = gr.Slider(1.0, 10.0, value=shared.settings['smoothing_curve'], step=0.01, label='smoothing_curve', info='Adjusts the dropoff curve of Quadratic Sampling.')
gr.Markdown('## Curve cutoff')
- shared.gradio['min_p'] = gr.Slider(0.0, 1.0, value=generate_params['min_p'], step=0.01, label='min_p')
- shared.gradio['top_n_sigma'] = gr.Slider(0.0, 5.0, value=generate_params['top_n_sigma'], step=0.01, label='top_n_sigma')
- shared.gradio['top_p'] = gr.Slider(0.0, 1.0, value=generate_params['top_p'], step=0.01, label='top_p')
- shared.gradio['top_k'] = gr.Slider(0, 200, value=generate_params['top_k'], step=1, label='top_k')
- shared.gradio['typical_p'] = gr.Slider(0.0, 1.0, value=generate_params['typical_p'], step=0.01, label='typical_p')
- shared.gradio['xtc_threshold'] = gr.Slider(0, 0.5, value=generate_params['xtc_threshold'], step=0.01, label='xtc_threshold', info='If 2 or more tokens have probability above this threshold, consider removing all but the last one.')
- shared.gradio['xtc_probability'] = gr.Slider(0, 1, value=generate_params['xtc_probability'], step=0.01, label='xtc_probability', info='Probability that the removal will actually happen. 0 disables the sampler. 1 makes it always happen.')
- shared.gradio['epsilon_cutoff'] = gr.Slider(0, 9, value=generate_params['epsilon_cutoff'], step=0.01, label='epsilon_cutoff')
- shared.gradio['eta_cutoff'] = gr.Slider(0, 20, value=generate_params['eta_cutoff'], step=0.01, label='eta_cutoff')
- shared.gradio['tfs'] = gr.Slider(0.0, 1.0, value=generate_params['tfs'], step=0.01, label='tfs')
- shared.gradio['top_a'] = gr.Slider(0.0, 1.0, value=generate_params['top_a'], step=0.01, label='top_a')
+ shared.gradio['min_p'] = gr.Slider(0.0, 1.0, value=shared.settings['min_p'], step=0.01, label='min_p')
+ shared.gradio['top_n_sigma'] = gr.Slider(0.0, 5.0, value=shared.settings['top_n_sigma'], step=0.01, label='top_n_sigma')
+ shared.gradio['top_p'] = gr.Slider(0.0, 1.0, value=shared.settings['top_p'], step=0.01, label='top_p')
+ shared.gradio['top_k'] = gr.Slider(0, 200, value=shared.settings['top_k'], step=1, label='top_k')
+ shared.gradio['typical_p'] = gr.Slider(0.0, 1.0, value=shared.settings['typical_p'], step=0.01, label='typical_p')
+ shared.gradio['xtc_threshold'] = gr.Slider(0, 0.5, value=shared.settings['xtc_threshold'], step=0.01, label='xtc_threshold', info='If 2 or more tokens have probability above this threshold, consider removing all but the last one.')
+ shared.gradio['xtc_probability'] = gr.Slider(0, 1, value=shared.settings['xtc_probability'], step=0.01, label='xtc_probability', info='Probability that the removal will actually happen. 0 disables the sampler. 1 makes it always happen.')
+ shared.gradio['epsilon_cutoff'] = gr.Slider(0, 9, value=shared.settings['epsilon_cutoff'], step=0.01, label='epsilon_cutoff')
+ shared.gradio['eta_cutoff'] = gr.Slider(0, 20, value=shared.settings['eta_cutoff'], step=0.01, label='eta_cutoff')
+ shared.gradio['tfs'] = gr.Slider(0.0, 1.0, value=shared.settings['tfs'], step=0.01, label='tfs')
+ shared.gradio['top_a'] = gr.Slider(0.0, 1.0, value=shared.settings['top_a'], step=0.01, label='top_a')
gr.Markdown('## Repetition suppression')
- shared.gradio['dry_multiplier'] = gr.Slider(0, 5, value=generate_params['dry_multiplier'], step=0.01, label='dry_multiplier', info='Set to greater than 0 to enable DRY. Recommended value: 0.8.')
- shared.gradio['dry_allowed_length'] = gr.Slider(1, 20, value=generate_params['dry_allowed_length'], step=1, label='dry_allowed_length', info='Longest sequence that can be repeated without being penalized.')
- shared.gradio['dry_base'] = gr.Slider(1, 4, value=generate_params['dry_base'], step=0.01, label='dry_base', info='Controls how fast the penalty grows with increasing sequence length.')
- shared.gradio['repetition_penalty'] = gr.Slider(1.0, 1.5, value=generate_params['repetition_penalty'], step=0.01, label='repetition_penalty')
- shared.gradio['frequency_penalty'] = gr.Slider(0, 2, value=generate_params['frequency_penalty'], step=0.05, label='frequency_penalty')
- shared.gradio['presence_penalty'] = gr.Slider(0, 2, value=generate_params['presence_penalty'], step=0.05, label='presence_penalty')
- shared.gradio['encoder_repetition_penalty'] = gr.Slider(0.8, 1.5, value=generate_params['encoder_repetition_penalty'], step=0.01, label='encoder_repetition_penalty')
- shared.gradio['no_repeat_ngram_size'] = gr.Slider(0, 20, step=1, value=generate_params['no_repeat_ngram_size'], label='no_repeat_ngram_size')
- shared.gradio['repetition_penalty_range'] = gr.Slider(0, 4096, step=64, value=generate_params['repetition_penalty_range'], label='repetition_penalty_range')
+ shared.gradio['dry_multiplier'] = gr.Slider(0, 5, value=shared.settings['dry_multiplier'], step=0.01, label='dry_multiplier', info='Set to greater than 0 to enable DRY. Recommended value: 0.8.')
+ shared.gradio['dry_allowed_length'] = gr.Slider(1, 20, value=shared.settings['dry_allowed_length'], step=1, label='dry_allowed_length', info='Longest sequence that can be repeated without being penalized.')
+ shared.gradio['dry_base'] = gr.Slider(1, 4, value=shared.settings['dry_base'], step=0.01, label='dry_base', info='Controls how fast the penalty grows with increasing sequence length.')
+ shared.gradio['repetition_penalty'] = gr.Slider(1.0, 1.5, value=shared.settings['repetition_penalty'], step=0.01, label='repetition_penalty')
+ shared.gradio['frequency_penalty'] = gr.Slider(0, 2, value=shared.settings['frequency_penalty'], step=0.05, label='frequency_penalty')
+ shared.gradio['presence_penalty'] = gr.Slider(0, 2, value=shared.settings['presence_penalty'], step=0.05, label='presence_penalty')
+ shared.gradio['encoder_repetition_penalty'] = gr.Slider(0.8, 1.5, value=shared.settings['encoder_repetition_penalty'], step=0.01, label='encoder_repetition_penalty')
+ shared.gradio['no_repeat_ngram_size'] = gr.Slider(0, 20, step=1, value=shared.settings['no_repeat_ngram_size'], label='no_repeat_ngram_size')
+ shared.gradio['repetition_penalty_range'] = gr.Slider(0, 4096, step=64, value=shared.settings['repetition_penalty_range'], label='repetition_penalty_range')
with gr.Column():
gr.Markdown('## Alternative sampling methods')
- shared.gradio['penalty_alpha'] = gr.Slider(0, 5, value=generate_params['penalty_alpha'], label='penalty_alpha', info='For Contrastive Search. do_sample must be unchecked.')
- shared.gradio['guidance_scale'] = gr.Slider(-0.5, 2.5, step=0.05, value=generate_params['guidance_scale'], label='guidance_scale', info='For CFG. 1.5 is a good value.')
- shared.gradio['mirostat_mode'] = gr.Slider(0, 2, step=1, value=generate_params['mirostat_mode'], label='mirostat_mode', info='mode=1 is for llama.cpp only.')
- shared.gradio['mirostat_tau'] = gr.Slider(0, 10, step=0.01, value=generate_params['mirostat_tau'], label='mirostat_tau')
- shared.gradio['mirostat_eta'] = gr.Slider(0, 1, step=0.01, value=generate_params['mirostat_eta'], label='mirostat_eta')
+ shared.gradio['penalty_alpha'] = gr.Slider(0, 5, value=shared.settings['penalty_alpha'], label='penalty_alpha', info='For Contrastive Search. do_sample must be unchecked.')
+ shared.gradio['guidance_scale'] = gr.Slider(-0.5, 2.5, step=0.05, value=shared.settings['guidance_scale'], label='guidance_scale', info='For CFG. 1.5 is a good value.')
+ shared.gradio['mirostat_mode'] = gr.Slider(0, 2, step=1, value=shared.settings['mirostat_mode'], label='mirostat_mode', info='mode=1 is for llama.cpp only.')
+ shared.gradio['mirostat_tau'] = gr.Slider(0, 10, step=0.01, value=shared.settings['mirostat_tau'], label='mirostat_tau')
+ shared.gradio['mirostat_eta'] = gr.Slider(0, 1, step=0.01, value=shared.settings['mirostat_eta'], label='mirostat_eta')
gr.Markdown('## Other options')
shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.')
@@ -74,9 +74,9 @@ def create_ui(default_preset):
with gr.Column():
with gr.Row():
with gr.Column():
- shared.gradio['do_sample'] = gr.Checkbox(value=generate_params['do_sample'], label='do_sample')
- shared.gradio['dynamic_temperature'] = gr.Checkbox(value=generate_params['dynamic_temperature'], label='dynamic_temperature')
- shared.gradio['temperature_last'] = gr.Checkbox(value=generate_params['temperature_last'], label='temperature_last', info='Moves temperature/dynamic temperature/quadratic sampling to the end of the sampler stack, ignoring their positions in "Sampler priority".')
+ shared.gradio['do_sample'] = gr.Checkbox(value=shared.settings['do_sample'], label='do_sample')
+ shared.gradio['dynamic_temperature'] = gr.Checkbox(value=shared.settings['dynamic_temperature'], label='dynamic_temperature')
+ shared.gradio['temperature_last'] = gr.Checkbox(value=shared.settings['temperature_last'], label='temperature_last', info='Moves temperature/dynamic temperature/quadratic sampling to the end of the sampler stack, ignoring their positions in "Sampler priority".')
shared.gradio['auto_max_new_tokens'] = gr.Checkbox(value=shared.settings['auto_max_new_tokens'], label='auto_max_new_tokens', info='Expand max_new_tokens to the available context length.')
shared.gradio['ban_eos_token'] = gr.Checkbox(value=shared.settings['ban_eos_token'], label='Ban the eos_token', info='Forces the model to never end the generation prematurely.')
shared.gradio['add_bos_token'] = gr.Checkbox(value=shared.settings['add_bos_token'], label='Add the bos_token to the beginning of prompts', info='Disabling this can make the replies more creative.')
@@ -89,18 +89,18 @@ def create_ui(default_preset):
shared.gradio['truncation_length'] = gr.Number(precision=0, step=256, value=get_truncation_length(), label='Truncate the prompt up to this length', info='The leftmost tokens are removed if the prompt exceeds this length.')
shared.gradio['seed'] = gr.Number(value=shared.settings['seed'], label='Seed (-1 for random)')
- shared.gradio['sampler_priority'] = gr.Textbox(value=generate_params['sampler_priority'], lines=12, label='Sampler priority', info='Parameter names separated by new lines or commas.', elem_classes=['add_scrollbar'])
+ shared.gradio['sampler_priority'] = gr.Textbox(value=shared.settings['sampler_priority'], lines=12, label='Sampler priority', info='Parameter names separated by new lines or commas.', elem_classes=['add_scrollbar'])
shared.gradio['custom_stopping_strings'] = gr.Textbox(lines=2, value=shared.settings["custom_stopping_strings"] or None, label='Custom stopping strings', info='Written between "" and separated by commas.', placeholder='"\\n", "\\nYou:"')
shared.gradio['custom_token_bans'] = gr.Textbox(value=shared.settings['custom_token_bans'] or None, label='Token bans', info='Token IDs to ban, separated by commas. The IDs can be found in the Default or Notebook tab.')
shared.gradio['negative_prompt'] = gr.Textbox(value=shared.settings['negative_prompt'], label='Negative prompt', info='For CFG. Only used when guidance_scale is different than 1.', lines=3, elem_classes=['add_scrollbar'])
- shared.gradio['dry_sequence_breakers'] = gr.Textbox(value=generate_params['dry_sequence_breakers'], label='dry_sequence_breakers', info='Tokens across which sequence matching is not continued. Specified as a comma-separated list of quoted strings.')
+ shared.gradio['dry_sequence_breakers'] = gr.Textbox(value=shared.settings['dry_sequence_breakers'], label='dry_sequence_breakers', info='Tokens across which sequence matching is not continued. Specified as a comma-separated list of quoted strings.')
with gr.Row() as shared.gradio['grammar_file_row']:
shared.gradio['grammar_file'] = gr.Dropdown(value='None', choices=utils.get_available_grammars(), label='Load grammar from file (.gbnf)', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['grammar_file'], lambda: None, lambda: {'choices': utils.get_available_grammars()}, 'refresh-button', interactive=not mu)
shared.gradio['save_grammar'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_grammar'] = gr.Button('🗑️ ', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['grammar_string'] = gr.Textbox(value='', label='Grammar', lines=16, elem_classes=['add_scrollbar', 'monospace'])
+ shared.gradio['grammar_string'] = gr.Textbox(value=shared.settings['grammar_string'], label='Grammar', lines=16, elem_classes=['add_scrollbar', 'monospace'])
ui_chat.create_chat_settings_ui()
@@ -111,9 +111,13 @@ def create_event_handlers():
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
presets.load_preset_for_ui, gradio('preset_menu', 'interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
- shared.gradio['random_preset'].click(
+ shared.gradio['reset_preset'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
- presets.random_preset, gradio('interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
+ presets.reset_preset_for_ui, gradio('preset_menu', 'interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
+
+ shared.gradio['neutralize_samplers'].click(
+ ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ presets.neutralize_samplers_for_ui, gradio('interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
shared.gradio['grammar_file'].change(load_grammar, gradio('grammar_file'), gradio('grammar_string'), show_progress=False)
shared.gradio['dynamic_temperature'].change(lambda x: [gr.update(visible=x)] * 3, gradio('dynamic_temperature'), gradio('dynatemp_low', 'dynatemp_high', 'dynatemp_exponent'), show_progress=False)
diff --git a/server.py b/server.py
index 3be4c27c..3dd0a3f3 100644
--- a/server.py
+++ b/server.py
@@ -45,6 +45,7 @@ from modules import (
ui_session,
utils
)
+from modules.chat import generate_pfp_cache
from modules.extensions import apply_extensions
from modules.LoRA import add_lora_to_model
from modules.models import load_model, unload_model_if_idle
@@ -93,17 +94,20 @@ def create_interface():
# Force some events to be triggered on page load
shared.persistent_interface_state.update({
+ 'mode': shared.settings['mode'],
'loader': shared.args.loader or 'llama.cpp',
- 'mode': shared.settings['mode'] if shared.settings['mode'] == 'instruct' else gr.update(),
- 'character_menu': shared.args.character or shared.settings['character'],
- 'instruction_template_str': shared.settings['instruction_template_str'],
- 'prompt_menu-default': shared.settings['prompt-default'],
- 'prompt_menu-notebook': shared.settings['prompt-notebook'],
'filter_by_loader': (shared.args.loader or 'All') if not shared.args.portable else 'llama.cpp'
})
- if Path("user_data/cache/pfp_character.png").exists():
- Path("user_data/cache/pfp_character.png").unlink()
+ # Clear existing cache files
+ for cache_file in ['pfp_character.png', 'pfp_character_thumb.png']:
+ cache_path = Path(f"user_data/cache/{cache_file}")
+ if cache_path.exists():
+ cache_path.unlink()
+
+ # Regenerate for default character
+ if shared.settings['mode'] != 'instruct':
+ generate_pfp_cache(shared.settings['character'])
# css/js strings
css = ui.css
@@ -134,7 +138,7 @@ def create_interface():
ui_default.create_ui()
ui_notebook.create_ui()
- ui_parameters.create_ui(shared.settings['preset']) # Parameters tab
+ ui_parameters.create_ui() # Parameters tab
ui_model_menu.create_ui() # Model tab
if not shared.args.portable:
training.create_ui() # Training tab
From 3650a6fd1fc332f462bc1aa08f7671a3d517d847 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:02:34 -0700
Subject: [PATCH 37/82] Small UI changes
---
css/main.css | 2 +-
modules/ui_chat.py | 3 +--
modules/ui_parameters.py | 2 +-
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/css/main.css b/css/main.css
index a9cb36ab..4277519a 100644
--- a/css/main.css
+++ b/css/main.css
@@ -53,7 +53,7 @@ div.svelte-iyf88w {
}
.refresh-button {
- max-width: 4.4em;
+ max-width: none;
min-width: 2.2em !important;
height: 39.594px;
align-self: end;
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 712843d3..f8e2fc32 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -122,8 +122,7 @@ def create_chat_settings_ui():
ui.create_refresh_button(shared.gradio['character_menu'], lambda: None, lambda: {'choices': utils.get_available_characters()}, 'refresh-button', interactive=not mu)
shared.gradio['save_character'] = gr.Button('💾', elem_classes='refresh-button', elem_id="save-character", interactive=not mu)
shared.gradio['delete_character'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['reset_character'] = gr.Button('↺', elem_classes='refresh-button', interactive=True)
- shared.gradio['clear_character'] = gr.Button('⚪', elem_classes='refresh-button', interactive=True)
+ shared.gradio['reset_character'] = gr.Button('Restore character', elem_classes='refresh-button', interactive=True)
shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name')
shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=10, label='Context', elem_classes=['add_scrollbar'])
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index ff965d39..9b5cb3ab 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -17,7 +17,7 @@ def create_ui():
ui.create_refresh_button(shared.gradio['preset_menu'], lambda: None, lambda: {'choices': utils.get_available_presets()}, 'refresh-button', interactive=not mu)
shared.gradio['save_preset'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_preset'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['reset_preset'] = gr.Button('Reload preset', elem_classes='refresh-button', interactive=True)
+ shared.gradio['reset_preset'] = gr.Button('Restore preset', elem_classes='refresh-button', interactive=True)
shared.gradio['neutralize_samplers'] = gr.Button('Neutralize samplers', elem_classes='refresh-button', interactive=True)
with gr.Column():
From caf9fca5f339e4018913608f8accb3897352853f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:11:35 -0700
Subject: [PATCH 38/82] Avoid some code repetition
---
modules/shared.py | 190 +++++++++++++++++++++++-----------------------
1 file changed, 97 insertions(+), 93 deletions(-)
diff --git a/modules/shared.py b/modules/shared.py
index f4f8e180..08200399 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -9,6 +9,7 @@ from pathlib import Path
import yaml
from modules.logging_colors import logger
+from modules.presets import default_preset
# Model variables
model = None
@@ -28,99 +29,6 @@ gradio = {}
persistent_interface_state = {}
need_restart = False
-# UI defaults
-settings = {
- 'show_controls': True,
- 'start_with': '',
- 'mode': 'instruct',
- 'chat_style': 'cai-chat',
- 'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
- 'enable_web_search': False,
- 'web_search_pages': 3,
- 'prompt-default': 'QA',
- 'prompt-notebook': 'QA',
- 'preset': 'min_p',
- 'max_new_tokens': 512,
- 'max_new_tokens_min': 1,
- 'max_new_tokens_max': 4096,
- 'prompt_lookup_num_tokens': 0,
- 'max_tokens_second': 0,
- 'auto_max_new_tokens': True,
- 'ban_eos_token': False,
- 'add_bos_token': True,
- 'enable_thinking': True,
- 'skip_special_tokens': True,
- 'stream': True,
- 'static_cache': False,
- 'truncation_length': 8192,
- 'seed': -1,
- 'custom_stopping_strings': '',
- 'custom_token_bans': '',
- 'negative_prompt': '',
- 'dark_theme': True,
- 'default_extensions': [],
-
- # Character settings
- 'character': 'Assistant',
- 'name1': 'You',
- 'name2': 'AI',
- 'user_bio': '',
- 'context': 'The following is a conversation with an AI Large Language Model. The AI has been trained to answer questions, provide recommendations, and help with decision making. The AI follows user requests. The AI thinks outside the box.',
- 'greeting': 'How can I help you today?',
- 'custom_system_message': '',
- 'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}",
- 'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
-
- # Generation parameters - Curve shape
- 'temperature': 1.0,
- 'dynatemp_low': 1.0,
- 'dynatemp_high': 1.0,
- 'dynatemp_exponent': 1.0,
- 'smoothing_factor': 0.0,
- 'smoothing_curve': 1.0,
-
- # Generation parameters - Curve cutoff
- 'min_p': 0.0,
- 'top_p': 1.0,
- 'top_k': 0,
- 'typical_p': 1.0,
- 'xtc_threshold': 0.1,
- 'xtc_probability': 0.0,
- 'epsilon_cutoff': 0.0,
- 'eta_cutoff': 0.0,
- 'tfs': 1.0,
- 'top_a': 0.0,
- 'top_n_sigma': 0.0,
-
- # Generation parameters - Repetition suppression
- 'dry_multiplier': 0.0,
- 'dry_allowed_length': 2,
- 'dry_base': 1.75,
- 'repetition_penalty': 1.0,
- 'frequency_penalty': 0.0,
- 'presence_penalty': 0.0,
- 'encoder_repetition_penalty': 1.0,
- 'no_repeat_ngram_size': 0,
- 'repetition_penalty_range': 1024,
-
- # Generation parameters - Alternative sampling methods
- 'penalty_alpha': 0.0,
- 'guidance_scale': 1.0,
- 'mirostat_mode': 0,
- 'mirostat_tau': 5.0,
- 'mirostat_eta': 0.1,
-
- # Generation parameters - Other options
- 'do_sample': True,
- 'dynamic_temperature': False,
- 'temperature_last': False,
- 'sampler_priority': 'repetition_penalty\npresence_penalty\nfrequency_penalty\ndry\ntop_n_sigma\ntemperature\ndynamic_temperature\nquadratic_sampling\ntop_k\ntop_p\ntypical_p\nepsilon_cutoff\neta_cutoff\ntfs\ntop_a\nmin_p\nmirostat\nxtc\nencoder_repetition_penalty\nno_repeat_ngram',
- 'dry_sequence_breakers': '"\\n", ":", "\\"", "*"',
- 'grammar_string': '',
-}
-
-default_settings = copy.deepcopy(settings)
-
# Parser copied from https://github.com/vladmandic/automatic
parser = argparse.ArgumentParser(description="Text generation web UI", conflict_handler='resolve', add_help=True, formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
@@ -282,6 +190,102 @@ for arg in sys.argv[1:]:
elif hasattr(args, arg):
provided_arguments.append(arg)
+# Default generation parameters
+neutral_samplers = default_preset()
+
+# UI defaults
+settings = {
+ 'show_controls': True,
+ 'start_with': '',
+ 'mode': 'instruct',
+ 'chat_style': 'cai-chat',
+ 'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
+ 'enable_web_search': False,
+ 'web_search_pages': 3,
+ 'prompt-default': 'QA',
+ 'prompt-notebook': 'QA',
+ 'preset': 'min_p',
+ 'max_new_tokens': 512,
+ 'max_new_tokens_min': 1,
+ 'max_new_tokens_max': 4096,
+ 'prompt_lookup_num_tokens': 0,
+ 'max_tokens_second': 0,
+ 'auto_max_new_tokens': True,
+ 'ban_eos_token': False,
+ 'add_bos_token': True,
+ 'enable_thinking': True,
+ 'skip_special_tokens': True,
+ 'stream': True,
+ 'static_cache': False,
+ 'truncation_length': 8192,
+ 'seed': -1,
+ 'custom_stopping_strings': '',
+ 'custom_token_bans': '',
+ 'negative_prompt': '',
+ 'dark_theme': True,
+ 'default_extensions': [],
+
+ # Character settings
+ 'character': 'Assistant',
+ 'name1': 'You',
+ 'name2': 'AI',
+ 'user_bio': '',
+ 'context': 'The following is a conversation with an AI Large Language Model. The AI has been trained to answer questions, provide recommendations, and help with decision making. The AI follows user requests. The AI thinks outside the box.',
+ 'greeting': 'How can I help you today?',
+ 'custom_system_message': '',
+ 'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}",
+ 'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
+
+ # Generation parameters - Curve shape
+ 'temperature': neutral_samplers['temperature'],
+ 'dynatemp_low': neutral_samplers['dynatemp_low'],
+ 'dynatemp_high': neutral_samplers['dynatemp_high'],
+ 'dynatemp_exponent': neutral_samplers['dynatemp_exponent'],
+ 'smoothing_factor': neutral_samplers['smoothing_factor'],
+ 'smoothing_curve': neutral_samplers['smoothing_curve'],
+
+ # Generation parameters - Curve cutoff
+ 'min_p': neutral_samplers['min_p'],
+ 'top_p': neutral_samplers['top_p'],
+ 'top_k': neutral_samplers['top_k'],
+ 'typical_p': neutral_samplers['typical_p'],
+ 'xtc_threshold': neutral_samplers['xtc_threshold'],
+ 'xtc_probability': neutral_samplers['xtc_probability'],
+ 'epsilon_cutoff': neutral_samplers['epsilon_cutoff'],
+ 'eta_cutoff': neutral_samplers['eta_cutoff'],
+ 'tfs': neutral_samplers['tfs'],
+ 'top_a': neutral_samplers['top_a'],
+ 'top_n_sigma': neutral_samplers['top_n_sigma'],
+
+ # Generation parameters - Repetition suppression
+ 'dry_multiplier': neutral_samplers['dry_multiplier'],
+ 'dry_allowed_length': neutral_samplers['dry_allowed_length'],
+ 'dry_base': neutral_samplers['dry_base'],
+ 'repetition_penalty': neutral_samplers['repetition_penalty'],
+ 'frequency_penalty': neutral_samplers['frequency_penalty'],
+ 'presence_penalty': neutral_samplers['presence_penalty'],
+ 'encoder_repetition_penalty': neutral_samplers['encoder_repetition_penalty'],
+ 'no_repeat_ngram_size': neutral_samplers['no_repeat_ngram_size'],
+ 'repetition_penalty_range': neutral_samplers['repetition_penalty_range'],
+
+ # Generation parameters - Alternative sampling methods
+ 'penalty_alpha': neutral_samplers['penalty_alpha'],
+ 'guidance_scale': neutral_samplers['guidance_scale'],
+ 'mirostat_mode': neutral_samplers['mirostat_mode'],
+ 'mirostat_tau': neutral_samplers['mirostat_tau'],
+ 'mirostat_eta': neutral_samplers['mirostat_eta'],
+
+ # Generation parameters - Other options
+ 'do_sample': neutral_samplers['do_sample'],
+ 'dynamic_temperature': neutral_samplers['dynamic_temperature'],
+ 'temperature_last': neutral_samplers['temperature_last'],
+ 'sampler_priority': neutral_samplers['sampler_priority'],
+ 'dry_sequence_breakers': neutral_samplers['dry_sequence_breakers'],
+ 'grammar_string': '',
+}
+
+default_settings = copy.deepcopy(settings)
+
def do_cmd_flags_warnings():
# Security warnings
From fe955cac1fd7298d820e42790e257d9ace1c5eb4 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:15:19 -0700
Subject: [PATCH 39/82] Small UI changes
---
css/main.css | 4 ++++
modules/ui_chat.py | 8 ++++----
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/css/main.css b/css/main.css
index 4277519a..7d7b0cbd 100644
--- a/css/main.css
+++ b/css/main.css
@@ -62,6 +62,10 @@ div.svelte-iyf88w {
flex: none;
}
+.refresh-button-medium {
+ max-width: 4.4em;
+}
+
.refresh-button-small {
max-width: 2.2em;
}
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index f8e2fc32..ef9330e4 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -24,11 +24,11 @@ def create_ui():
with gr.Row(elem_id='past-chats-row', elem_classes=['pretty_scrollbar']):
with gr.Column():
with gr.Row(elem_id='past-chats-buttons'):
- shared.gradio['branch_chat'] = gr.Button('Branch', elem_classes='refresh-button', elem_id='Branch', interactive=not mu)
- shared.gradio['branch_index'] = gr.Number(value=-1, precision=0, visible=False, elem_id="Branch-index", interactive=True)
- shared.gradio['rename_chat'] = gr.Button('Rename', elem_classes='refresh-button', interactive=not mu)
+ shared.gradio['branch_chat'] = gr.Button('Branch', elem_classes=['refresh-button', 'refresh-button-medium'], elem_id='Branch', interactive=not mu)
+ shared.gradio['rename_chat'] = gr.Button('Rename', elem_classes=['refresh-button', 'refresh-button-medium'], interactive=not mu)
shared.gradio['delete_chat'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['Start new chat'] = gr.Button('New chat', elem_classes=['refresh-button', 'focus-on-chat-input'])
+ shared.gradio['Start new chat'] = gr.Button('New chat', elem_classes=['refresh-button', 'refresh-button-medium', 'focus-on-chat-input'])
+ shared.gradio['branch_index'] = gr.Number(value=-1, precision=0, visible=False, elem_id="Branch-index", interactive=True)
shared.gradio['search_chat'] = gr.Textbox(placeholder='Search chats...', max_lines=1, elem_id='search_chat')
From 0dbc4cbc71e7eed1d51fd9169048a54cb8d2d927 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:20:58 -0700
Subject: [PATCH 40/82] Add Qwen3 presets
---
user_data/presets/Qwen3 - No Thinking.yaml | 3 +++
user_data/presets/Qwen3 - Thinking.yaml | 3 +++
2 files changed, 6 insertions(+)
create mode 100644 user_data/presets/Qwen3 - No Thinking.yaml
create mode 100644 user_data/presets/Qwen3 - Thinking.yaml
diff --git a/user_data/presets/Qwen3 - No Thinking.yaml b/user_data/presets/Qwen3 - No Thinking.yaml
new file mode 100644
index 00000000..b1c1e03c
--- /dev/null
+++ b/user_data/presets/Qwen3 - No Thinking.yaml
@@ -0,0 +1,3 @@
+temperature: 0.7
+top_p: 0.8
+top_k: 20
diff --git a/user_data/presets/Qwen3 - Thinking.yaml b/user_data/presets/Qwen3 - Thinking.yaml
new file mode 100644
index 00000000..cb2942f9
--- /dev/null
+++ b/user_data/presets/Qwen3 - Thinking.yaml
@@ -0,0 +1,3 @@
+temperature: 0.6
+top_p: 0.95
+top_k: 20
From 1bdf11b511b69f6eb10f9067847c64d39c8872f4 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:23:09 -0700
Subject: [PATCH 41/82] Use the Qwen3 - Thinking preset by default
---
modules/shared.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/modules/shared.py b/modules/shared.py
index 08200399..3794206b 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -204,7 +204,7 @@ settings = {
'web_search_pages': 3,
'prompt-default': 'QA',
'prompt-notebook': 'QA',
- 'preset': 'min_p',
+ 'preset': 'Qwen3 - Thinking' if Path('user_data/presets/Qwen3 - Thinking.yaml').exists() else '',
'max_new_tokens': 512,
'max_new_tokens_min': 1,
'max_new_tokens_max': 4096,
@@ -237,7 +237,7 @@ settings = {
'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
# Generation parameters - Curve shape
- 'temperature': neutral_samplers['temperature'],
+ 'temperature': 0.6,
'dynatemp_low': neutral_samplers['dynatemp_low'],
'dynatemp_high': neutral_samplers['dynatemp_high'],
'dynatemp_exponent': neutral_samplers['dynatemp_exponent'],
@@ -246,8 +246,8 @@ settings = {
# Generation parameters - Curve cutoff
'min_p': neutral_samplers['min_p'],
- 'top_p': neutral_samplers['top_p'],
- 'top_k': neutral_samplers['top_k'],
+ 'top_p': 0.95,
+ 'top_k': 20,
'typical_p': neutral_samplers['typical_p'],
'xtc_threshold': neutral_samplers['xtc_threshold'],
'xtc_probability': neutral_samplers['xtc_probability'],
From ae150fa24f204fb56c9fe4e2530435bfcf0ec1d7 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:25:46 -0700
Subject: [PATCH 42/82] Remove the null preset
---
user_data/presets/Null preset.yaml | 1 -
1 file changed, 1 deletion(-)
delete mode 100644 user_data/presets/Null preset.yaml
diff --git a/user_data/presets/Null preset.yaml b/user_data/presets/Null preset.yaml
deleted file mode 100644
index 714aa9a3..00000000
--- a/user_data/presets/Null preset.yaml
+++ /dev/null
@@ -1 +0,0 @@
-temperature: 1
From 1cab149c1a020c86f215bafd48b809cbc02a225b Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:26:13 -0700
Subject: [PATCH 43/82] Remove the contrastive search preset
---
user_data/presets/Contrastive Search.yaml | 3 ---
1 file changed, 3 deletions(-)
delete mode 100644 user_data/presets/Contrastive Search.yaml
diff --git a/user_data/presets/Contrastive Search.yaml b/user_data/presets/Contrastive Search.yaml
deleted file mode 100644
index d9a47a9f..00000000
--- a/user_data/presets/Contrastive Search.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-do_sample: false
-top_k: 4
-penalty_alpha: 0.3
From af6bb7513afc6a45cba1716a8cd665d4ee264779 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sun, 8 Jun 2025 09:09:36 -0700
Subject: [PATCH 44/82] Add back the "Save UI defaults" button
It's useful for saving extensions settings.
---
modules/ui_session.py | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/modules/ui_session.py b/modules/ui_session.py
index 4ed740cd..db9ce3b0 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -12,6 +12,7 @@ def create_ui():
with gr.Column():
with gr.Row():
shared.gradio['toggle_dark_mode'] = gr.Button('Toggle 💡')
+ shared.gradio['save_settings'] = gr.Button('Save UI defaults to user_data/settings.yaml', interactive=not mu)
shared.gradio['reset_interface'] = gr.Button("Apply flags/extensions and restart", interactive=not mu)
with gr.Row():
@@ -32,14 +33,28 @@ def create_ui():
if not shared.args.portable:
extension_name.submit(clone_or_pull_repository, extension_name, extension_status, show_progress=False)
+ shared.gradio['save_settings'].click(
+ ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ handle_save_settings, gradio('interface_state', 'preset_menu', 'extensions_menu', 'show_controls', 'theme_state'), gradio('save_contents', 'save_filename', 'save_root', 'file_saver'), show_progress=False)
+
+ shared.gradio['toggle_dark_mode'].click(
+ lambda x: 'dark' if x == 'light' else 'light', gradio('theme_state'), gradio('theme_state')).then(
+ None, None, None, js=f'() => {{{ui.dark_theme_js}; toggleDarkMode()}}')
+
# Reset interface event
shared.gradio['reset_interface'].click(
set_interface_arguments, gradio('extensions_menu', 'bool_menu'), None).then(
None, None, None, js='() => {document.body.innerHTML=\'
Date: Mon, 9 Jun 2025 19:22:01 -0700
Subject: [PATCH 68/82] check .attention.head_count if .attention.head_count_kv
doesn't exist (#7048)
---
modules/models_settings.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/modules/models_settings.py b/modules/models_settings.py
index c914bdea..283a9744 100644
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -329,6 +329,7 @@ def estimate_vram(gguf_file, gpu_layers, ctx_size, cache_type):
# Extract values from metadata
n_layers = None
n_kv_heads = None
+ n_attention_heads = None # Fallback for models without separate KV heads
embedding_dim = None
for key, value in metadata.items():
@@ -336,9 +337,14 @@ def estimate_vram(gguf_file, gpu_layers, ctx_size, cache_type):
n_layers = value
elif key.endswith('.attention.head_count_kv'):
n_kv_heads = max(value) if isinstance(value, list) else value
+ elif key.endswith('.attention.head_count'):
+ n_attention_heads = max(value) if isinstance(value, list) else value
elif key.endswith('.embedding_length'):
embedding_dim = value
+ if n_kv_heads is None:
+ n_kv_heads = n_attention_heads
+
if gpu_layers > n_layers:
gpu_layers = n_layers
From 331d03c33fd69dd50b5c3b9f9ed8fb7d9da56bf0 Mon Sep 17 00:00:00 2001
From: Miriam
Date: Mon, 9 Jun 2025 19:25:39 -0700
Subject: [PATCH 69/82] fix failure when --nowebui called without --api (#7055)
---
server.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/server.py b/server.py
index 5cad1d8a..80cc2f18 100644
--- a/server.py
+++ b/server.py
@@ -318,8 +318,8 @@ if __name__ == "__main__":
if shared.args.nowebui:
# Start the API in standalone mode
- shared.args.extensions = [x for x in shared.args.extensions if x != 'gallery']
- if shared.args.extensions is not None and len(shared.args.extensions) > 0:
+ shared.args.extensions = [x for x in (shared.args.extensions or []) if x != 'gallery']
+ if shared.args.extensions:
extensions_module.load_extensions()
else:
# Launch the web UI
From ec731210200ab53dc26bf56e6616d4533f64c165 Mon Sep 17 00:00:00 2001
From: Mykeehu
Date: Tue, 10 Jun 2025 05:17:05 +0200
Subject: [PATCH 70/82] Fix continue/start reply with when using translation
extensions (#6944)
---------
Co-authored-by: oobabooga
---
modules/chat.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/modules/chat.py b/modules/chat.py
index 25a0607b..88ba33bb 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -760,7 +760,18 @@ def chatbot_wrapper(text, state, regenerate=False, _continue=False, loading_mess
if is_stream:
yield output
- output['visible'][-1][1] = apply_extensions('output', output['visible'][-1][1], state, is_chat=True)
+ if _continue:
+ # Reprocess the entire internal text for extensions (like translation)
+ full_internal = output['internal'][-1][1]
+ if state['mode'] in ['chat', 'chat-instruct']:
+ full_visible = re.sub("(||{{user}})", state['name1'], full_internal)
+ else:
+ full_visible = full_internal
+
+ full_visible = html.escape(full_visible)
+ output['visible'][-1][1] = apply_extensions('output', full_visible, state, is_chat=True)
+ else:
+ output['visible'][-1][1] = apply_extensions('output', output['visible'][-1][1], state, is_chat=True)
# Final sync for version metadata (in case streaming was disabled)
if regenerate:
From df98f4b3312f84995d4dbdbe73681ed78c8e08d2 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 9 Jun 2025 20:28:16 -0700
Subject: [PATCH 71/82] Don't save active extensions through the UI
Prevents command-line activated extensions from becoming permanently active due to autosave.
---
modules/shared.py | 1 -
modules/ui.py | 1 -
server.py | 6 ------
3 files changed, 8 deletions(-)
diff --git a/modules/shared.py b/modules/shared.py
index 59c7dbcd..cc2884c1 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -224,7 +224,6 @@ settings = {
'negative_prompt': '',
'dark_theme': True,
'paste_to_attachment': False,
- 'default_extensions': [],
# Character settings
'character': 'Assistant',
diff --git a/modules/ui.py b/modules/ui.py
index 38693da8..ca1ff528 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -327,7 +327,6 @@ def save_settings(state, preset, extensions_list, show_controls, theme_state):
output['prompt-default'] = state['prompt_menu-default']
output['prompt-notebook'] = state['prompt_menu-notebook']
output['character'] = state['character_menu']
- output['default_extensions'] = extensions_list
output['seed'] = int(output['seed'])
output['show_controls'] = show_controls
output['dark_theme'] = True if theme_state == 'dark' else False
diff --git a/server.py b/server.py
index 80cc2f18..e0e3fbe5 100644
--- a/server.py
+++ b/server.py
@@ -249,13 +249,7 @@ if __name__ == "__main__":
shared.model_config['.*'] = get_fallback_settings()
shared.model_config.move_to_end('.*', last=False) # Move to the beginning
- # Activate the extensions listed on settings.yaml
extensions_module.available_extensions = utils.get_available_extensions()
- for extension in shared.settings['default_extensions']:
- shared.args.extensions = shared.args.extensions or []
- if extension not in shared.args.extensions:
- shared.args.extensions.append(extension)
-
available_models = utils.get_available_models()
# Model defined through --model
From efd9c9707b809754b2e467b53bbeb76688768e6d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 9 Jun 2025 20:57:25 -0700
Subject: [PATCH 72/82] Fix random seeds being saved to settings.yaml
---
modules/text_generation.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/modules/text_generation.py b/modules/text_generation.py
index 0d499d50..55b538b0 100644
--- a/modules/text_generation.py
+++ b/modules/text_generation.py
@@ -479,6 +479,7 @@ def generate_reply_custom(question, original_question, state, stopping_strings=N
For models that do not use the transformers library for sampling
"""
+ state = copy.deepcopy(state)
state['seed'] = set_manual_seed(state['seed'])
t0 = time.time()
reply = ''
From c92eba0b0a0776119ce1912e25dbeeb7a1dfc749 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Mon, 9 Jun 2025 22:03:23 -0700
Subject: [PATCH 73/82] Reorganize the Parameters tab (left: preset parameters,
right: everything else)
---
modules/ui_parameters.py | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index 9b5cb3ab..e2b10554 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -34,6 +34,7 @@ def create_ui():
shared.gradio['dynatemp_exponent'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_exponent'], step=0.01, label='dynatemp_exponent', visible=shared.settings['dynamic_temperature'])
shared.gradio['smoothing_factor'] = gr.Slider(0.0, 10.0, value=shared.settings['smoothing_factor'], step=0.01, label='smoothing_factor', info='Activates Quadratic Sampling.')
shared.gradio['smoothing_curve'] = gr.Slider(1.0, 10.0, value=shared.settings['smoothing_curve'], step=0.01, label='smoothing_curve', info='Adjusts the dropoff curve of Quadratic Sampling.')
+ shared.gradio['dynamic_temperature'] = gr.Checkbox(value=shared.settings['dynamic_temperature'], label='dynamic_temperature')
gr.Markdown('## Curve cutoff')
shared.gradio['min_p'] = gr.Slider(0.0, 1.0, value=shared.settings['min_p'], step=0.01, label='min_p')
@@ -68,15 +69,19 @@ def create_ui():
shared.gradio['mirostat_eta'] = gr.Slider(0, 1, step=0.01, value=shared.settings['mirostat_eta'], label='mirostat_eta')
gr.Markdown('## Other options')
- shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.')
- shared.gradio['prompt_lookup_num_tokens'] = gr.Slider(value=shared.settings['prompt_lookup_num_tokens'], minimum=0, maximum=10, step=1, label='prompt_lookup_num_tokens', info='Activates Prompt Lookup Decoding.')
- shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.')
+ shared.gradio['do_sample'] = gr.Checkbox(value=shared.settings['do_sample'], label='do_sample')
+ shared.gradio['temperature_last'] = gr.Checkbox(value=shared.settings['temperature_last'], label='temperature_last', info='Moves temperature/dynamic temperature/quadratic sampling to the end of the sampler stack, ignoring their positions in "Sampler priority".')
+ shared.gradio['sampler_priority'] = gr.Textbox(value=shared.settings['sampler_priority'], lines=10, label='Sampler priority', info='Parameter names separated by new lines or commas.', elem_classes=['add_scrollbar'])
+ shared.gradio['dry_sequence_breakers'] = gr.Textbox(value=shared.settings['dry_sequence_breakers'], label='dry_sequence_breakers', info='Tokens across which sequence matching is not continued. Specified as a comma-separated list of quoted strings.')
+
with gr.Column():
with gr.Row():
with gr.Column():
- shared.gradio['do_sample'] = gr.Checkbox(value=shared.settings['do_sample'], label='do_sample')
- shared.gradio['dynamic_temperature'] = gr.Checkbox(value=shared.settings['dynamic_temperature'], label='dynamic_temperature')
- shared.gradio['temperature_last'] = gr.Checkbox(value=shared.settings['temperature_last'], label='temperature_last', info='Moves temperature/dynamic temperature/quadratic sampling to the end of the sampler stack, ignoring their positions in "Sampler priority".')
+ with gr.Blocks():
+ shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.')
+ shared.gradio['prompt_lookup_num_tokens'] = gr.Slider(value=shared.settings['prompt_lookup_num_tokens'], minimum=0, maximum=10, step=1, label='prompt_lookup_num_tokens', info='Activates Prompt Lookup Decoding.')
+ shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.')
+
shared.gradio['auto_max_new_tokens'] = gr.Checkbox(value=shared.settings['auto_max_new_tokens'], label='auto_max_new_tokens', info='Expand max_new_tokens to the available context length.')
shared.gradio['ban_eos_token'] = gr.Checkbox(value=shared.settings['ban_eos_token'], label='Ban the eos_token', info='Forces the model to never end the generation prematurely.')
shared.gradio['add_bos_token'] = gr.Checkbox(value=shared.settings['add_bos_token'], label='Add the bos_token to the beginning of prompts', info='Disabling this can make the replies more creative.')
@@ -89,11 +94,9 @@ def create_ui():
shared.gradio['truncation_length'] = gr.Number(precision=0, step=256, value=get_truncation_length(), label='Truncate the prompt up to this length', info='The leftmost tokens are removed if the prompt exceeds this length.')
shared.gradio['seed'] = gr.Number(value=shared.settings['seed'], label='Seed (-1 for random)')
- shared.gradio['sampler_priority'] = gr.Textbox(value=shared.settings['sampler_priority'], lines=12, label='Sampler priority', info='Parameter names separated by new lines or commas.', elem_classes=['add_scrollbar'])
shared.gradio['custom_stopping_strings'] = gr.Textbox(lines=2, value=shared.settings["custom_stopping_strings"] or None, label='Custom stopping strings', info='Written between "" and separated by commas.', placeholder='"\\n", "\\nYou:"')
shared.gradio['custom_token_bans'] = gr.Textbox(value=shared.settings['custom_token_bans'] or None, label='Token bans', info='Token IDs to ban, separated by commas. The IDs can be found in the Default or Notebook tab.')
shared.gradio['negative_prompt'] = gr.Textbox(value=shared.settings['negative_prompt'], label='Negative prompt', info='For CFG. Only used when guidance_scale is different than 1.', lines=3, elem_classes=['add_scrollbar'])
- shared.gradio['dry_sequence_breakers'] = gr.Textbox(value=shared.settings['dry_sequence_breakers'], label='dry_sequence_breakers', info='Tokens across which sequence matching is not continued. Specified as a comma-separated list of quoted strings.')
with gr.Row() as shared.gradio['grammar_file_row']:
shared.gradio['grammar_file'] = gr.Dropdown(value='None', choices=utils.get_available_grammars(), label='Load grammar from file (.gbnf)', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['grammar_file'], lambda: None, lambda: {'choices': utils.get_available_grammars()}, 'refresh-button', interactive=not mu)
From 2dabdbc7da85f91ea9ddd74dfb49c2d161a6ef4d Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 05:25:23 -0700
Subject: [PATCH 74/82] Update llama.cpp
---
requirements/full/requirements.txt | 4 ++--
requirements/full/requirements_amd.txt | 4 ++--
requirements/full/requirements_amd_noavx2.txt | 4 ++--
requirements/full/requirements_apple_intel.txt | 4 ++--
requirements/full/requirements_apple_silicon.txt | 6 +++---
requirements/full/requirements_cpu_only.txt | 4 ++--
requirements/full/requirements_cpu_only_noavx2.txt | 4 ++--
requirements/full/requirements_cuda128.txt | 4 ++--
requirements/full/requirements_cuda128_noavx2.txt | 4 ++--
requirements/full/requirements_noavx2.txt | 4 ++--
requirements/portable/requirements.txt | 4 ++--
requirements/portable/requirements_apple_intel.txt | 4 ++--
requirements/portable/requirements_apple_silicon.txt | 6 +++---
requirements/portable/requirements_cpu_only.txt | 4 ++--
requirements/portable/requirements_cpu_only_noavx2.txt | 4 ++--
requirements/portable/requirements_noavx2.txt | 4 ++--
requirements/portable/requirements_vulkan.txt | 4 ++--
requirements/portable/requirements_vulkan_noavx2.txt | 4 ++--
18 files changed, 38 insertions(+), 38 deletions(-)
diff --git a/requirements/full/requirements.txt b/requirements/full/requirements.txt
index b751482a..a71e5240 100644
--- a/requirements/full/requirements.txt
+++ b/requirements/full/requirements.txt
@@ -34,8 +34,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_amd.txt b/requirements/full/requirements_amd.txt
index 11bacf97..db1ead1a 100644
--- a/requirements/full/requirements_amd.txt
+++ b/requirements/full/requirements_amd.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements/full/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt
index a64a93f0..a08aa392 100644
--- a/requirements/full/requirements_amd_noavx2.txt
+++ b/requirements/full/requirements_amd_noavx2.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# AMD wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+rocm6.2.4.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64"
diff --git a/requirements/full/requirements_apple_intel.txt b/requirements/full/requirements_apple_intel.txt
index 62747ac4..fa217c3e 100644
--- a/requirements/full/requirements_apple_intel.txt
+++ b/requirements/full/requirements_apple_intel.txt
@@ -33,7 +33,7 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3-py3-none-any.whl
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl
diff --git a/requirements/full/requirements_apple_silicon.txt b/requirements/full/requirements_apple_silicon.txt
index bc82f07a..52581f1a 100644
--- a/requirements/full/requirements_apple_silicon.txt
+++ b/requirements/full/requirements_apple_silicon.txt
@@ -33,8 +33,8 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3-py3-none-any.whl
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1-py3-none-any.whl
diff --git a/requirements/full/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt
index f880f40a..b72f22aa 100644
--- a/requirements/full/requirements_cpu_only.txt
+++ b/requirements/full/requirements_cpu_only.txt
@@ -33,5 +33,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt
index 6d8875cb..e8de6057 100644
--- a/requirements/full/requirements_cpu_only_noavx2.txt
+++ b/requirements/full/requirements_cpu_only_noavx2.txt
@@ -33,5 +33,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_cuda128.txt b/requirements/full/requirements_cuda128.txt
index b2bcf91c..801009fb 100644
--- a/requirements/full/requirements_cuda128.txt
+++ b/requirements/full/requirements_cuda128.txt
@@ -35,8 +35,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_cuda128_noavx2.txt b/requirements/full/requirements_cuda128_noavx2.txt
index 54496cd7..8b77e144 100644
--- a/requirements/full/requirements_cuda128_noavx2.txt
+++ b/requirements/full/requirements_cuda128_noavx2.txt
@@ -35,8 +35,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/turboderp-org/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu128.torch2.7.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu128.torch2.7.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/full/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt
index eabcdbd0..5e81ce1f 100644
--- a/requirements/full/requirements_noavx2.txt
+++ b/requirements/full/requirements_noavx2.txt
@@ -34,8 +34,8 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/oobabooga/exllamav3/releases/download/v0.0.3/exllamav3-0.0.3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp-org/exllamav2/releases/download/v0.3.1/exllamav2-0.3.1+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt
index 5e5d4ba5..4ddcf43f 100644
--- a/requirements/portable/requirements.txt
+++ b/requirements/portable/requirements.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt
index 4909f5a2..38a21618 100644
--- a/requirements/portable/requirements_apple_intel.txt
+++ b/requirements/portable/requirements_apple_intel.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt
index e54b2593..0b70c800 100644
--- a/requirements/portable/requirements_apple_silicon.txt
+++ b/requirements/portable/requirements_apple_silicon.txt
@@ -19,6 +19,6 @@ sse-starlette==1.6.5
tiktoken
# Mac wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0-py3-none-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0"
diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt
index 74c0c5a7..510a20f4 100644
--- a/requirements/portable/requirements_cpu_only.txt
+++ b/requirements/portable/requirements_cpu_only.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx2-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx2-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt
index 264bc378..e6d9f0c5 100644
--- a/requirements/portable/requirements_cpu_only_noavx2.txt
+++ b/requirements/portable/requirements_cpu_only_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# llama.cpp (CPU only, no AVX2)
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cpuavx-py3-none-win_amd64.whl; platform_system == "Windows"
diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt
index fcb8f05e..48f92e0a 100644
--- a/requirements/portable/requirements_noavx2.txt
+++ b/requirements/portable/requirements_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124avx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+cu124avx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan.txt b/requirements/portable/requirements_vulkan.txt
index 395f225f..9f93424f 100644
--- a/requirements/portable/requirements_vulkan.txt
+++ b/requirements/portable/requirements_vulkan.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkan-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkan-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
diff --git a/requirements/portable/requirements_vulkan_noavx2.txt b/requirements/portable/requirements_vulkan_noavx2.txt
index 0d41f541..9070b9a6 100644
--- a/requirements/portable/requirements_vulkan_noavx2.txt
+++ b/requirements/portable/requirements_vulkan_noavx2.txt
@@ -19,5 +19,5 @@ sse-starlette==1.6.5
tiktoken
# CUDA wheels
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
-https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.16.0/llama_cpp_binaries-0.16.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkanavx-py3-none-win_amd64.whl; platform_system == "Windows"
+https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.18.0/llama_cpp_binaries-0.18.0+vulkanavx-py3-none-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64"
From 889153952f5545fb2b94b7a853aa24ae754ef79f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 09:02:52 -0700
Subject: [PATCH 75/82] Lint
---
modules/ui_model_menu.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/modules/ui_model_menu.py b/modules/ui_model_menu.py
index 2a7d3d9d..9e982f0e 100644
--- a/modules/ui_model_menu.py
+++ b/modules/ui_model_menu.py
@@ -260,6 +260,7 @@ def download_model_wrapper(repo_id, specific_file, progress=gr.Progress(), retur
output = "```\n"
for link in links:
output += f"{Path(link).name}" + "\n"
+
output += "```"
yield output
return
From 18bd78f1f038b2b69178d464184b937f7d2b15d6 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 14:03:25 -0700
Subject: [PATCH 76/82] Make the llama.cpp prompt processing messages shorter
---
modules/llama_cpp_server.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index f0a72de8..a79e24e4 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -422,9 +422,17 @@ def filter_stderr_with_progress(process_stderr):
if match:
progress = float(match.group(1))
+
+ # Extract just the part from "prompt processing" onwards
+ prompt_processing_idx = line.find('prompt processing')
+ if prompt_processing_idx != -1:
+ display_line = line[prompt_processing_idx:]
+ else:
+ display_line = line # fallback to full line
+
# choose carriage return for in-progress or newline at completion
end_char = '\r' if progress < 1.0 else '\n'
- print(line, end=end_char, file=sys.stderr, flush=True)
+ print(display_line, end=end_char, file=sys.stderr, flush=True)
last_was_progress = (progress < 1.0)
# skip noise lines
From 3f9eb3aad1d1a734791dd41e4817570d81b485eb Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 14:22:37 -0700
Subject: [PATCH 77/82] Fix the preset dropdown when the default preset file is
not present
---
modules/shared.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/shared.py b/modules/shared.py
index cc2884c1..b8ab2426 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -204,7 +204,7 @@ settings = {
'web_search_pages': 3,
'prompt-default': 'QA',
'prompt-notebook': 'QA',
- 'preset': 'Qwen3 - Thinking' if Path('user_data/presets/Qwen3 - Thinking.yaml').exists() else '',
+ 'preset': 'Qwen3 - Thinking' if Path('user_data/presets/Qwen3 - Thinking.yaml').exists() else None,
'max_new_tokens': 512,
'max_new_tokens_min': 1,
'max_new_tokens_max': 4096,
From 1c1fd3be46f66b2f3110aed52a155817ca3392b7 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 14:29:28 -0700
Subject: [PATCH 78/82] Remove some log messages
---
modules/chat.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/modules/chat.py b/modules/chat.py
index 88ba33bb..dfc301df 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -1271,8 +1271,6 @@ def clear_character_for_ui(state):
if cache_path.exists():
cache_path.unlink()
- logger.info("Cleared character fields and picture cache")
-
return state, state['name2'], state['context'], state['greeting'], None
@@ -1779,7 +1777,6 @@ def handle_character_picture_change(picture):
picture.save(Path(f'{cache_folder}/pfp_character.png'), format='PNG')
thumb = make_thumbnail(picture)
thumb.save(Path(f'{cache_folder}/pfp_character_thumb.png'), format='PNG')
- logger.info("Updated character picture cache")
else:
# Remove cache files when picture is cleared
for cache_file in ['pfp_character.png', 'pfp_character_thumb.png']:
@@ -1787,8 +1784,6 @@ def handle_character_picture_change(picture):
if cache_path.exists():
cache_path.unlink()
- logger.info("Cleared character picture cache")
-
def handle_mode_change(state):
history = load_latest_history(state)
From 75da90190f33317dd9731e14d844dd7238a2c870 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 17:34:54 -0700
Subject: [PATCH 79/82] Fix character dropdown sometimes disappearing in the
Parameters tab
---
modules/ui_chat.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 428b64c9..3b841b8b 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -298,7 +298,7 @@ def create_event_handlers():
shared.gradio['mode'].change(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
chat.handle_mode_change, gradio('interface_state'), gradio('history', 'display', 'chat_style', 'chat-instruct_command', 'unique_id'), show_progress=False).then(
- None, gradio('mode'), None, js="(mode) => {mode === 'instruct' ? document.getElementById('character-menu').parentNode.parentNode.style.display = 'none' : document.getElementById('character-menu').parentNode.parentNode.style.display = ''}")
+ None, gradio('mode'), None, js="(mode) => {const characterContainer = document.getElementById('character-menu').parentNode.parentNode; const isInChatTab = document.querySelector('#chat-controls').contains(characterContainer); if (isInChatTab) { characterContainer.style.display = mode === 'instruct' ? 'none' : ''; }}")
shared.gradio['chat_style'].change(chat.redraw_html, gradio(reload_arr), gradio('display'), show_progress=False)
From 4cf39120fca88fc78aeaaa55946ab74e5db1a512 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 18:03:00 -0700
Subject: [PATCH 80/82] Fix chat area sometimes not scrolling up to edit
message
---
js/global_scope_js.js | 9 +++++++++
js/main.js | 8 ++++----
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/js/global_scope_js.js b/js/global_scope_js.js
index 801f1574..205d9375 100644
--- a/js/global_scope_js.js
+++ b/js/global_scope_js.js
@@ -95,12 +95,21 @@ function startEditing(messageElement, messageBody, isUserMessage) {
editingInterface.textarea.focus();
editingInterface.textarea.setSelectionRange(rawText.length, rawText.length);
+ // Temporarily mark as scrolled to prevent auto-scroll
+ const wasScrolled = window.isScrolled;
+ window.isScrolled = true;
+
// Scroll the textarea into view
editingInterface.textarea.scrollIntoView({
behavior: "smooth",
block: "center"
});
+ // Restore the original scroll state after animation
+ setTimeout(() => {
+ window.isScrolled = wasScrolled;
+ }, 500);
+
// Setup event handlers
setupEditingHandlers(editingInterface.textarea, messageElement, originalHTML, messageBody, isUserMessage);
}
diff --git a/js/main.js b/js/main.js
index 1953a6be..e970884d 100644
--- a/js/main.js
+++ b/js/main.js
@@ -145,7 +145,7 @@ typingSibling.insertBefore(typing, typingSibling.childNodes[2]);
const targetElement = document.getElementById("chat").parentNode.parentNode.parentNode;
targetElement.classList.add("pretty_scrollbar");
targetElement.classList.add("chat-parent");
-let isScrolled = false;
+window.isScrolled = false;
let scrollTimeout;
targetElement.addEventListener("scroll", function() {
@@ -154,9 +154,9 @@ targetElement.addEventListener("scroll", function() {
let diff = targetElement.scrollHeight - targetElement.clientHeight;
if(Math.abs(targetElement.scrollTop - diff) <= 10 || diff == 0) {
- isScrolled = false;
+ window.isScrolled = false;
} else {
- isScrolled = true;
+ window.isScrolled = true;
}
// Clear previous timeout and set new one
@@ -182,7 +182,7 @@ const observer = new MutationObserver(function(mutations) {
doSyntaxHighlighting();
- if (!isScrolled && targetElement.scrollTop !== targetElement.scrollHeight) {
+ if (!window.isScrolled && targetElement.scrollTop !== targetElement.scrollHeight) {
targetElement.scrollTop = targetElement.scrollHeight;
}
From bc921c66e5ef2f2a14286702e564c1b22dbba5a7 Mon Sep 17 00:00:00 2001
From: LawnMauer
Date: Wed, 11 Jun 2025 03:16:50 +0200
Subject: [PATCH 81/82] Load js and css sources in UTF-8 (#7059)
---
modules/html_generator.py | 7 ++++---
modules/ui.py | 22 +++++++++++-----------
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/modules/html_generator.py b/modules/html_generator.py
index eac7d91a..af64894e 100644
--- a/modules/html_generator.py
+++ b/modules/html_generator.py
@@ -39,15 +39,16 @@ def minify_css(css: str) -> str:
return css
-with open(Path(__file__).resolve().parent / '../css/html_readable_style.css', 'r') as f:
+with open(Path(__file__).resolve().parent / '../css/html_readable_style.css', 'r', encoding='utf-8') as f:
readable_css = f.read()
-with open(Path(__file__).resolve().parent / '../css/html_instruct_style.css', 'r') as f:
+with open(Path(__file__).resolve().parent / '../css/html_instruct_style.css', 'r', encoding='utf-8') as f:
instruct_css = f.read()
# Custom chat styles
chat_styles = {}
for k in get_available_chat_styles():
- chat_styles[k] = open(Path(f'css/chat_style-{k}.css'), 'r').read()
+ with open(Path(f'css/chat_style-{k}.css'), 'r', encoding='utf-8') as f:
+ chat_styles[k] = f.read()
# Handle styles that derive from other styles
for k in chat_styles:
diff --git a/modules/ui.py b/modules/ui.py
index ca1ff528..59da5118 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -19,27 +19,27 @@ _last_extensions = None
_last_show_controls = None
_last_theme_state = None
-with open(Path(__file__).resolve().parent / '../css/NotoSans/stylesheet.css', 'r') as f:
+with open(Path(__file__).resolve().parent / '../css/NotoSans/stylesheet.css', 'r', encoding='utf-8') as f:
css = f.read()
-with open(Path(__file__).resolve().parent / '../css/main.css', 'r') as f:
+with open(Path(__file__).resolve().parent / '../css/main.css', 'r', encoding='utf-8') as f:
css += f.read()
-with open(Path(__file__).resolve().parent / '../css/katex/katex.min.css', 'r') as f:
+with open(Path(__file__).resolve().parent / '../css/katex/katex.min.css', 'r', encoding='utf-8') as f:
css += f.read()
-with open(Path(__file__).resolve().parent / '../css/highlightjs/highlightjs-copy.min.css', 'r') as f:
+with open(Path(__file__).resolve().parent / '../css/highlightjs/highlightjs-copy.min.css', 'r', encoding='utf-8') as f:
css += f.read()
-with open(Path(__file__).resolve().parent / '../js/main.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/main.js', 'r', encoding='utf-8') as f:
js = f.read()
-with open(Path(__file__).resolve().parent / '../js/global_scope_js.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/global_scope_js.js', 'r', encoding='utf-8') as f:
global_scope_js = f.read()
-with open(Path(__file__).resolve().parent / '../js/save_files.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/save_files.js', 'r', encoding='utf-8') as f:
save_files_js = f.read()
-with open(Path(__file__).resolve().parent / '../js/switch_tabs.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/switch_tabs.js', 'r', encoding='utf-8') as f:
switch_tabs_js = f.read()
-with open(Path(__file__).resolve().parent / '../js/show_controls.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/show_controls.js', 'r', encoding='utf-8') as f:
show_controls_js = f.read()
-with open(Path(__file__).resolve().parent / '../js/update_big_picture.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/update_big_picture.js', 'r', encoding='utf-8') as f:
update_big_picture_js = f.read()
-with open(Path(__file__).resolve().parent / '../js/dark_theme.js', 'r') as f:
+with open(Path(__file__).resolve().parent / '../js/dark_theme.js', 'r', encoding='utf-8') as f:
dark_theme_js = f.read()
refresh_symbol = '🔄'
From 552cb09f09de0d3a9442334580fd4a66f95a1ce3 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Tue, 10 Jun 2025 18:45:42 -0700
Subject: [PATCH 82/82] Do not bump Transformers to 4.52 on CUDA 12.8
Performance is slow, and the older version works fine with torch 2.7.
---
requirements/full/requirements_cuda128.txt | 3 +--
requirements/full/requirements_cuda128_noavx2.txt | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/requirements/full/requirements_cuda128.txt b/requirements/full/requirements_cuda128.txt
index 801009fb..7851041f 100644
--- a/requirements/full/requirements_cuda128.txt
+++ b/requirements/full/requirements_cuda128.txt
@@ -24,8 +24,7 @@ safetensors==0.5.*
scipy
sentencepiece
tensorboard
-transformers==4.52.*
-triton-windows; platform_system == "Windows"
+transformers==4.50.*
tqdm
wandb
diff --git a/requirements/full/requirements_cuda128_noavx2.txt b/requirements/full/requirements_cuda128_noavx2.txt
index 8b77e144..c8015166 100644
--- a/requirements/full/requirements_cuda128_noavx2.txt
+++ b/requirements/full/requirements_cuda128_noavx2.txt
@@ -24,8 +24,7 @@ safetensors==0.5.*
scipy
sentencepiece
tensorboard
-transformers==4.52.*
-triton-windows; platform_system == "Windows"
+transformers==4.50.*
tqdm
wandb
'
- if Path("user_data/cache/pfp_character_thumb.png").exists() else ''
- )
+ img_bot = get_character_image_with_cache_buster()
def create_message(role, content, raw_content):
"""Inner function for CAI-style messages."""
diff --git a/modules/presets.py b/modules/presets.py
index cf706605..3eb1f5fc 100644
--- a/modules/presets.py
+++ b/modules/presets.py
@@ -1,6 +1,5 @@
import functools
import pprint
-import random
from pathlib import Path
import yaml
@@ -93,68 +92,17 @@ def load_preset_for_ui(name, state):
return state, *[generate_params[k] for k in presets_params()]
-def random_preset(state):
- params_and_values = {
- 'remove_tail_tokens': {
- 'top_p': [0.5, 0.8, 0.9, 0.95, 0.99],
- 'min_p': [0.5, 0.2, 0.1, 0.05, 0.01],
- 'top_k': [3, 5, 10, 20, 30, 40],
- 'typical_p': [0.2, 0.575, 0.95],
- 'tfs': [0.5, 0.8, 0.9, 0.95, 0.99],
- 'top_a': [0.5, 0.2, 0.1, 0.05, 0.01],
- 'epsilon_cutoff': [1, 3, 5, 7, 9],
- 'eta_cutoff': [3, 6, 9, 12, 15, 18],
- },
- 'flatten_distribution': {
- 'temperature': [0.1, 0.5, 0.7, 0.8, 1, 1.2, 1.5, 2.0, 5.0],
- 'dynamic_temperature': [
- [0.1, 1],
- [0.1, 1.5],
- [0.1, 2],
- [0.1, 5],
- [0.5, 1],
- [0.5, 1.5],
- [0.5, 2],
- [0.5, 5],
- [0.8, 1],
- [0.8, 1.5],
- [0.8, 2],
- [0.8, 5],
- [1, 1.5],
- [1, 2],
- [1, 5]
- ],
- 'smoothing_factor': [0.2, 0.3, 0.6, 1.2],
- },
- 'repetition': {
- 'repetition_penalty': [1, 1.05, 1.1, 1.15, 1.20, 1.25],
- 'presence_penalty': [0, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2.0],
- 'frequency_penalty': [0, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2.0],
- },
- 'other': {
- 'temperature_last': [True, False],
- }
- }
-
- generate_params = default_preset()
- for cat in params_and_values:
- choices = list(params_and_values[cat].keys())
- if shared.args.loader is not None:
- choices = [x for x in choices if loader_contains(x)]
-
- if len(choices) > 0:
- choice = random.choice(choices)
- value = random.choice(params_and_values[cat][choice])
- if choice == 'dynamic_temperature':
- generate_params['dynamic_temperature'] = True
- generate_params['dynatemp_low'] = value[0]
- generate_params['dynatemp_high'] = value[1]
- else:
- generate_params[choice] = value
-
+def reset_preset_for_ui(name, state):
+ """Reset current preset to its saved values from file"""
+ generate_params = load_preset(name, verbose=True)
+ state.update(generate_params)
+ return state, *[generate_params[k] for k in presets_params()]
+
+
+def neutralize_samplers_for_ui(state):
+ """Set all samplers to their default/neutral values"""
+ generate_params = default_preset()
state.update(generate_params)
- logger.info("GENERATED_PRESET=")
- pprint.PrettyPrinter(indent=4, width=1, sort_dicts=False).pprint(remove_defaults(state))
return state, *[generate_params[k] for k in presets_params()]
diff --git a/modules/shared.py b/modules/shared.py
index 2e500779..f4f8e180 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -39,10 +39,6 @@ settings = {
'web_search_pages': 3,
'prompt-default': 'QA',
'prompt-notebook': 'QA',
- 'character': 'Assistant',
- 'name1': 'You',
- 'user_bio': '',
- 'custom_system_message': '',
'preset': 'min_p',
'max_new_tokens': 512,
'max_new_tokens_min': 1,
@@ -63,8 +59,64 @@ settings = {
'negative_prompt': '',
'dark_theme': True,
'default_extensions': [],
+
+ # Character settings
+ 'character': 'Assistant',
+ 'name1': 'You',
+ 'name2': 'AI',
+ 'user_bio': '',
+ 'context': 'The following is a conversation with an AI Large Language Model. The AI has been trained to answer questions, provide recommendations, and help with decision making. The AI follows user requests. The AI thinks outside the box.',
+ 'greeting': 'How can I help you today?',
+ 'custom_system_message': '',
'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}",
'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
+
+ # Generation parameters - Curve shape
+ 'temperature': 1.0,
+ 'dynatemp_low': 1.0,
+ 'dynatemp_high': 1.0,
+ 'dynatemp_exponent': 1.0,
+ 'smoothing_factor': 0.0,
+ 'smoothing_curve': 1.0,
+
+ # Generation parameters - Curve cutoff
+ 'min_p': 0.0,
+ 'top_p': 1.0,
+ 'top_k': 0,
+ 'typical_p': 1.0,
+ 'xtc_threshold': 0.1,
+ 'xtc_probability': 0.0,
+ 'epsilon_cutoff': 0.0,
+ 'eta_cutoff': 0.0,
+ 'tfs': 1.0,
+ 'top_a': 0.0,
+ 'top_n_sigma': 0.0,
+
+ # Generation parameters - Repetition suppression
+ 'dry_multiplier': 0.0,
+ 'dry_allowed_length': 2,
+ 'dry_base': 1.75,
+ 'repetition_penalty': 1.0,
+ 'frequency_penalty': 0.0,
+ 'presence_penalty': 0.0,
+ 'encoder_repetition_penalty': 1.0,
+ 'no_repeat_ngram_size': 0,
+ 'repetition_penalty_range': 1024,
+
+ # Generation parameters - Alternative sampling methods
+ 'penalty_alpha': 0.0,
+ 'guidance_scale': 1.0,
+ 'mirostat_mode': 0,
+ 'mirostat_tau': 5.0,
+ 'mirostat_eta': 0.1,
+
+ # Generation parameters - Other options
+ 'do_sample': True,
+ 'dynamic_temperature': False,
+ 'temperature_last': False,
+ 'sampler_priority': 'repetition_penalty\npresence_penalty\nfrequency_penalty\ndry\ntop_n_sigma\ntemperature\ndynamic_temperature\nquadratic_sampling\ntop_k\ntop_p\ntypical_p\nepsilon_cutoff\neta_cutoff\ntfs\ntop_a\nmin_p\nmirostat\nxtc\nencoder_repetition_penalty\nno_repeat_ngram',
+ 'dry_sequence_breakers': '"\\n", ":", "\\"", "*"',
+ 'grammar_string': '',
}
default_settings = copy.deepcopy(settings)
@@ -75,7 +127,6 @@ parser = argparse.ArgumentParser(description="Text generation web UI", conflict_
# Basic settings
group = parser.add_argument_group('Basic settings')
group.add_argument('--multi-user', action='store_true', help='Multi-user mode. Chat histories are not saved or automatically loaded. Warning: this is likely not safe for sharing publicly.')
-group.add_argument('--character', type=str, help='The name of the character to load in chat mode by default.')
group.add_argument('--model', type=str, help='Name of the model to load by default.')
group.add_argument('--lora', type=str, nargs='+', help='The list of LoRAs to load. If you want to load more than one LoRA, write the names separated by spaces.')
group.add_argument('--model-dir', type=str, default='user_data/models', help='Path to directory with all the models.')
diff --git a/modules/ui.py b/modules/ui.py
index db3adf0f..8ec4b165 100644
--- a/modules/ui.py
+++ b/modules/ui.py
@@ -310,7 +310,7 @@ def apply_interface_values(state, use_persistent=False):
def save_settings(state, preset, extensions_list, show_controls, theme_state):
output = copy.deepcopy(shared.settings)
- exclude = ['name2', 'greeting', 'context', 'truncation_length', 'instruction_template_str']
+ exclude = []
for k in state:
if k in shared.settings and k not in exclude:
output[k] = state[k]
@@ -323,6 +323,7 @@ def save_settings(state, preset, extensions_list, show_controls, theme_state):
output['seed'] = int(output['seed'])
output['show_controls'] = show_controls
output['dark_theme'] = True if theme_state == 'dark' else False
+ output.pop('instruction_template_str')
# Save extension values in the UI
for extension_name in extensions_list:
@@ -364,7 +365,7 @@ def store_current_state_and_debounce(interface_state, preset, extensions, show_c
if _auto_save_timer is not None:
_auto_save_timer.cancel()
- _auto_save_timer = threading.Timer(2.0, _perform_debounced_save)
+ _auto_save_timer = threading.Timer(1.0, _perform_debounced_save)
_auto_save_timer.start()
@@ -401,15 +402,52 @@ def setup_auto_save():
'chat-instruct_command',
'character_menu',
'name1',
+ 'name2',
+ 'context',
+ 'greeting',
'user_bio',
'custom_system_message',
'chat_template_str',
- # Parameters tab (ui_parameters.py)
+ # Parameters tab (ui_parameters.py) - Generation parameters
'preset_menu',
+ 'temperature',
+ 'dynatemp_low',
+ 'dynatemp_high',
+ 'dynatemp_exponent',
+ 'smoothing_factor',
+ 'smoothing_curve',
+ 'min_p',
+ 'top_p',
+ 'top_k',
+ 'typical_p',
+ 'xtc_threshold',
+ 'xtc_probability',
+ 'epsilon_cutoff',
+ 'eta_cutoff',
+ 'tfs',
+ 'top_a',
+ 'top_n_sigma',
+ 'dry_multiplier',
+ 'dry_allowed_length',
+ 'dry_base',
+ 'repetition_penalty',
+ 'frequency_penalty',
+ 'presence_penalty',
+ 'encoder_repetition_penalty',
+ 'no_repeat_ngram_size',
+ 'repetition_penalty_range',
+ 'penalty_alpha',
+ 'guidance_scale',
+ 'mirostat_mode',
+ 'mirostat_tau',
+ 'mirostat_eta',
'max_new_tokens',
'prompt_lookup_num_tokens',
'max_tokens_second',
+ 'do_sample',
+ 'dynamic_temperature',
+ 'temperature_last',
'auto_max_new_tokens',
'ban_eos_token',
'add_bos_token',
@@ -417,10 +455,14 @@ def setup_auto_save():
'skip_special_tokens',
'stream',
'static_cache',
+ 'truncation_length',
'seed',
+ 'sampler_priority',
'custom_stopping_strings',
'custom_token_bans',
'negative_prompt',
+ 'dry_sequence_breakers',
+ 'grammar_string',
# Default tab (ui_default.py)
'prompt_menu-default',
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 0d5a2c18..712843d3 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -86,7 +86,7 @@ def create_ui():
shared.gradio['web_search_pages'] = gr.Number(value=shared.settings.get('web_search_pages', 3), precision=0, label='Number of pages to download', minimum=1, maximum=10)
with gr.Row():
- shared.gradio['mode'] = gr.Radio(choices=['instruct', 'chat-instruct', 'chat'], value=shared.settings['mode'] if shared.settings['mode'] in ['chat', 'chat-instruct'] else None, label='Mode', info='Defines how the chat prompt is generated. In instruct and chat-instruct modes, the instruction template Parameters > Instruction template is used.', elem_id='chat-mode')
+ shared.gradio['mode'] = gr.Radio(choices=['instruct', 'chat-instruct', 'chat'], value=None, label='Mode', info='Defines how the chat prompt is generated. In instruct and chat-instruct modes, the instruction template Parameters > Instruction template is used.', elem_id='chat-mode')
with gr.Row():
shared.gradio['chat_style'] = gr.Dropdown(choices=utils.get_available_chat_styles(), label='Chat style', value=shared.settings['chat_style'], visible=shared.settings['mode'] != 'instruct')
@@ -118,14 +118,16 @@ def create_chat_settings_ui():
with gr.Column(scale=8):
with gr.Tab("Character"):
with gr.Row():
- shared.gradio['character_menu'] = gr.Dropdown(value=None, choices=utils.get_available_characters(), label='Character', elem_id='character-menu', info='Used in chat and chat-instruct modes.', elem_classes='slim-dropdown')
+ shared.gradio['character_menu'] = gr.Dropdown(value=shared.settings['character'], choices=utils.get_available_characters(), label='Character', elem_id='character-menu', info='Used in chat and chat-instruct modes.', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['character_menu'], lambda: None, lambda: {'choices': utils.get_available_characters()}, 'refresh-button', interactive=not mu)
shared.gradio['save_character'] = gr.Button('💾', elem_classes='refresh-button', elem_id="save-character", interactive=not mu)
shared.gradio['delete_character'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
+ shared.gradio['reset_character'] = gr.Button('↺', elem_classes='refresh-button', interactive=True)
+ shared.gradio['clear_character'] = gr.Button('⚪', elem_classes='refresh-button', interactive=True)
- shared.gradio['name2'] = gr.Textbox(value='', lines=1, label='Character\'s name')
- shared.gradio['context'] = gr.Textbox(value='', lines=10, label='Context', elem_classes=['add_scrollbar'])
- shared.gradio['greeting'] = gr.Textbox(value='', lines=5, label='Greeting', elem_classes=['add_scrollbar'])
+ shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name')
+ shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=10, label='Context', elem_classes=['add_scrollbar'])
+ shared.gradio['greeting'] = gr.Textbox(value=shared.settings['greeting'], lines=5, label='Greeting', elem_classes=['add_scrollbar'])
with gr.Tab("User"):
shared.gradio['name1'] = gr.Textbox(value=shared.settings['name1'], lines=1, label='Name')
@@ -178,7 +180,7 @@ def create_chat_settings_ui():
with gr.Row():
with gr.Column():
shared.gradio['custom_system_message'] = gr.Textbox(value=shared.settings['custom_system_message'], lines=2, label='Custom system message', info='If not empty, will be used instead of the default one.', elem_classes=['add_scrollbar'])
- shared.gradio['instruction_template_str'] = gr.Textbox(value='', label='Instruction template', lines=24, info='This gets autodetected; you usually don\'t need to change it. Used in instruct and chat-instruct modes.', elem_classes=['add_scrollbar', 'monospace'])
+ shared.gradio['instruction_template_str'] = gr.Textbox(value=shared.settings['instruction_template_str'], label='Instruction template', lines=24, info='This gets autodetected; you usually don\'t need to change it. Used in instruct and chat-instruct modes.', elem_classes=['add_scrollbar', 'monospace'])
with gr.Row():
shared.gradio['send_instruction_to_default'] = gr.Button('Send to default', elem_classes=['small-button'])
shared.gradio['send_instruction_to_notebook'] = gr.Button('Send to notebook', elem_classes=['small-button'])
@@ -294,6 +296,8 @@ def create_event_handlers():
chat.handle_character_menu_change, gradio('interface_state'), gradio('history', 'display', 'name1', 'name2', 'character_picture', 'greeting', 'context', 'unique_id'), show_progress=False).then(
None, None, None, js=f'() => {{{ui.update_big_picture_js}; updateBigPicture()}}')
+ shared.gradio['character_picture'].change(chat.handle_character_picture_change, gradio('character_picture'), None, show_progress=False)
+
shared.gradio['mode'].change(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
chat.handle_mode_change, gradio('interface_state'), gradio('history', 'display', 'chat_style', 'chat-instruct_command', 'unique_id'), show_progress=False).then(
@@ -317,6 +321,10 @@ def create_event_handlers():
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
chat.handle_save_template_click, gradio('instruction_template_str'), gradio('save_filename', 'save_root', 'save_contents', 'file_saver'), show_progress=False)
+ shared.gradio['reset_character'].click(
+ ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ chat.reset_character_for_ui, gradio('interface_state'), gradio('interface_state', 'name2', 'context', 'greeting', 'character_picture'), show_progress=False)
+
shared.gradio['delete_template'].click(chat.handle_delete_template_click, gradio('instruction_template'), gradio('delete_filename', 'delete_root', 'file_deleter'), show_progress=False)
shared.gradio['save_chat_history'].click(
lambda x: json.dumps(x, indent=4), gradio('history'), gradio('temporary_text')).then(
diff --git a/modules/ui_default.py b/modules/ui_default.py
index c2946b37..8acc4b10 100644
--- a/modules/ui_default.py
+++ b/modules/ui_default.py
@@ -19,7 +19,7 @@ def create_ui():
with gr.Row():
with gr.Column():
with gr.Row():
- shared.gradio['textbox-default'] = gr.Textbox(value='', lines=27, label='Input', elem_classes=['textbox_default', 'add_scrollbar'])
+ shared.gradio['textbox-default'] = gr.Textbox(value=load_prompt(shared.settings['prompt-default']), lines=27, label='Input', elem_classes=['textbox_default', 'add_scrollbar'])
shared.gradio['token-counter-default'] = gr.HTML(value="0", elem_id="default-token-counter")
with gr.Row():
@@ -28,7 +28,7 @@ def create_ui():
shared.gradio['Generate-default'] = gr.Button('Generate', variant='primary')
with gr.Row():
- shared.gradio['prompt_menu-default'] = gr.Dropdown(choices=utils.get_available_prompts(), value='None', label='Prompt', elem_classes='slim-dropdown')
+ shared.gradio['prompt_menu-default'] = gr.Dropdown(choices=utils.get_available_prompts(), value=shared.settings['prompt-default'], label='Prompt', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['prompt_menu-default'], lambda: None, lambda: {'choices': utils.get_available_prompts()}, 'refresh-button', interactive=not mu)
shared.gradio['save_prompt-default'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_prompt-default'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
diff --git a/modules/ui_notebook.py b/modules/ui_notebook.py
index b234ac57..3f79a93c 100644
--- a/modules/ui_notebook.py
+++ b/modules/ui_notebook.py
@@ -22,7 +22,7 @@ def create_ui():
with gr.Column(scale=4):
with gr.Tab('Raw'):
with gr.Row():
- shared.gradio['textbox-notebook'] = gr.Textbox(value='', lines=27, elem_id='textbox-notebook', elem_classes=['textbox', 'add_scrollbar'])
+ shared.gradio['textbox-notebook'] = gr.Textbox(value=load_prompt(shared.settings['prompt-notebook']), lines=27, elem_id='textbox-notebook', elem_classes=['textbox', 'add_scrollbar'])
shared.gradio['token-counter-notebook'] = gr.HTML(value="0", elem_id="notebook-token-counter")
with gr.Tab('Markdown'):
@@ -56,7 +56,7 @@ def create_ui():
with gr.Column(scale=1):
gr.HTML('')
with gr.Row():
- shared.gradio['prompt_menu-notebook'] = gr.Dropdown(choices=utils.get_available_prompts(), value='None', label='Prompt', elem_classes='slim-dropdown')
+ shared.gradio['prompt_menu-notebook'] = gr.Dropdown(choices=utils.get_available_prompts(), value=shared.settings['prompt-notebook'], label='Prompt', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['prompt_menu-notebook'], lambda: None, lambda: {'choices': utils.get_available_prompts()}, ['refresh-button', 'refresh-button-small'], interactive=not mu)
shared.gradio['save_prompt-notebook'] = gr.Button('💾', elem_classes=['refresh-button', 'refresh-button-small'], interactive=not mu)
shared.gradio['delete_prompt-notebook'] = gr.Button('🗑️', elem_classes=['refresh-button', 'refresh-button-small'], interactive=not mu)
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index 84f9fbfc..ff965d39 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -6,19 +6,19 @@ from modules import loaders, presets, shared, ui, ui_chat, utils
from modules.utils import gradio
-def create_ui(default_preset):
+def create_ui():
mu = shared.args.multi_user
- generate_params = presets.load_preset(default_preset)
with gr.Tab("Parameters", elem_id="parameters"):
with gr.Tab("Generation"):
with gr.Row():
with gr.Column():
with gr.Row():
- shared.gradio['preset_menu'] = gr.Dropdown(choices=utils.get_available_presets(), value=default_preset, label='Preset', elem_classes='slim-dropdown')
+ shared.gradio['preset_menu'] = gr.Dropdown(choices=utils.get_available_presets(), value=shared.settings['preset'], label='Preset', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['preset_menu'], lambda: None, lambda: {'choices': utils.get_available_presets()}, 'refresh-button', interactive=not mu)
shared.gradio['save_preset'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_preset'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['random_preset'] = gr.Button('🎲', elem_classes='refresh-button')
+ shared.gradio['reset_preset'] = gr.Button('Reload preset', elem_classes='refresh-button', interactive=True)
+ shared.gradio['neutralize_samplers'] = gr.Button('Neutralize samplers', elem_classes='refresh-button', interactive=True)
with gr.Column():
shared.gradio['filter_by_loader'] = gr.Dropdown(label="Filter by loader", choices=["All"] + list(loaders.loaders_and_params.keys()) if not shared.args.portable else ['llama.cpp'], value="All", elem_classes='slim-dropdown')
@@ -28,44 +28,44 @@ def create_ui(default_preset):
with gr.Row():
with gr.Column():
gr.Markdown('## Curve shape')
- shared.gradio['temperature'] = gr.Slider(0.01, 5, value=generate_params['temperature'], step=0.01, label='temperature')
- shared.gradio['dynatemp_low'] = gr.Slider(0.01, 5, value=generate_params['dynatemp_low'], step=0.01, label='dynatemp_low', visible=generate_params['dynamic_temperature'])
- shared.gradio['dynatemp_high'] = gr.Slider(0.01, 5, value=generate_params['dynatemp_high'], step=0.01, label='dynatemp_high', visible=generate_params['dynamic_temperature'])
- shared.gradio['dynatemp_exponent'] = gr.Slider(0.01, 5, value=generate_params['dynatemp_exponent'], step=0.01, label='dynatemp_exponent', visible=generate_params['dynamic_temperature'])
- shared.gradio['smoothing_factor'] = gr.Slider(0.0, 10.0, value=generate_params['smoothing_factor'], step=0.01, label='smoothing_factor', info='Activates Quadratic Sampling.')
- shared.gradio['smoothing_curve'] = gr.Slider(1.0, 10.0, value=generate_params['smoothing_curve'], step=0.01, label='smoothing_curve', info='Adjusts the dropoff curve of Quadratic Sampling.')
+ shared.gradio['temperature'] = gr.Slider(0.01, 5, value=shared.settings['temperature'], step=0.01, label='temperature')
+ shared.gradio['dynatemp_low'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_low'], step=0.01, label='dynatemp_low', visible=shared.settings['dynamic_temperature'])
+ shared.gradio['dynatemp_high'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_high'], step=0.01, label='dynatemp_high', visible=shared.settings['dynamic_temperature'])
+ shared.gradio['dynatemp_exponent'] = gr.Slider(0.01, 5, value=shared.settings['dynatemp_exponent'], step=0.01, label='dynatemp_exponent', visible=shared.settings['dynamic_temperature'])
+ shared.gradio['smoothing_factor'] = gr.Slider(0.0, 10.0, value=shared.settings['smoothing_factor'], step=0.01, label='smoothing_factor', info='Activates Quadratic Sampling.')
+ shared.gradio['smoothing_curve'] = gr.Slider(1.0, 10.0, value=shared.settings['smoothing_curve'], step=0.01, label='smoothing_curve', info='Adjusts the dropoff curve of Quadratic Sampling.')
gr.Markdown('## Curve cutoff')
- shared.gradio['min_p'] = gr.Slider(0.0, 1.0, value=generate_params['min_p'], step=0.01, label='min_p')
- shared.gradio['top_n_sigma'] = gr.Slider(0.0, 5.0, value=generate_params['top_n_sigma'], step=0.01, label='top_n_sigma')
- shared.gradio['top_p'] = gr.Slider(0.0, 1.0, value=generate_params['top_p'], step=0.01, label='top_p')
- shared.gradio['top_k'] = gr.Slider(0, 200, value=generate_params['top_k'], step=1, label='top_k')
- shared.gradio['typical_p'] = gr.Slider(0.0, 1.0, value=generate_params['typical_p'], step=0.01, label='typical_p')
- shared.gradio['xtc_threshold'] = gr.Slider(0, 0.5, value=generate_params['xtc_threshold'], step=0.01, label='xtc_threshold', info='If 2 or more tokens have probability above this threshold, consider removing all but the last one.')
- shared.gradio['xtc_probability'] = gr.Slider(0, 1, value=generate_params['xtc_probability'], step=0.01, label='xtc_probability', info='Probability that the removal will actually happen. 0 disables the sampler. 1 makes it always happen.')
- shared.gradio['epsilon_cutoff'] = gr.Slider(0, 9, value=generate_params['epsilon_cutoff'], step=0.01, label='epsilon_cutoff')
- shared.gradio['eta_cutoff'] = gr.Slider(0, 20, value=generate_params['eta_cutoff'], step=0.01, label='eta_cutoff')
- shared.gradio['tfs'] = gr.Slider(0.0, 1.0, value=generate_params['tfs'], step=0.01, label='tfs')
- shared.gradio['top_a'] = gr.Slider(0.0, 1.0, value=generate_params['top_a'], step=0.01, label='top_a')
+ shared.gradio['min_p'] = gr.Slider(0.0, 1.0, value=shared.settings['min_p'], step=0.01, label='min_p')
+ shared.gradio['top_n_sigma'] = gr.Slider(0.0, 5.0, value=shared.settings['top_n_sigma'], step=0.01, label='top_n_sigma')
+ shared.gradio['top_p'] = gr.Slider(0.0, 1.0, value=shared.settings['top_p'], step=0.01, label='top_p')
+ shared.gradio['top_k'] = gr.Slider(0, 200, value=shared.settings['top_k'], step=1, label='top_k')
+ shared.gradio['typical_p'] = gr.Slider(0.0, 1.0, value=shared.settings['typical_p'], step=0.01, label='typical_p')
+ shared.gradio['xtc_threshold'] = gr.Slider(0, 0.5, value=shared.settings['xtc_threshold'], step=0.01, label='xtc_threshold', info='If 2 or more tokens have probability above this threshold, consider removing all but the last one.')
+ shared.gradio['xtc_probability'] = gr.Slider(0, 1, value=shared.settings['xtc_probability'], step=0.01, label='xtc_probability', info='Probability that the removal will actually happen. 0 disables the sampler. 1 makes it always happen.')
+ shared.gradio['epsilon_cutoff'] = gr.Slider(0, 9, value=shared.settings['epsilon_cutoff'], step=0.01, label='epsilon_cutoff')
+ shared.gradio['eta_cutoff'] = gr.Slider(0, 20, value=shared.settings['eta_cutoff'], step=0.01, label='eta_cutoff')
+ shared.gradio['tfs'] = gr.Slider(0.0, 1.0, value=shared.settings['tfs'], step=0.01, label='tfs')
+ shared.gradio['top_a'] = gr.Slider(0.0, 1.0, value=shared.settings['top_a'], step=0.01, label='top_a')
gr.Markdown('## Repetition suppression')
- shared.gradio['dry_multiplier'] = gr.Slider(0, 5, value=generate_params['dry_multiplier'], step=0.01, label='dry_multiplier', info='Set to greater than 0 to enable DRY. Recommended value: 0.8.')
- shared.gradio['dry_allowed_length'] = gr.Slider(1, 20, value=generate_params['dry_allowed_length'], step=1, label='dry_allowed_length', info='Longest sequence that can be repeated without being penalized.')
- shared.gradio['dry_base'] = gr.Slider(1, 4, value=generate_params['dry_base'], step=0.01, label='dry_base', info='Controls how fast the penalty grows with increasing sequence length.')
- shared.gradio['repetition_penalty'] = gr.Slider(1.0, 1.5, value=generate_params['repetition_penalty'], step=0.01, label='repetition_penalty')
- shared.gradio['frequency_penalty'] = gr.Slider(0, 2, value=generate_params['frequency_penalty'], step=0.05, label='frequency_penalty')
- shared.gradio['presence_penalty'] = gr.Slider(0, 2, value=generate_params['presence_penalty'], step=0.05, label='presence_penalty')
- shared.gradio['encoder_repetition_penalty'] = gr.Slider(0.8, 1.5, value=generate_params['encoder_repetition_penalty'], step=0.01, label='encoder_repetition_penalty')
- shared.gradio['no_repeat_ngram_size'] = gr.Slider(0, 20, step=1, value=generate_params['no_repeat_ngram_size'], label='no_repeat_ngram_size')
- shared.gradio['repetition_penalty_range'] = gr.Slider(0, 4096, step=64, value=generate_params['repetition_penalty_range'], label='repetition_penalty_range')
+ shared.gradio['dry_multiplier'] = gr.Slider(0, 5, value=shared.settings['dry_multiplier'], step=0.01, label='dry_multiplier', info='Set to greater than 0 to enable DRY. Recommended value: 0.8.')
+ shared.gradio['dry_allowed_length'] = gr.Slider(1, 20, value=shared.settings['dry_allowed_length'], step=1, label='dry_allowed_length', info='Longest sequence that can be repeated without being penalized.')
+ shared.gradio['dry_base'] = gr.Slider(1, 4, value=shared.settings['dry_base'], step=0.01, label='dry_base', info='Controls how fast the penalty grows with increasing sequence length.')
+ shared.gradio['repetition_penalty'] = gr.Slider(1.0, 1.5, value=shared.settings['repetition_penalty'], step=0.01, label='repetition_penalty')
+ shared.gradio['frequency_penalty'] = gr.Slider(0, 2, value=shared.settings['frequency_penalty'], step=0.05, label='frequency_penalty')
+ shared.gradio['presence_penalty'] = gr.Slider(0, 2, value=shared.settings['presence_penalty'], step=0.05, label='presence_penalty')
+ shared.gradio['encoder_repetition_penalty'] = gr.Slider(0.8, 1.5, value=shared.settings['encoder_repetition_penalty'], step=0.01, label='encoder_repetition_penalty')
+ shared.gradio['no_repeat_ngram_size'] = gr.Slider(0, 20, step=1, value=shared.settings['no_repeat_ngram_size'], label='no_repeat_ngram_size')
+ shared.gradio['repetition_penalty_range'] = gr.Slider(0, 4096, step=64, value=shared.settings['repetition_penalty_range'], label='repetition_penalty_range')
with gr.Column():
gr.Markdown('## Alternative sampling methods')
- shared.gradio['penalty_alpha'] = gr.Slider(0, 5, value=generate_params['penalty_alpha'], label='penalty_alpha', info='For Contrastive Search. do_sample must be unchecked.')
- shared.gradio['guidance_scale'] = gr.Slider(-0.5, 2.5, step=0.05, value=generate_params['guidance_scale'], label='guidance_scale', info='For CFG. 1.5 is a good value.')
- shared.gradio['mirostat_mode'] = gr.Slider(0, 2, step=1, value=generate_params['mirostat_mode'], label='mirostat_mode', info='mode=1 is for llama.cpp only.')
- shared.gradio['mirostat_tau'] = gr.Slider(0, 10, step=0.01, value=generate_params['mirostat_tau'], label='mirostat_tau')
- shared.gradio['mirostat_eta'] = gr.Slider(0, 1, step=0.01, value=generate_params['mirostat_eta'], label='mirostat_eta')
+ shared.gradio['penalty_alpha'] = gr.Slider(0, 5, value=shared.settings['penalty_alpha'], label='penalty_alpha', info='For Contrastive Search. do_sample must be unchecked.')
+ shared.gradio['guidance_scale'] = gr.Slider(-0.5, 2.5, step=0.05, value=shared.settings['guidance_scale'], label='guidance_scale', info='For CFG. 1.5 is a good value.')
+ shared.gradio['mirostat_mode'] = gr.Slider(0, 2, step=1, value=shared.settings['mirostat_mode'], label='mirostat_mode', info='mode=1 is for llama.cpp only.')
+ shared.gradio['mirostat_tau'] = gr.Slider(0, 10, step=0.01, value=shared.settings['mirostat_tau'], label='mirostat_tau')
+ shared.gradio['mirostat_eta'] = gr.Slider(0, 1, step=0.01, value=shared.settings['mirostat_eta'], label='mirostat_eta')
gr.Markdown('## Other options')
shared.gradio['max_new_tokens'] = gr.Slider(minimum=shared.settings['max_new_tokens_min'], maximum=shared.settings['max_new_tokens_max'], value=shared.settings['max_new_tokens'], step=1, label='max_new_tokens', info='⚠️ Setting this too high can cause prompt truncation.')
@@ -74,9 +74,9 @@ def create_ui(default_preset):
with gr.Column():
with gr.Row():
with gr.Column():
- shared.gradio['do_sample'] = gr.Checkbox(value=generate_params['do_sample'], label='do_sample')
- shared.gradio['dynamic_temperature'] = gr.Checkbox(value=generate_params['dynamic_temperature'], label='dynamic_temperature')
- shared.gradio['temperature_last'] = gr.Checkbox(value=generate_params['temperature_last'], label='temperature_last', info='Moves temperature/dynamic temperature/quadratic sampling to the end of the sampler stack, ignoring their positions in "Sampler priority".')
+ shared.gradio['do_sample'] = gr.Checkbox(value=shared.settings['do_sample'], label='do_sample')
+ shared.gradio['dynamic_temperature'] = gr.Checkbox(value=shared.settings['dynamic_temperature'], label='dynamic_temperature')
+ shared.gradio['temperature_last'] = gr.Checkbox(value=shared.settings['temperature_last'], label='temperature_last', info='Moves temperature/dynamic temperature/quadratic sampling to the end of the sampler stack, ignoring their positions in "Sampler priority".')
shared.gradio['auto_max_new_tokens'] = gr.Checkbox(value=shared.settings['auto_max_new_tokens'], label='auto_max_new_tokens', info='Expand max_new_tokens to the available context length.')
shared.gradio['ban_eos_token'] = gr.Checkbox(value=shared.settings['ban_eos_token'], label='Ban the eos_token', info='Forces the model to never end the generation prematurely.')
shared.gradio['add_bos_token'] = gr.Checkbox(value=shared.settings['add_bos_token'], label='Add the bos_token to the beginning of prompts', info='Disabling this can make the replies more creative.')
@@ -89,18 +89,18 @@ def create_ui(default_preset):
shared.gradio['truncation_length'] = gr.Number(precision=0, step=256, value=get_truncation_length(), label='Truncate the prompt up to this length', info='The leftmost tokens are removed if the prompt exceeds this length.')
shared.gradio['seed'] = gr.Number(value=shared.settings['seed'], label='Seed (-1 for random)')
- shared.gradio['sampler_priority'] = gr.Textbox(value=generate_params['sampler_priority'], lines=12, label='Sampler priority', info='Parameter names separated by new lines or commas.', elem_classes=['add_scrollbar'])
+ shared.gradio['sampler_priority'] = gr.Textbox(value=shared.settings['sampler_priority'], lines=12, label='Sampler priority', info='Parameter names separated by new lines or commas.', elem_classes=['add_scrollbar'])
shared.gradio['custom_stopping_strings'] = gr.Textbox(lines=2, value=shared.settings["custom_stopping_strings"] or None, label='Custom stopping strings', info='Written between "" and separated by commas.', placeholder='"\\n", "\\nYou:"')
shared.gradio['custom_token_bans'] = gr.Textbox(value=shared.settings['custom_token_bans'] or None, label='Token bans', info='Token IDs to ban, separated by commas. The IDs can be found in the Default or Notebook tab.')
shared.gradio['negative_prompt'] = gr.Textbox(value=shared.settings['negative_prompt'], label='Negative prompt', info='For CFG. Only used when guidance_scale is different than 1.', lines=3, elem_classes=['add_scrollbar'])
- shared.gradio['dry_sequence_breakers'] = gr.Textbox(value=generate_params['dry_sequence_breakers'], label='dry_sequence_breakers', info='Tokens across which sequence matching is not continued. Specified as a comma-separated list of quoted strings.')
+ shared.gradio['dry_sequence_breakers'] = gr.Textbox(value=shared.settings['dry_sequence_breakers'], label='dry_sequence_breakers', info='Tokens across which sequence matching is not continued. Specified as a comma-separated list of quoted strings.')
with gr.Row() as shared.gradio['grammar_file_row']:
shared.gradio['grammar_file'] = gr.Dropdown(value='None', choices=utils.get_available_grammars(), label='Load grammar from file (.gbnf)', elem_classes='slim-dropdown')
ui.create_refresh_button(shared.gradio['grammar_file'], lambda: None, lambda: {'choices': utils.get_available_grammars()}, 'refresh-button', interactive=not mu)
shared.gradio['save_grammar'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_grammar'] = gr.Button('🗑️ ', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['grammar_string'] = gr.Textbox(value='', label='Grammar', lines=16, elem_classes=['add_scrollbar', 'monospace'])
+ shared.gradio['grammar_string'] = gr.Textbox(value=shared.settings['grammar_string'], label='Grammar', lines=16, elem_classes=['add_scrollbar', 'monospace'])
ui_chat.create_chat_settings_ui()
@@ -111,9 +111,13 @@ def create_event_handlers():
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
presets.load_preset_for_ui, gradio('preset_menu', 'interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
- shared.gradio['random_preset'].click(
+ shared.gradio['reset_preset'].click(
ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
- presets.random_preset, gradio('interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
+ presets.reset_preset_for_ui, gradio('preset_menu', 'interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
+
+ shared.gradio['neutralize_samplers'].click(
+ ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ presets.neutralize_samplers_for_ui, gradio('interface_state'), gradio('interface_state') + gradio(presets.presets_params()), show_progress=False)
shared.gradio['grammar_file'].change(load_grammar, gradio('grammar_file'), gradio('grammar_string'), show_progress=False)
shared.gradio['dynamic_temperature'].change(lambda x: [gr.update(visible=x)] * 3, gradio('dynamic_temperature'), gradio('dynatemp_low', 'dynatemp_high', 'dynatemp_exponent'), show_progress=False)
diff --git a/server.py b/server.py
index 3be4c27c..3dd0a3f3 100644
--- a/server.py
+++ b/server.py
@@ -45,6 +45,7 @@ from modules import (
ui_session,
utils
)
+from modules.chat import generate_pfp_cache
from modules.extensions import apply_extensions
from modules.LoRA import add_lora_to_model
from modules.models import load_model, unload_model_if_idle
@@ -93,17 +94,20 @@ def create_interface():
# Force some events to be triggered on page load
shared.persistent_interface_state.update({
+ 'mode': shared.settings['mode'],
'loader': shared.args.loader or 'llama.cpp',
- 'mode': shared.settings['mode'] if shared.settings['mode'] == 'instruct' else gr.update(),
- 'character_menu': shared.args.character or shared.settings['character'],
- 'instruction_template_str': shared.settings['instruction_template_str'],
- 'prompt_menu-default': shared.settings['prompt-default'],
- 'prompt_menu-notebook': shared.settings['prompt-notebook'],
'filter_by_loader': (shared.args.loader or 'All') if not shared.args.portable else 'llama.cpp'
})
- if Path("user_data/cache/pfp_character.png").exists():
- Path("user_data/cache/pfp_character.png").unlink()
+ # Clear existing cache files
+ for cache_file in ['pfp_character.png', 'pfp_character_thumb.png']:
+ cache_path = Path(f"user_data/cache/{cache_file}")
+ if cache_path.exists():
+ cache_path.unlink()
+
+ # Regenerate for default character
+ if shared.settings['mode'] != 'instruct':
+ generate_pfp_cache(shared.settings['character'])
# css/js strings
css = ui.css
@@ -134,7 +138,7 @@ def create_interface():
ui_default.create_ui()
ui_notebook.create_ui()
- ui_parameters.create_ui(shared.settings['preset']) # Parameters tab
+ ui_parameters.create_ui() # Parameters tab
ui_model_menu.create_ui() # Model tab
if not shared.args.portable:
training.create_ui() # Training tab
From 3650a6fd1fc332f462bc1aa08f7671a3d517d847 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:02:34 -0700
Subject: [PATCH 37/82] Small UI changes
---
css/main.css | 2 +-
modules/ui_chat.py | 3 +--
modules/ui_parameters.py | 2 +-
3 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/css/main.css b/css/main.css
index a9cb36ab..4277519a 100644
--- a/css/main.css
+++ b/css/main.css
@@ -53,7 +53,7 @@ div.svelte-iyf88w {
}
.refresh-button {
- max-width: 4.4em;
+ max-width: none;
min-width: 2.2em !important;
height: 39.594px;
align-self: end;
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index 712843d3..f8e2fc32 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -122,8 +122,7 @@ def create_chat_settings_ui():
ui.create_refresh_button(shared.gradio['character_menu'], lambda: None, lambda: {'choices': utils.get_available_characters()}, 'refresh-button', interactive=not mu)
shared.gradio['save_character'] = gr.Button('💾', elem_classes='refresh-button', elem_id="save-character", interactive=not mu)
shared.gradio['delete_character'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['reset_character'] = gr.Button('↺', elem_classes='refresh-button', interactive=True)
- shared.gradio['clear_character'] = gr.Button('⚪', elem_classes='refresh-button', interactive=True)
+ shared.gradio['reset_character'] = gr.Button('Restore character', elem_classes='refresh-button', interactive=True)
shared.gradio['name2'] = gr.Textbox(value=shared.settings['name2'], lines=1, label='Character\'s name')
shared.gradio['context'] = gr.Textbox(value=shared.settings['context'], lines=10, label='Context', elem_classes=['add_scrollbar'])
diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py
index ff965d39..9b5cb3ab 100644
--- a/modules/ui_parameters.py
+++ b/modules/ui_parameters.py
@@ -17,7 +17,7 @@ def create_ui():
ui.create_refresh_button(shared.gradio['preset_menu'], lambda: None, lambda: {'choices': utils.get_available_presets()}, 'refresh-button', interactive=not mu)
shared.gradio['save_preset'] = gr.Button('💾', elem_classes='refresh-button', interactive=not mu)
shared.gradio['delete_preset'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['reset_preset'] = gr.Button('Reload preset', elem_classes='refresh-button', interactive=True)
+ shared.gradio['reset_preset'] = gr.Button('Restore preset', elem_classes='refresh-button', interactive=True)
shared.gradio['neutralize_samplers'] = gr.Button('Neutralize samplers', elem_classes='refresh-button', interactive=True)
with gr.Column():
From caf9fca5f339e4018913608f8accb3897352853f Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:11:35 -0700
Subject: [PATCH 38/82] Avoid some code repetition
---
modules/shared.py | 190 +++++++++++++++++++++++-----------------------
1 file changed, 97 insertions(+), 93 deletions(-)
diff --git a/modules/shared.py b/modules/shared.py
index f4f8e180..08200399 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -9,6 +9,7 @@ from pathlib import Path
import yaml
from modules.logging_colors import logger
+from modules.presets import default_preset
# Model variables
model = None
@@ -28,99 +29,6 @@ gradio = {}
persistent_interface_state = {}
need_restart = False
-# UI defaults
-settings = {
- 'show_controls': True,
- 'start_with': '',
- 'mode': 'instruct',
- 'chat_style': 'cai-chat',
- 'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
- 'enable_web_search': False,
- 'web_search_pages': 3,
- 'prompt-default': 'QA',
- 'prompt-notebook': 'QA',
- 'preset': 'min_p',
- 'max_new_tokens': 512,
- 'max_new_tokens_min': 1,
- 'max_new_tokens_max': 4096,
- 'prompt_lookup_num_tokens': 0,
- 'max_tokens_second': 0,
- 'auto_max_new_tokens': True,
- 'ban_eos_token': False,
- 'add_bos_token': True,
- 'enable_thinking': True,
- 'skip_special_tokens': True,
- 'stream': True,
- 'static_cache': False,
- 'truncation_length': 8192,
- 'seed': -1,
- 'custom_stopping_strings': '',
- 'custom_token_bans': '',
- 'negative_prompt': '',
- 'dark_theme': True,
- 'default_extensions': [],
-
- # Character settings
- 'character': 'Assistant',
- 'name1': 'You',
- 'name2': 'AI',
- 'user_bio': '',
- 'context': 'The following is a conversation with an AI Large Language Model. The AI has been trained to answer questions, provide recommendations, and help with decision making. The AI follows user requests. The AI thinks outside the box.',
- 'greeting': 'How can I help you today?',
- 'custom_system_message': '',
- 'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}",
- 'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
-
- # Generation parameters - Curve shape
- 'temperature': 1.0,
- 'dynatemp_low': 1.0,
- 'dynatemp_high': 1.0,
- 'dynatemp_exponent': 1.0,
- 'smoothing_factor': 0.0,
- 'smoothing_curve': 1.0,
-
- # Generation parameters - Curve cutoff
- 'min_p': 0.0,
- 'top_p': 1.0,
- 'top_k': 0,
- 'typical_p': 1.0,
- 'xtc_threshold': 0.1,
- 'xtc_probability': 0.0,
- 'epsilon_cutoff': 0.0,
- 'eta_cutoff': 0.0,
- 'tfs': 1.0,
- 'top_a': 0.0,
- 'top_n_sigma': 0.0,
-
- # Generation parameters - Repetition suppression
- 'dry_multiplier': 0.0,
- 'dry_allowed_length': 2,
- 'dry_base': 1.75,
- 'repetition_penalty': 1.0,
- 'frequency_penalty': 0.0,
- 'presence_penalty': 0.0,
- 'encoder_repetition_penalty': 1.0,
- 'no_repeat_ngram_size': 0,
- 'repetition_penalty_range': 1024,
-
- # Generation parameters - Alternative sampling methods
- 'penalty_alpha': 0.0,
- 'guidance_scale': 1.0,
- 'mirostat_mode': 0,
- 'mirostat_tau': 5.0,
- 'mirostat_eta': 0.1,
-
- # Generation parameters - Other options
- 'do_sample': True,
- 'dynamic_temperature': False,
- 'temperature_last': False,
- 'sampler_priority': 'repetition_penalty\npresence_penalty\nfrequency_penalty\ndry\ntop_n_sigma\ntemperature\ndynamic_temperature\nquadratic_sampling\ntop_k\ntop_p\ntypical_p\nepsilon_cutoff\neta_cutoff\ntfs\ntop_a\nmin_p\nmirostat\nxtc\nencoder_repetition_penalty\nno_repeat_ngram',
- 'dry_sequence_breakers': '"\\n", ":", "\\"", "*"',
- 'grammar_string': '',
-}
-
-default_settings = copy.deepcopy(settings)
-
# Parser copied from https://github.com/vladmandic/automatic
parser = argparse.ArgumentParser(description="Text generation web UI", conflict_handler='resolve', add_help=True, formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=55, indent_increment=2, width=200))
@@ -282,6 +190,102 @@ for arg in sys.argv[1:]:
elif hasattr(args, arg):
provided_arguments.append(arg)
+# Default generation parameters
+neutral_samplers = default_preset()
+
+# UI defaults
+settings = {
+ 'show_controls': True,
+ 'start_with': '',
+ 'mode': 'instruct',
+ 'chat_style': 'cai-chat',
+ 'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
+ 'enable_web_search': False,
+ 'web_search_pages': 3,
+ 'prompt-default': 'QA',
+ 'prompt-notebook': 'QA',
+ 'preset': 'min_p',
+ 'max_new_tokens': 512,
+ 'max_new_tokens_min': 1,
+ 'max_new_tokens_max': 4096,
+ 'prompt_lookup_num_tokens': 0,
+ 'max_tokens_second': 0,
+ 'auto_max_new_tokens': True,
+ 'ban_eos_token': False,
+ 'add_bos_token': True,
+ 'enable_thinking': True,
+ 'skip_special_tokens': True,
+ 'stream': True,
+ 'static_cache': False,
+ 'truncation_length': 8192,
+ 'seed': -1,
+ 'custom_stopping_strings': '',
+ 'custom_token_bans': '',
+ 'negative_prompt': '',
+ 'dark_theme': True,
+ 'default_extensions': [],
+
+ # Character settings
+ 'character': 'Assistant',
+ 'name1': 'You',
+ 'name2': 'AI',
+ 'user_bio': '',
+ 'context': 'The following is a conversation with an AI Large Language Model. The AI has been trained to answer questions, provide recommendations, and help with decision making. The AI follows user requests. The AI thinks outside the box.',
+ 'greeting': 'How can I help you today?',
+ 'custom_system_message': '',
+ 'instruction_template_str': "{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not ns.found -%}\n {{- '' + 'Below is an instruction that describes a task. Write a response that appropriately completes the request.' + '\\n\\n' -}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {{- '' + message['content'] + '\\n\\n' -}}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{-'### Instruction:\\n' + message['content'] + '\\n\\n'-}}\n {%- else -%}\n {{-'### Response:\\n' + message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{-'### Response:\\n'-}}\n{%- endif -%}",
+ 'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
+
+ # Generation parameters - Curve shape
+ 'temperature': neutral_samplers['temperature'],
+ 'dynatemp_low': neutral_samplers['dynatemp_low'],
+ 'dynatemp_high': neutral_samplers['dynatemp_high'],
+ 'dynatemp_exponent': neutral_samplers['dynatemp_exponent'],
+ 'smoothing_factor': neutral_samplers['smoothing_factor'],
+ 'smoothing_curve': neutral_samplers['smoothing_curve'],
+
+ # Generation parameters - Curve cutoff
+ 'min_p': neutral_samplers['min_p'],
+ 'top_p': neutral_samplers['top_p'],
+ 'top_k': neutral_samplers['top_k'],
+ 'typical_p': neutral_samplers['typical_p'],
+ 'xtc_threshold': neutral_samplers['xtc_threshold'],
+ 'xtc_probability': neutral_samplers['xtc_probability'],
+ 'epsilon_cutoff': neutral_samplers['epsilon_cutoff'],
+ 'eta_cutoff': neutral_samplers['eta_cutoff'],
+ 'tfs': neutral_samplers['tfs'],
+ 'top_a': neutral_samplers['top_a'],
+ 'top_n_sigma': neutral_samplers['top_n_sigma'],
+
+ # Generation parameters - Repetition suppression
+ 'dry_multiplier': neutral_samplers['dry_multiplier'],
+ 'dry_allowed_length': neutral_samplers['dry_allowed_length'],
+ 'dry_base': neutral_samplers['dry_base'],
+ 'repetition_penalty': neutral_samplers['repetition_penalty'],
+ 'frequency_penalty': neutral_samplers['frequency_penalty'],
+ 'presence_penalty': neutral_samplers['presence_penalty'],
+ 'encoder_repetition_penalty': neutral_samplers['encoder_repetition_penalty'],
+ 'no_repeat_ngram_size': neutral_samplers['no_repeat_ngram_size'],
+ 'repetition_penalty_range': neutral_samplers['repetition_penalty_range'],
+
+ # Generation parameters - Alternative sampling methods
+ 'penalty_alpha': neutral_samplers['penalty_alpha'],
+ 'guidance_scale': neutral_samplers['guidance_scale'],
+ 'mirostat_mode': neutral_samplers['mirostat_mode'],
+ 'mirostat_tau': neutral_samplers['mirostat_tau'],
+ 'mirostat_eta': neutral_samplers['mirostat_eta'],
+
+ # Generation parameters - Other options
+ 'do_sample': neutral_samplers['do_sample'],
+ 'dynamic_temperature': neutral_samplers['dynamic_temperature'],
+ 'temperature_last': neutral_samplers['temperature_last'],
+ 'sampler_priority': neutral_samplers['sampler_priority'],
+ 'dry_sequence_breakers': neutral_samplers['dry_sequence_breakers'],
+ 'grammar_string': '',
+}
+
+default_settings = copy.deepcopy(settings)
+
def do_cmd_flags_warnings():
# Security warnings
From fe955cac1fd7298d820e42790e257d9ace1c5eb4 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:15:19 -0700
Subject: [PATCH 39/82] Small UI changes
---
css/main.css | 4 ++++
modules/ui_chat.py | 8 ++++----
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/css/main.css b/css/main.css
index 4277519a..7d7b0cbd 100644
--- a/css/main.css
+++ b/css/main.css
@@ -62,6 +62,10 @@ div.svelte-iyf88w {
flex: none;
}
+.refresh-button-medium {
+ max-width: 4.4em;
+}
+
.refresh-button-small {
max-width: 2.2em;
}
diff --git a/modules/ui_chat.py b/modules/ui_chat.py
index f8e2fc32..ef9330e4 100644
--- a/modules/ui_chat.py
+++ b/modules/ui_chat.py
@@ -24,11 +24,11 @@ def create_ui():
with gr.Row(elem_id='past-chats-row', elem_classes=['pretty_scrollbar']):
with gr.Column():
with gr.Row(elem_id='past-chats-buttons'):
- shared.gradio['branch_chat'] = gr.Button('Branch', elem_classes='refresh-button', elem_id='Branch', interactive=not mu)
- shared.gradio['branch_index'] = gr.Number(value=-1, precision=0, visible=False, elem_id="Branch-index", interactive=True)
- shared.gradio['rename_chat'] = gr.Button('Rename', elem_classes='refresh-button', interactive=not mu)
+ shared.gradio['branch_chat'] = gr.Button('Branch', elem_classes=['refresh-button', 'refresh-button-medium'], elem_id='Branch', interactive=not mu)
+ shared.gradio['rename_chat'] = gr.Button('Rename', elem_classes=['refresh-button', 'refresh-button-medium'], interactive=not mu)
shared.gradio['delete_chat'] = gr.Button('🗑️', elem_classes='refresh-button', interactive=not mu)
- shared.gradio['Start new chat'] = gr.Button('New chat', elem_classes=['refresh-button', 'focus-on-chat-input'])
+ shared.gradio['Start new chat'] = gr.Button('New chat', elem_classes=['refresh-button', 'refresh-button-medium', 'focus-on-chat-input'])
+ shared.gradio['branch_index'] = gr.Number(value=-1, precision=0, visible=False, elem_id="Branch-index", interactive=True)
shared.gradio['search_chat'] = gr.Textbox(placeholder='Search chats...', max_lines=1, elem_id='search_chat')
From 0dbc4cbc71e7eed1d51fd9169048a54cb8d2d927 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:20:58 -0700
Subject: [PATCH 40/82] Add Qwen3 presets
---
user_data/presets/Qwen3 - No Thinking.yaml | 3 +++
user_data/presets/Qwen3 - Thinking.yaml | 3 +++
2 files changed, 6 insertions(+)
create mode 100644 user_data/presets/Qwen3 - No Thinking.yaml
create mode 100644 user_data/presets/Qwen3 - Thinking.yaml
diff --git a/user_data/presets/Qwen3 - No Thinking.yaml b/user_data/presets/Qwen3 - No Thinking.yaml
new file mode 100644
index 00000000..b1c1e03c
--- /dev/null
+++ b/user_data/presets/Qwen3 - No Thinking.yaml
@@ -0,0 +1,3 @@
+temperature: 0.7
+top_p: 0.8
+top_k: 20
diff --git a/user_data/presets/Qwen3 - Thinking.yaml b/user_data/presets/Qwen3 - Thinking.yaml
new file mode 100644
index 00000000..cb2942f9
--- /dev/null
+++ b/user_data/presets/Qwen3 - Thinking.yaml
@@ -0,0 +1,3 @@
+temperature: 0.6
+top_p: 0.95
+top_k: 20
From 1bdf11b511b69f6eb10f9067847c64d39c8872f4 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:23:09 -0700
Subject: [PATCH 41/82] Use the Qwen3 - Thinking preset by default
---
modules/shared.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/modules/shared.py b/modules/shared.py
index 08200399..3794206b 100644
--- a/modules/shared.py
+++ b/modules/shared.py
@@ -204,7 +204,7 @@ settings = {
'web_search_pages': 3,
'prompt-default': 'QA',
'prompt-notebook': 'QA',
- 'preset': 'min_p',
+ 'preset': 'Qwen3 - Thinking' if Path('user_data/presets/Qwen3 - Thinking.yaml').exists() else '',
'max_new_tokens': 512,
'max_new_tokens_min': 1,
'max_new_tokens_max': 4096,
@@ -237,7 +237,7 @@ settings = {
'chat_template_str': "{%- for message in messages %}\n {%- if message['role'] == 'system' -%}\n {%- if message['content'] -%}\n {{- message['content'] + '\\n\\n' -}}\n {%- endif -%}\n {%- if user_bio -%}\n {{- user_bio + '\\n\\n' -}}\n {%- endif -%}\n {%- else -%}\n {%- if message['role'] == 'user' -%}\n {{- name1 + ': ' + message['content'] + '\\n'-}}\n {%- else -%}\n {{- name2 + ': ' + message['content'] + '\\n' -}}\n {%- endif -%}\n {%- endif -%}\n{%- endfor -%}",
# Generation parameters - Curve shape
- 'temperature': neutral_samplers['temperature'],
+ 'temperature': 0.6,
'dynatemp_low': neutral_samplers['dynatemp_low'],
'dynatemp_high': neutral_samplers['dynatemp_high'],
'dynatemp_exponent': neutral_samplers['dynatemp_exponent'],
@@ -246,8 +246,8 @@ settings = {
# Generation parameters - Curve cutoff
'min_p': neutral_samplers['min_p'],
- 'top_p': neutral_samplers['top_p'],
- 'top_k': neutral_samplers['top_k'],
+ 'top_p': 0.95,
+ 'top_k': 20,
'typical_p': neutral_samplers['typical_p'],
'xtc_threshold': neutral_samplers['xtc_threshold'],
'xtc_probability': neutral_samplers['xtc_probability'],
From ae150fa24f204fb56c9fe4e2530435bfcf0ec1d7 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:25:46 -0700
Subject: [PATCH 42/82] Remove the null preset
---
user_data/presets/Null preset.yaml | 1 -
1 file changed, 1 deletion(-)
delete mode 100644 user_data/presets/Null preset.yaml
diff --git a/user_data/presets/Null preset.yaml b/user_data/presets/Null preset.yaml
deleted file mode 100644
index 714aa9a3..00000000
--- a/user_data/presets/Null preset.yaml
+++ /dev/null
@@ -1 +0,0 @@
-temperature: 1
From 1cab149c1a020c86f215bafd48b809cbc02a225b Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sat, 7 Jun 2025 22:26:13 -0700
Subject: [PATCH 43/82] Remove the contrastive search preset
---
user_data/presets/Contrastive Search.yaml | 3 ---
1 file changed, 3 deletions(-)
delete mode 100644 user_data/presets/Contrastive Search.yaml
diff --git a/user_data/presets/Contrastive Search.yaml b/user_data/presets/Contrastive Search.yaml
deleted file mode 100644
index d9a47a9f..00000000
--- a/user_data/presets/Contrastive Search.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-do_sample: false
-top_k: 4
-penalty_alpha: 0.3
From af6bb7513afc6a45cba1716a8cd665d4ee264779 Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Sun, 8 Jun 2025 09:09:36 -0700
Subject: [PATCH 44/82] Add back the "Save UI defaults" button
It's useful for saving extensions settings.
---
modules/ui_session.py | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/modules/ui_session.py b/modules/ui_session.py
index 4ed740cd..db9ce3b0 100644
--- a/modules/ui_session.py
+++ b/modules/ui_session.py
@@ -12,6 +12,7 @@ def create_ui():
with gr.Column():
with gr.Row():
shared.gradio['toggle_dark_mode'] = gr.Button('Toggle 💡')
+ shared.gradio['save_settings'] = gr.Button('Save UI defaults to user_data/settings.yaml', interactive=not mu)
shared.gradio['reset_interface'] = gr.Button("Apply flags/extensions and restart", interactive=not mu)
with gr.Row():
@@ -32,14 +33,28 @@ def create_ui():
if not shared.args.portable:
extension_name.submit(clone_or_pull_repository, extension_name, extension_status, show_progress=False)
+ shared.gradio['save_settings'].click(
+ ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then(
+ handle_save_settings, gradio('interface_state', 'preset_menu', 'extensions_menu', 'show_controls', 'theme_state'), gradio('save_contents', 'save_filename', 'save_root', 'file_saver'), show_progress=False)
+
+ shared.gradio['toggle_dark_mode'].click(
+ lambda x: 'dark' if x == 'light' else 'light', gradio('theme_state'), gradio('theme_state')).then(
+ None, None, None, js=f'() => {{{ui.dark_theme_js}; toggleDarkMode()}}')
+
# Reset interface event
shared.gradio['reset_interface'].click(
set_interface_arguments, gradio('extensions_menu', 'bool_menu'), None).then(
None, None, None, js='() => {document.body.innerHTML=\'