diff --git a/extensions/openai/models.py b/extensions/openai/models.py index d6ef119d..82c65093 100644 --- a/extensions/openai/models.py +++ b/extensions/openai/models.py @@ -1,4 +1,4 @@ -from modules import shared, ui +from modules import loaders, shared from modules.logging_colors import logger from modules.LoRA import add_lora_to_model from modules.models import load_model, unload_model @@ -50,7 +50,7 @@ def _load_model(data): # parameters exposed in the UI. Never allow security-sensitive # flags like trust_remote_code or extra_flags to be set via the API. blocked_keys = {'extra_flags'} - allowed_keys = set(ui.list_model_elements()) - blocked_keys + allowed_keys = set(loaders.list_model_elements()) - blocked_keys if args: for k in args: if k in allowed_keys and hasattr(shared.args, k): diff --git a/modules/chat.py b/modules/chat.py index 36d373d6..62d1492d 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -11,7 +11,6 @@ from datetime import datetime from functools import partial from pathlib import Path -import gradio as gr import yaml from jinja2.ext import loopcontrols from jinja2.sandbox import ImmutableSandboxedEnvironment @@ -1333,6 +1332,7 @@ def load_history_after_deletion(state, idx): Loads the latest history for the given character in chat or chat-instruct mode, or the latest instruct history for instruct mode. ''' + import gradio as gr if shared.args.multi_user: return start_new_chat(state) @@ -1351,6 +1351,7 @@ def load_history_after_deletion(state, idx): def update_character_menu_after_deletion(idx): + import gradio as gr characters = utils.get_available_characters() idx = min(int(idx), len(characters) - 1) idx = max(0, idx) @@ -1565,24 +1566,6 @@ def clear_character_for_ui(state): return state, state['name2'], state['context'], state['greeting'], None -def load_instruction_template(template): - if template == 'None': - return '' - - for filepath in [shared.user_data_dir / 'instruction-templates' / f'{template}.yaml', shared.user_data_dir / 'instruction-templates' / 'Alpaca.yaml']: - if filepath.exists(): - break - else: - return '' - - file_contents = open(filepath, 'r', encoding='utf-8').read() - data = yaml.safe_load(file_contents) - if 'instruction_template' in data: - return data['instruction_template'] - else: - return jinja_template_from_old_format(data) - - @functools.cache def load_character_memoized(character, name1, name2): return load_character(character, name1, name2) @@ -1590,10 +1573,12 @@ def load_character_memoized(character, name1, name2): @functools.cache def load_instruction_template_memoized(template): + from modules.models_settings import load_instruction_template return load_instruction_template(template) def upload_character(file, img_path, tavern=False): + import gradio as gr img = open_image_safely(img_path) decoded_file = file if isinstance(file, str) else file.decode('utf-8') try: @@ -1647,6 +1632,7 @@ def upload_tavern_character(img_path, _json): def check_tavern_character(img_path): + import gradio as gr img = open_image_safely(img_path) if img is None: @@ -1832,6 +1818,7 @@ def delete_user(name): def update_user_menu_after_deletion(idx): """Update user menu after a user is deleted""" + import gradio as gr users = get_available_users() if len(users) == 0: # Create a default user if none exist @@ -1864,93 +1851,13 @@ def handle_user_menu_change(state): def handle_save_user_click(name1): """Handle save user button click""" + import gradio as gr return [ name1, gr.update(visible=True) ] -def jinja_template_from_old_format(params, verbose=False): - MASTER_TEMPLATE = """ -{%- set ns = namespace(found=false) -%} -{%- for message in messages -%} - {%- if message['role'] == 'system' -%} - {%- set ns.found = true -%} - {%- endif -%} -{%- endfor -%} -{%- if not ns.found -%} - {{- '<|PRE-SYSTEM|>' + '<|SYSTEM-MESSAGE|>' + '<|POST-SYSTEM|>' -}} -{%- endif %} -{%- for message in messages %} - {%- if message['role'] == 'system' -%} - {{- '<|PRE-SYSTEM|>' + message['content'] + '<|POST-SYSTEM|>' -}} - {%- else -%} - {%- if message['role'] == 'user' -%} - {{-'<|PRE-USER|>' + message['content'] + '<|POST-USER|>'-}} - {%- else -%} - {{-'<|PRE-ASSISTANT|>' + message['content'] + '<|POST-ASSISTANT|>' -}} - {%- endif -%} - {%- endif -%} -{%- endfor -%} -{%- if add_generation_prompt -%} - {{-'<|PRE-ASSISTANT-GENERATE|>'-}} -{%- endif -%} -""" - - if 'context' in params and '<|system-message|>' in params['context']: - pre_system = params['context'].split('<|system-message|>')[0] - post_system = params['context'].split('<|system-message|>')[1] - else: - pre_system = '' - post_system = '' - - pre_user = params['turn_template'].split('<|user-message|>')[0].replace('<|user|>', params['user']) - post_user = params['turn_template'].split('<|user-message|>')[1].split('<|bot|>')[0] - - pre_assistant = '<|bot|>' + params['turn_template'].split('<|bot-message|>')[0].split('<|bot|>')[1] - pre_assistant = pre_assistant.replace('<|bot|>', params['bot']) - post_assistant = params['turn_template'].split('<|bot-message|>')[1] - - def preprocess(string): - return string.replace('\n', '\\n').replace('\'', '\\\'') - - pre_system = preprocess(pre_system) - post_system = preprocess(post_system) - pre_user = preprocess(pre_user) - post_user = preprocess(post_user) - pre_assistant = preprocess(pre_assistant) - post_assistant = preprocess(post_assistant) - - if verbose: - print( - '\n', - repr(pre_system) + '\n', - repr(post_system) + '\n', - repr(pre_user) + '\n', - repr(post_user) + '\n', - repr(pre_assistant) + '\n', - repr(post_assistant) + '\n', - ) - - result = MASTER_TEMPLATE - if 'system_message' in params: - result = result.replace('<|SYSTEM-MESSAGE|>', preprocess(params['system_message'])) - else: - result = result.replace('<|SYSTEM-MESSAGE|>', '') - - result = result.replace('<|PRE-SYSTEM|>', pre_system) - result = result.replace('<|POST-SYSTEM|>', post_system) - result = result.replace('<|PRE-USER|>', pre_user) - result = result.replace('<|POST-USER|>', post_user) - result = result.replace('<|PRE-ASSISTANT|>', pre_assistant) - result = result.replace('<|PRE-ASSISTANT-GENERATE|>', pre_assistant.rstrip(' ')) - result = result.replace('<|POST-ASSISTANT|>', post_assistant) - - result = result.strip() - - return result - - def my_yaml_output(data): ''' pyyaml is very inconsistent with multiline strings. @@ -2002,6 +1909,7 @@ def handle_unique_id_select(state): def handle_start_new_chat_click(state): + import gradio as gr history = start_new_chat(state) histories = find_all_histories_with_first_prompts(state) html = redraw_html(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu']) @@ -2017,6 +1925,7 @@ def handle_start_new_chat_click(state): def handle_delete_chat_confirm_click(state): + import gradio as gr filtered_histories = find_all_histories_with_first_prompts(state) filtered_ids = [h[1] for h in filtered_histories] index = str(filtered_ids.index(state['unique_id'])) @@ -2037,6 +1946,7 @@ def handle_delete_chat_confirm_click(state): def handle_branch_chat_click(state): + import gradio as gr branch_from_index = state['branch_index'] if branch_from_index == -1: history = state['history'] @@ -2148,6 +2058,7 @@ def handle_navigate_version_click(state): def handle_rename_chat_click(): + import gradio as gr return [ gr.update(value="My New Chat"), gr.update(visible=True), @@ -2155,6 +2066,7 @@ def handle_rename_chat_click(): def handle_rename_chat_confirm(rename_to, state): + import gradio as gr rename_history(state['unique_id'], rename_to, state['character_menu'], state['mode']) histories = find_all_histories_with_first_prompts(state) @@ -2165,11 +2077,13 @@ def handle_rename_chat_confirm(rename_to, state): def handle_search_chat_change(state): + import gradio as gr histories = find_all_histories_with_first_prompts(state) return gr.update(choices=histories) def handle_upload_chat_history(load_chat_history, state): + import gradio as gr history = start_new_chat(state) history = load_history_json(load_chat_history, history) save_history(history, state['unique_id'], state['character_menu'], state['mode']) @@ -2192,6 +2106,7 @@ def handle_upload_chat_history(load_chat_history, state): def handle_character_menu_change(state): + import gradio as gr name1, name2, picture, greeting, context = load_character(state['character_menu'], state['name1'], state['name2']) state['name1'] = name1 @@ -2244,6 +2159,7 @@ def handle_character_picture_change(picture_path): def handle_mode_change(state): + import gradio as gr history, loaded_unique_id = load_latest_history(state) histories = find_all_histories_with_first_prompts(state) @@ -2270,6 +2186,7 @@ def handle_mode_change(state): def handle_save_character_click(name2): + import gradio as gr return [ name2, gr.update(visible=True) @@ -2277,6 +2194,7 @@ def handle_save_character_click(name2): def handle_load_template_click(instruction_template): + from modules.models_settings import load_instruction_template output = load_instruction_template(instruction_template) return [ output, @@ -2285,6 +2203,7 @@ def handle_load_template_click(instruction_template): def handle_save_template_click(instruction_template_str): + import gradio as gr contents = generate_instruction_template_yaml(instruction_template_str) return [ "My Template.yaml", @@ -2295,6 +2214,7 @@ def handle_save_template_click(instruction_template_str): def handle_delete_template_click(template): + import gradio as gr return [ f"{template}.yaml", str(shared.user_data_dir / 'instruction-templates') + '/', @@ -2310,6 +2230,7 @@ def handle_your_picture_change(picture, state): def handle_send_instruction_click(state): + import gradio as gr state['mode'] = 'instruct' state['history'] = {'internal': [], 'visible': [], 'metadata': {}} @@ -2322,6 +2243,7 @@ def handle_send_instruction_click(state): def handle_send_chat_click(state): + import gradio as gr output = generate_chat_prompt("", state, _continue=True) if state["show_two_notebook_columns"]: diff --git a/modules/extensions.py b/modules/extensions.py index dd327882..e58a9a4c 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -6,8 +6,6 @@ from functools import partial from inspect import signature from pathlib import Path -import gradio as gr - import modules.shared as shared from modules.logging_colors import logger @@ -214,6 +212,7 @@ def _apply_custom_js(): def create_extensions_block(): + import gradio as gr to_display = [] for extension, name in iterator(): if hasattr(extension, "ui") and not (hasattr(extension, 'params') and extension.params.get('is_tab', False)): @@ -228,6 +227,7 @@ def create_extensions_block(): def create_extensions_tabs(): + import gradio as gr for extension, name in iterator(): if hasattr(extension, "ui") and (hasattr(extension, 'params') and extension.params.get('is_tab', False)): display_name = getattr(extension, 'params', {}).get('display_name', name) diff --git a/modules/loaders.py b/modules/loaders.py index 64de3dde..22ee5ed9 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -1,8 +1,6 @@ import functools from collections import OrderedDict -import gradio as gr - loaders_and_params = OrderedDict({ 'llama.cpp': [ 'gpu_layers', @@ -276,6 +274,7 @@ def list_all_samplers(): def blacklist_samplers(loader, dynamic_temperature): + import gradio as gr all_samplers = list_all_samplers() output = [] @@ -301,7 +300,61 @@ def get_all_params(): return sorted(all_params) +def list_model_elements(): + return [ + 'filter_by_loader', + 'loader', + 'cpu_memory', + 'gpu_layers', + 'fit_target', + 'cpu_moe', + 'threads', + 'threads_batch', + 'batch_size', + 'ubatch_size', + 'ctx_size', + 'cache_type', + 'tensor_split', + 'extra_flags', + 'streaming_llm', + 'gpu_split', + 'alpha_value', + 'rope_freq_base', + 'compress_pos_emb', + 'compute_dtype', + 'quant_type', + 'load_in_8bit', + 'load_in_4bit', + 'attn_implementation', + 'cpu', + 'disk', + 'row_split', + 'no_kv_offload', + 'no_mmap', + 'mlock', + 'numa', + 'parallel', + 'use_double_quant', + 'bf16', + 'enable_tp', + 'tp_backend', + 'cfg_cache', + 'no_use_fast', + 'model_draft', + 'draft_max', + 'gpu_layers_draft', + 'device_draft', + 'ctx_size_draft', + 'spec_type', + 'spec_ngram_size_n', + 'spec_ngram_size_m', + 'spec_ngram_min_hits', + 'mmproj', + ] + + def make_loader_params_visible(loader): + import gradio as gr params = [] all_params = get_all_params() if loader in loaders_and_params: diff --git a/modules/models_settings.py b/modules/models_settings.py index 472871ce..5e69b60e 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -4,10 +4,9 @@ import re from math import floor from pathlib import Path -import gradio as gr import yaml -from modules import chat, loaders, metadata_gguf, shared, ui +from modules import loaders, metadata_gguf, shared from modules.logging_colors import logger from modules.utils import resolve_model_path @@ -199,7 +198,7 @@ def get_model_metadata(model): # Load instruction template if defined by name rather than by value if model_settings['instruction_template'] != 'Custom (obtained from model metadata)': - model_settings['instruction_template_str'] = chat.load_instruction_template(model_settings['instruction_template']) + model_settings['instruction_template_str'] = load_instruction_template(model_settings['instruction_template']) return model_settings @@ -228,7 +227,7 @@ def update_model_parameters(state, initial=False): ''' UI: update the command-line arguments based on the interface values ''' - elements = ui.list_model_elements() # the names of the parameters + elements = loaders.list_model_elements() # the names of the parameters for i, element in enumerate(elements): if element not in state: @@ -248,6 +247,7 @@ def apply_model_settings_to_state(model, state): ''' UI: update the state variable with the model settings ''' + import gradio as gr model_settings = get_model_metadata(model) if 'loader' in model_settings: loader = model_settings.pop('loader') @@ -290,7 +290,7 @@ def save_model_settings(model, state): if model_regex not in user_config: user_config[model_regex] = {} - for k in ui.list_model_elements(): + for k in loaders.list_model_elements(): if k == 'loader' or k in loaders.loaders_and_params[state['loader']]: user_config[model_regex][k] = state[k] @@ -419,3 +419,102 @@ def update_gpu_layers_and_vram(loader, model, gpu_layers, ctx_size, cache_type): vram_usage = estimate_vram(model, gpu_layers, ctx_size, cache_type) return f"
Estimated VRAM to load the model: {vram_usage:.0f} MiB
" + + +def load_instruction_template(template): + if template == 'None': + return '' + + for filepath in [shared.user_data_dir / 'instruction-templates' / f'{template}.yaml', shared.user_data_dir / 'instruction-templates' / 'Alpaca.yaml']: + if filepath.exists(): + break + else: + return '' + + file_contents = open(filepath, 'r', encoding='utf-8').read() + data = yaml.safe_load(file_contents) + if 'instruction_template' in data: + return data['instruction_template'] + else: + return _jinja_template_from_old_format(data) + + +def _jinja_template_from_old_format(params, verbose=False): + MASTER_TEMPLATE = """ +{%- set ns = namespace(found=false) -%} +{%- for message in messages -%} + {%- if message['role'] == 'system' -%} + {%- set ns.found = true -%} + {%- endif -%} +{%- endfor -%} +{%- if not ns.found -%} + {{- '<|PRE-SYSTEM|>' + '<|SYSTEM-MESSAGE|>' + '<|POST-SYSTEM|>' -}} +{%- endif %} +{%- for message in messages %} + {%- if message['role'] == 'system' -%} + {{- '<|PRE-SYSTEM|>' + message['content'] + '<|POST-SYSTEM|>' -}} + {%- else -%} + {%- if message['role'] == 'user' -%} + {{-'<|PRE-USER|>' + message['content'] + '<|POST-USER|>'-}} + {%- else -%} + {{-'<|PRE-ASSISTANT|>' + message['content'] + '<|POST-ASSISTANT|>' -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{-'<|PRE-ASSISTANT-GENERATE|>'-}} +{%- endif -%} +""" + + if 'context' in params and '<|system-message|>' in params['context']: + pre_system = params['context'].split('<|system-message|>')[0] + post_system = params['context'].split('<|system-message|>')[1] + else: + pre_system = '' + post_system = '' + + pre_user = params['turn_template'].split('<|user-message|>')[0].replace('<|user|>', params['user']) + post_user = params['turn_template'].split('<|user-message|>')[1].split('<|bot|>')[0] + + pre_assistant = '<|bot|>' + params['turn_template'].split('<|bot-message|>')[0].split('<|bot|>')[1] + pre_assistant = pre_assistant.replace('<|bot|>', params['bot']) + post_assistant = params['turn_template'].split('<|bot-message|>')[1] + + def preprocess(string): + return string.replace('\n', '\\n').replace('\'', '\\\'') + + pre_system = preprocess(pre_system) + post_system = preprocess(post_system) + pre_user = preprocess(pre_user) + post_user = preprocess(post_user) + pre_assistant = preprocess(pre_assistant) + post_assistant = preprocess(post_assistant) + + if verbose: + print( + '\n', + repr(pre_system) + '\n', + repr(post_system) + '\n', + repr(pre_user) + '\n', + repr(post_user) + '\n', + repr(pre_assistant) + '\n', + repr(post_assistant) + '\n', + ) + + result = MASTER_TEMPLATE + if 'system_message' in params: + result = result.replace('<|SYSTEM-MESSAGE|>', preprocess(params['system_message'])) + else: + result = result.replace('<|SYSTEM-MESSAGE|>', '') + + result = result.replace('<|PRE-SYSTEM|>', pre_system) + result = result.replace('<|POST-SYSTEM|>', post_system) + result = result.replace('<|PRE-USER|>', pre_user) + result = result.replace('<|POST-USER|>', post_user) + result = result.replace('<|PRE-ASSISTANT|>', pre_assistant) + result = result.replace('<|PRE-ASSISTANT-GENERATE|>', pre_assistant.rstrip(' ')) + result = result.replace('<|POST-ASSISTANT|>', post_assistant) + + result = result.strip() + + return result diff --git a/modules/ui.py b/modules/ui.py index 70e929f2..abbfde49 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -120,58 +120,8 @@ else: def list_model_elements(): - elements = [ - 'filter_by_loader', - 'loader', - 'cpu_memory', - 'gpu_layers', - 'fit_target', - 'cpu_moe', - 'threads', - 'threads_batch', - 'batch_size', - 'ubatch_size', - 'ctx_size', - 'cache_type', - 'tensor_split', - 'extra_flags', - 'streaming_llm', - 'gpu_split', - 'alpha_value', - 'rope_freq_base', - 'compress_pos_emb', - 'compute_dtype', - 'quant_type', - 'load_in_8bit', - 'load_in_4bit', - 'attn_implementation', - 'cpu', - 'disk', - 'row_split', - 'no_kv_offload', - 'no_mmap', - 'mlock', - 'numa', - 'parallel', - 'use_double_quant', - 'bf16', - 'enable_tp', - 'tp_backend', - 'cfg_cache', - 'no_use_fast', - 'model_draft', - 'draft_max', - 'gpu_layers_draft', - 'device_draft', - 'ctx_size_draft', - 'spec_type', - 'spec_ngram_size_n', - 'spec_ngram_size_m', - 'spec_ngram_min_hits', - 'mmproj', - ] - - return elements + from modules.loaders import list_model_elements + return list_model_elements() def list_interface_input_elements(): diff --git a/server.py b/server.py index ff2d1db2..73f190b6 100644 --- a/server.py +++ b/server.py @@ -1,58 +1,20 @@ -import os -import shutil -import warnings -from pathlib import Path - -from modules import shared, ui # ui must be imported early to avoid circular imports -from modules.image_models import load_image_model -from modules.logging_colors import logger -from modules.prompts import load_prompt - -# Set up Gradio temp directory path -gradio_temp_path = shared.user_data_dir / 'cache' / 'gradio' -shutil.rmtree(gradio_temp_path, ignore_errors=True) -gradio_temp_path.mkdir(parents=True, exist_ok=True) - -# Set environment variables -os.environ.update({ - 'GRADIO_ANALYTICS_ENABLED': 'False', - 'BITSANDBYTES_NOWELCOME': '1', - 'GRADIO_TEMP_DIR': str(gradio_temp_path) -}) - -warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated') -warnings.filterwarnings('ignore', category=UserWarning, message='Using the update method is deprecated') -warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_name" has conflict') -warnings.filterwarnings('ignore', category=UserWarning, message='The value passed into gr.Dropdown()') -warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_names" has conflict') - -import gradio as gr - import os import signal import sys import time +import warnings from functools import partial +from pathlib import Path from threading import Lock, Thread import yaml +from modules import shared, utils +from modules.image_models import load_image_model +from modules.logging_colors import logger +from modules.prompts import load_prompt + import modules.extensions as extensions_module -from modules import ( - training, - ui, - ui_chat, - ui_default, - ui_file_saving, - ui_image_generation, - ui_model_menu, - ui_notebook, - ui_parameters, - ui_session, - utils -) -from modules.chat import generate_pfp_cache -from modules.extensions import apply_extensions from modules.LoRA import add_lora_to_model from modules.models import load_model, unload_model_if_idle from modules.models_settings import ( @@ -61,7 +23,13 @@ from modules.models_settings import ( update_model_parameters ) from modules.shared import do_cmd_flags_warnings -from modules.utils import gradio + +os.environ['BITSANDBYTES_NOWELCOME'] = '1' + +warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated') +warnings.filterwarnings('ignore', category=UserWarning, message='Using the update method is deprecated') +warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_name" has conflict') +warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_names" has conflict') def signal_handler(sig, frame): @@ -83,6 +51,37 @@ signal.signal(signal.SIGTERM, signal_handler) def create_interface(): + import shutil + + import gradio as gr + + from modules import ( + training, + ui, + ui_chat, + ui_default, + ui_file_saving, + ui_image_generation, + ui_model_menu, + ui_notebook, + ui_parameters, + ui_session, + ) + from modules.chat import generate_pfp_cache + from modules.extensions import apply_extensions + from modules.utils import gradio + + warnings.filterwarnings('ignore', category=UserWarning, message='The value passed into gr.Dropdown()') + + # Set up Gradio temp directory path + gradio_temp_path = shared.user_data_dir / 'cache' / 'gradio' + shutil.rmtree(gradio_temp_path, ignore_errors=True) + gradio_temp_path.mkdir(parents=True, exist_ok=True) + os.environ.update({ + 'GRADIO_ANALYTICS_ENABLED': 'False', + 'GRADIO_TEMP_DIR': str(gradio_temp_path) + }) + title = 'Text Generation Web UI' # Password authentication