mirror of
https://github.com/oobabooga/text-generation-webui.git
synced 2026-03-17 19:04:39 +01:00
Refactor to not import gradio in --nowebui mode
This commit is contained in:
oobabooga
parent
970055ca00
commit
39e6c997cc
|
|
@ -1,4 +1,4 @@
|
|||
from modules import shared, ui
|
||||
from modules import loaders, shared
|
||||
from modules.logging_colors import logger
|
||||
from modules.LoRA import add_lora_to_model
|
||||
from modules.models import load_model, unload_model
|
||||
|
|
@ -50,7 +50,7 @@ def _load_model(data):
|
|||
# parameters exposed in the UI. Never allow security-sensitive
|
||||
# flags like trust_remote_code or extra_flags to be set via the API.
|
||||
blocked_keys = {'extra_flags'}
|
||||
allowed_keys = set(ui.list_model_elements()) - blocked_keys
|
||||
allowed_keys = set(loaders.list_model_elements()) - blocked_keys
|
||||
if args:
|
||||
for k in args:
|
||||
if k in allowed_keys and hasattr(shared.args, k):
|
||||
|
|
|
|||
122
modules/chat.py
122
modules/chat.py
|
|
@ -11,7 +11,6 @@ from datetime import datetime
|
|||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import gradio as gr
|
||||
import yaml
|
||||
from jinja2.ext import loopcontrols
|
||||
from jinja2.sandbox import ImmutableSandboxedEnvironment
|
||||
|
|
@ -1333,6 +1332,7 @@ def load_history_after_deletion(state, idx):
|
|||
Loads the latest history for the given character in chat or chat-instruct
|
||||
mode, or the latest instruct history for instruct mode.
|
||||
'''
|
||||
import gradio as gr
|
||||
|
||||
if shared.args.multi_user:
|
||||
return start_new_chat(state)
|
||||
|
|
@ -1351,6 +1351,7 @@ def load_history_after_deletion(state, idx):
|
|||
|
||||
|
||||
def update_character_menu_after_deletion(idx):
|
||||
import gradio as gr
|
||||
characters = utils.get_available_characters()
|
||||
idx = min(int(idx), len(characters) - 1)
|
||||
idx = max(0, idx)
|
||||
|
|
@ -1565,24 +1566,6 @@ def clear_character_for_ui(state):
|
|||
return state, state['name2'], state['context'], state['greeting'], None
|
||||
|
||||
|
||||
def load_instruction_template(template):
|
||||
if template == 'None':
|
||||
return ''
|
||||
|
||||
for filepath in [shared.user_data_dir / 'instruction-templates' / f'{template}.yaml', shared.user_data_dir / 'instruction-templates' / 'Alpaca.yaml']:
|
||||
if filepath.exists():
|
||||
break
|
||||
else:
|
||||
return ''
|
||||
|
||||
file_contents = open(filepath, 'r', encoding='utf-8').read()
|
||||
data = yaml.safe_load(file_contents)
|
||||
if 'instruction_template' in data:
|
||||
return data['instruction_template']
|
||||
else:
|
||||
return jinja_template_from_old_format(data)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def load_character_memoized(character, name1, name2):
|
||||
return load_character(character, name1, name2)
|
||||
|
|
@ -1590,10 +1573,12 @@ def load_character_memoized(character, name1, name2):
|
|||
|
||||
@functools.cache
|
||||
def load_instruction_template_memoized(template):
|
||||
from modules.models_settings import load_instruction_template
|
||||
return load_instruction_template(template)
|
||||
|
||||
|
||||
def upload_character(file, img_path, tavern=False):
|
||||
import gradio as gr
|
||||
img = open_image_safely(img_path)
|
||||
decoded_file = file if isinstance(file, str) else file.decode('utf-8')
|
||||
try:
|
||||
|
|
@ -1647,6 +1632,7 @@ def upload_tavern_character(img_path, _json):
|
|||
|
||||
|
||||
def check_tavern_character(img_path):
|
||||
import gradio as gr
|
||||
img = open_image_safely(img_path)
|
||||
|
||||
if img is None:
|
||||
|
|
@ -1832,6 +1818,7 @@ def delete_user(name):
|
|||
|
||||
def update_user_menu_after_deletion(idx):
|
||||
"""Update user menu after a user is deleted"""
|
||||
import gradio as gr
|
||||
users = get_available_users()
|
||||
if len(users) == 0:
|
||||
# Create a default user if none exist
|
||||
|
|
@ -1864,93 +1851,13 @@ def handle_user_menu_change(state):
|
|||
|
||||
def handle_save_user_click(name1):
|
||||
"""Handle save user button click"""
|
||||
import gradio as gr
|
||||
return [
|
||||
name1,
|
||||
gr.update(visible=True)
|
||||
]
|
||||
|
||||
|
||||
def jinja_template_from_old_format(params, verbose=False):
|
||||
MASTER_TEMPLATE = """
|
||||
{%- set ns = namespace(found=false) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'system' -%}
|
||||
{%- set ns.found = true -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- if not ns.found -%}
|
||||
{{- '<|PRE-SYSTEM|>' + '<|SYSTEM-MESSAGE|>' + '<|POST-SYSTEM|>' -}}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'system' -%}
|
||||
{{- '<|PRE-SYSTEM|>' + message['content'] + '<|POST-SYSTEM|>' -}}
|
||||
{%- else -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{-'<|PRE-USER|>' + message['content'] + '<|POST-USER|>'-}}
|
||||
{%- else -%}
|
||||
{{-'<|PRE-ASSISTANT|>' + message['content'] + '<|POST-ASSISTANT|>' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{-'<|PRE-ASSISTANT-GENERATE|>'-}}
|
||||
{%- endif -%}
|
||||
"""
|
||||
|
||||
if 'context' in params and '<|system-message|>' in params['context']:
|
||||
pre_system = params['context'].split('<|system-message|>')[0]
|
||||
post_system = params['context'].split('<|system-message|>')[1]
|
||||
else:
|
||||
pre_system = ''
|
||||
post_system = ''
|
||||
|
||||
pre_user = params['turn_template'].split('<|user-message|>')[0].replace('<|user|>', params['user'])
|
||||
post_user = params['turn_template'].split('<|user-message|>')[1].split('<|bot|>')[0]
|
||||
|
||||
pre_assistant = '<|bot|>' + params['turn_template'].split('<|bot-message|>')[0].split('<|bot|>')[1]
|
||||
pre_assistant = pre_assistant.replace('<|bot|>', params['bot'])
|
||||
post_assistant = params['turn_template'].split('<|bot-message|>')[1]
|
||||
|
||||
def preprocess(string):
|
||||
return string.replace('\n', '\\n').replace('\'', '\\\'')
|
||||
|
||||
pre_system = preprocess(pre_system)
|
||||
post_system = preprocess(post_system)
|
||||
pre_user = preprocess(pre_user)
|
||||
post_user = preprocess(post_user)
|
||||
pre_assistant = preprocess(pre_assistant)
|
||||
post_assistant = preprocess(post_assistant)
|
||||
|
||||
if verbose:
|
||||
print(
|
||||
'\n',
|
||||
repr(pre_system) + '\n',
|
||||
repr(post_system) + '\n',
|
||||
repr(pre_user) + '\n',
|
||||
repr(post_user) + '\n',
|
||||
repr(pre_assistant) + '\n',
|
||||
repr(post_assistant) + '\n',
|
||||
)
|
||||
|
||||
result = MASTER_TEMPLATE
|
||||
if 'system_message' in params:
|
||||
result = result.replace('<|SYSTEM-MESSAGE|>', preprocess(params['system_message']))
|
||||
else:
|
||||
result = result.replace('<|SYSTEM-MESSAGE|>', '')
|
||||
|
||||
result = result.replace('<|PRE-SYSTEM|>', pre_system)
|
||||
result = result.replace('<|POST-SYSTEM|>', post_system)
|
||||
result = result.replace('<|PRE-USER|>', pre_user)
|
||||
result = result.replace('<|POST-USER|>', post_user)
|
||||
result = result.replace('<|PRE-ASSISTANT|>', pre_assistant)
|
||||
result = result.replace('<|PRE-ASSISTANT-GENERATE|>', pre_assistant.rstrip(' '))
|
||||
result = result.replace('<|POST-ASSISTANT|>', post_assistant)
|
||||
|
||||
result = result.strip()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def my_yaml_output(data):
|
||||
'''
|
||||
pyyaml is very inconsistent with multiline strings.
|
||||
|
|
@ -2002,6 +1909,7 @@ def handle_unique_id_select(state):
|
|||
|
||||
|
||||
def handle_start_new_chat_click(state):
|
||||
import gradio as gr
|
||||
history = start_new_chat(state)
|
||||
histories = find_all_histories_with_first_prompts(state)
|
||||
html = redraw_html(history, state['name1'], state['name2'], state['mode'], state['chat_style'], state['character_menu'])
|
||||
|
|
@ -2017,6 +1925,7 @@ def handle_start_new_chat_click(state):
|
|||
|
||||
|
||||
def handle_delete_chat_confirm_click(state):
|
||||
import gradio as gr
|
||||
filtered_histories = find_all_histories_with_first_prompts(state)
|
||||
filtered_ids = [h[1] for h in filtered_histories]
|
||||
index = str(filtered_ids.index(state['unique_id']))
|
||||
|
|
@ -2037,6 +1946,7 @@ def handle_delete_chat_confirm_click(state):
|
|||
|
||||
|
||||
def handle_branch_chat_click(state):
|
||||
import gradio as gr
|
||||
branch_from_index = state['branch_index']
|
||||
if branch_from_index == -1:
|
||||
history = state['history']
|
||||
|
|
@ -2148,6 +2058,7 @@ def handle_navigate_version_click(state):
|
|||
|
||||
|
||||
def handle_rename_chat_click():
|
||||
import gradio as gr
|
||||
return [
|
||||
gr.update(value="My New Chat"),
|
||||
gr.update(visible=True),
|
||||
|
|
@ -2155,6 +2066,7 @@ def handle_rename_chat_click():
|
|||
|
||||
|
||||
def handle_rename_chat_confirm(rename_to, state):
|
||||
import gradio as gr
|
||||
rename_history(state['unique_id'], rename_to, state['character_menu'], state['mode'])
|
||||
histories = find_all_histories_with_first_prompts(state)
|
||||
|
||||
|
|
@ -2165,11 +2077,13 @@ def handle_rename_chat_confirm(rename_to, state):
|
|||
|
||||
|
||||
def handle_search_chat_change(state):
|
||||
import gradio as gr
|
||||
histories = find_all_histories_with_first_prompts(state)
|
||||
return gr.update(choices=histories)
|
||||
|
||||
|
||||
def handle_upload_chat_history(load_chat_history, state):
|
||||
import gradio as gr
|
||||
history = start_new_chat(state)
|
||||
history = load_history_json(load_chat_history, history)
|
||||
save_history(history, state['unique_id'], state['character_menu'], state['mode'])
|
||||
|
|
@ -2192,6 +2106,7 @@ def handle_upload_chat_history(load_chat_history, state):
|
|||
|
||||
|
||||
def handle_character_menu_change(state):
|
||||
import gradio as gr
|
||||
name1, name2, picture, greeting, context = load_character(state['character_menu'], state['name1'], state['name2'])
|
||||
|
||||
state['name1'] = name1
|
||||
|
|
@ -2244,6 +2159,7 @@ def handle_character_picture_change(picture_path):
|
|||
|
||||
|
||||
def handle_mode_change(state):
|
||||
import gradio as gr
|
||||
history, loaded_unique_id = load_latest_history(state)
|
||||
histories = find_all_histories_with_first_prompts(state)
|
||||
|
||||
|
|
@ -2270,6 +2186,7 @@ def handle_mode_change(state):
|
|||
|
||||
|
||||
def handle_save_character_click(name2):
|
||||
import gradio as gr
|
||||
return [
|
||||
name2,
|
||||
gr.update(visible=True)
|
||||
|
|
@ -2277,6 +2194,7 @@ def handle_save_character_click(name2):
|
|||
|
||||
|
||||
def handle_load_template_click(instruction_template):
|
||||
from modules.models_settings import load_instruction_template
|
||||
output = load_instruction_template(instruction_template)
|
||||
return [
|
||||
output,
|
||||
|
|
@ -2285,6 +2203,7 @@ def handle_load_template_click(instruction_template):
|
|||
|
||||
|
||||
def handle_save_template_click(instruction_template_str):
|
||||
import gradio as gr
|
||||
contents = generate_instruction_template_yaml(instruction_template_str)
|
||||
return [
|
||||
"My Template.yaml",
|
||||
|
|
@ -2295,6 +2214,7 @@ def handle_save_template_click(instruction_template_str):
|
|||
|
||||
|
||||
def handle_delete_template_click(template):
|
||||
import gradio as gr
|
||||
return [
|
||||
f"{template}.yaml",
|
||||
str(shared.user_data_dir / 'instruction-templates') + '/',
|
||||
|
|
@ -2310,6 +2230,7 @@ def handle_your_picture_change(picture, state):
|
|||
|
||||
|
||||
def handle_send_instruction_click(state):
|
||||
import gradio as gr
|
||||
state['mode'] = 'instruct'
|
||||
state['history'] = {'internal': [], 'visible': [], 'metadata': {}}
|
||||
|
||||
|
|
@ -2322,6 +2243,7 @@ def handle_send_instruction_click(state):
|
|||
|
||||
|
||||
def handle_send_chat_click(state):
|
||||
import gradio as gr
|
||||
output = generate_chat_prompt("", state, _continue=True)
|
||||
|
||||
if state["show_two_notebook_columns"]:
|
||||
|
|
|
|||
|
|
@ -6,8 +6,6 @@ from functools import partial
|
|||
from inspect import signature
|
||||
from pathlib import Path
|
||||
|
||||
import gradio as gr
|
||||
|
||||
import modules.shared as shared
|
||||
from modules.logging_colors import logger
|
||||
|
||||
|
|
@ -214,6 +212,7 @@ def _apply_custom_js():
|
|||
|
||||
|
||||
def create_extensions_block():
|
||||
import gradio as gr
|
||||
to_display = []
|
||||
for extension, name in iterator():
|
||||
if hasattr(extension, "ui") and not (hasattr(extension, 'params') and extension.params.get('is_tab', False)):
|
||||
|
|
@ -228,6 +227,7 @@ def create_extensions_block():
|
|||
|
||||
|
||||
def create_extensions_tabs():
|
||||
import gradio as gr
|
||||
for extension, name in iterator():
|
||||
if hasattr(extension, "ui") and (hasattr(extension, 'params') and extension.params.get('is_tab', False)):
|
||||
display_name = getattr(extension, 'params', {}).get('display_name', name)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import functools
|
||||
from collections import OrderedDict
|
||||
|
||||
import gradio as gr
|
||||
|
||||
loaders_and_params = OrderedDict({
|
||||
'llama.cpp': [
|
||||
'gpu_layers',
|
||||
|
|
@ -276,6 +274,7 @@ def list_all_samplers():
|
|||
|
||||
|
||||
def blacklist_samplers(loader, dynamic_temperature):
|
||||
import gradio as gr
|
||||
all_samplers = list_all_samplers()
|
||||
output = []
|
||||
|
||||
|
|
@ -301,7 +300,61 @@ def get_all_params():
|
|||
return sorted(all_params)
|
||||
|
||||
|
||||
def list_model_elements():
|
||||
return [
|
||||
'filter_by_loader',
|
||||
'loader',
|
||||
'cpu_memory',
|
||||
'gpu_layers',
|
||||
'fit_target',
|
||||
'cpu_moe',
|
||||
'threads',
|
||||
'threads_batch',
|
||||
'batch_size',
|
||||
'ubatch_size',
|
||||
'ctx_size',
|
||||
'cache_type',
|
||||
'tensor_split',
|
||||
'extra_flags',
|
||||
'streaming_llm',
|
||||
'gpu_split',
|
||||
'alpha_value',
|
||||
'rope_freq_base',
|
||||
'compress_pos_emb',
|
||||
'compute_dtype',
|
||||
'quant_type',
|
||||
'load_in_8bit',
|
||||
'load_in_4bit',
|
||||
'attn_implementation',
|
||||
'cpu',
|
||||
'disk',
|
||||
'row_split',
|
||||
'no_kv_offload',
|
||||
'no_mmap',
|
||||
'mlock',
|
||||
'numa',
|
||||
'parallel',
|
||||
'use_double_quant',
|
||||
'bf16',
|
||||
'enable_tp',
|
||||
'tp_backend',
|
||||
'cfg_cache',
|
||||
'no_use_fast',
|
||||
'model_draft',
|
||||
'draft_max',
|
||||
'gpu_layers_draft',
|
||||
'device_draft',
|
||||
'ctx_size_draft',
|
||||
'spec_type',
|
||||
'spec_ngram_size_n',
|
||||
'spec_ngram_size_m',
|
||||
'spec_ngram_min_hits',
|
||||
'mmproj',
|
||||
]
|
||||
|
||||
|
||||
def make_loader_params_visible(loader):
|
||||
import gradio as gr
|
||||
params = []
|
||||
all_params = get_all_params()
|
||||
if loader in loaders_and_params:
|
||||
|
|
|
|||
|
|
@ -4,10 +4,9 @@ import re
|
|||
from math import floor
|
||||
from pathlib import Path
|
||||
|
||||
import gradio as gr
|
||||
import yaml
|
||||
|
||||
from modules import chat, loaders, metadata_gguf, shared, ui
|
||||
from modules import loaders, metadata_gguf, shared
|
||||
from modules.logging_colors import logger
|
||||
from modules.utils import resolve_model_path
|
||||
|
||||
|
|
@ -199,7 +198,7 @@ def get_model_metadata(model):
|
|||
|
||||
# Load instruction template if defined by name rather than by value
|
||||
if model_settings['instruction_template'] != 'Custom (obtained from model metadata)':
|
||||
model_settings['instruction_template_str'] = chat.load_instruction_template(model_settings['instruction_template'])
|
||||
model_settings['instruction_template_str'] = load_instruction_template(model_settings['instruction_template'])
|
||||
|
||||
return model_settings
|
||||
|
||||
|
|
@ -228,7 +227,7 @@ def update_model_parameters(state, initial=False):
|
|||
'''
|
||||
UI: update the command-line arguments based on the interface values
|
||||
'''
|
||||
elements = ui.list_model_elements() # the names of the parameters
|
||||
elements = loaders.list_model_elements() # the names of the parameters
|
||||
|
||||
for i, element in enumerate(elements):
|
||||
if element not in state:
|
||||
|
|
@ -248,6 +247,7 @@ def apply_model_settings_to_state(model, state):
|
|||
'''
|
||||
UI: update the state variable with the model settings
|
||||
'''
|
||||
import gradio as gr
|
||||
model_settings = get_model_metadata(model)
|
||||
if 'loader' in model_settings:
|
||||
loader = model_settings.pop('loader')
|
||||
|
|
@ -290,7 +290,7 @@ def save_model_settings(model, state):
|
|||
if model_regex not in user_config:
|
||||
user_config[model_regex] = {}
|
||||
|
||||
for k in ui.list_model_elements():
|
||||
for k in loaders.list_model_elements():
|
||||
if k == 'loader' or k in loaders.loaders_and_params[state['loader']]:
|
||||
user_config[model_regex][k] = state[k]
|
||||
|
||||
|
|
@ -419,3 +419,102 @@ def update_gpu_layers_and_vram(loader, model, gpu_layers, ctx_size, cache_type):
|
|||
|
||||
vram_usage = estimate_vram(model, gpu_layers, ctx_size, cache_type)
|
||||
return f"<div id=\"vram-info\"'>Estimated VRAM to load the model: <span class=\"value\">{vram_usage:.0f} MiB</span></div>"
|
||||
|
||||
|
||||
def load_instruction_template(template):
|
||||
if template == 'None':
|
||||
return ''
|
||||
|
||||
for filepath in [shared.user_data_dir / 'instruction-templates' / f'{template}.yaml', shared.user_data_dir / 'instruction-templates' / 'Alpaca.yaml']:
|
||||
if filepath.exists():
|
||||
break
|
||||
else:
|
||||
return ''
|
||||
|
||||
file_contents = open(filepath, 'r', encoding='utf-8').read()
|
||||
data = yaml.safe_load(file_contents)
|
||||
if 'instruction_template' in data:
|
||||
return data['instruction_template']
|
||||
else:
|
||||
return _jinja_template_from_old_format(data)
|
||||
|
||||
|
||||
def _jinja_template_from_old_format(params, verbose=False):
|
||||
MASTER_TEMPLATE = """
|
||||
{%- set ns = namespace(found=false) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message['role'] == 'system' -%}
|
||||
{%- set ns.found = true -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- if not ns.found -%}
|
||||
{{- '<|PRE-SYSTEM|>' + '<|SYSTEM-MESSAGE|>' + '<|POST-SYSTEM|>' -}}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if message['role'] == 'system' -%}
|
||||
{{- '<|PRE-SYSTEM|>' + message['content'] + '<|POST-SYSTEM|>' -}}
|
||||
{%- else -%}
|
||||
{%- if message['role'] == 'user' -%}
|
||||
{{-'<|PRE-USER|>' + message['content'] + '<|POST-USER|>'-}}
|
||||
{%- else -%}
|
||||
{{-'<|PRE-ASSISTANT|>' + message['content'] + '<|POST-ASSISTANT|>' -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{-'<|PRE-ASSISTANT-GENERATE|>'-}}
|
||||
{%- endif -%}
|
||||
"""
|
||||
|
||||
if 'context' in params and '<|system-message|>' in params['context']:
|
||||
pre_system = params['context'].split('<|system-message|>')[0]
|
||||
post_system = params['context'].split('<|system-message|>')[1]
|
||||
else:
|
||||
pre_system = ''
|
||||
post_system = ''
|
||||
|
||||
pre_user = params['turn_template'].split('<|user-message|>')[0].replace('<|user|>', params['user'])
|
||||
post_user = params['turn_template'].split('<|user-message|>')[1].split('<|bot|>')[0]
|
||||
|
||||
pre_assistant = '<|bot|>' + params['turn_template'].split('<|bot-message|>')[0].split('<|bot|>')[1]
|
||||
pre_assistant = pre_assistant.replace('<|bot|>', params['bot'])
|
||||
post_assistant = params['turn_template'].split('<|bot-message|>')[1]
|
||||
|
||||
def preprocess(string):
|
||||
return string.replace('\n', '\\n').replace('\'', '\\\'')
|
||||
|
||||
pre_system = preprocess(pre_system)
|
||||
post_system = preprocess(post_system)
|
||||
pre_user = preprocess(pre_user)
|
||||
post_user = preprocess(post_user)
|
||||
pre_assistant = preprocess(pre_assistant)
|
||||
post_assistant = preprocess(post_assistant)
|
||||
|
||||
if verbose:
|
||||
print(
|
||||
'\n',
|
||||
repr(pre_system) + '\n',
|
||||
repr(post_system) + '\n',
|
||||
repr(pre_user) + '\n',
|
||||
repr(post_user) + '\n',
|
||||
repr(pre_assistant) + '\n',
|
||||
repr(post_assistant) + '\n',
|
||||
)
|
||||
|
||||
result = MASTER_TEMPLATE
|
||||
if 'system_message' in params:
|
||||
result = result.replace('<|SYSTEM-MESSAGE|>', preprocess(params['system_message']))
|
||||
else:
|
||||
result = result.replace('<|SYSTEM-MESSAGE|>', '')
|
||||
|
||||
result = result.replace('<|PRE-SYSTEM|>', pre_system)
|
||||
result = result.replace('<|POST-SYSTEM|>', post_system)
|
||||
result = result.replace('<|PRE-USER|>', pre_user)
|
||||
result = result.replace('<|POST-USER|>', post_user)
|
||||
result = result.replace('<|PRE-ASSISTANT|>', pre_assistant)
|
||||
result = result.replace('<|PRE-ASSISTANT-GENERATE|>', pre_assistant.rstrip(' '))
|
||||
result = result.replace('<|POST-ASSISTANT|>', post_assistant)
|
||||
|
||||
result = result.strip()
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -120,58 +120,8 @@ else:
|
|||
|
||||
|
||||
def list_model_elements():
|
||||
elements = [
|
||||
'filter_by_loader',
|
||||
'loader',
|
||||
'cpu_memory',
|
||||
'gpu_layers',
|
||||
'fit_target',
|
||||
'cpu_moe',
|
||||
'threads',
|
||||
'threads_batch',
|
||||
'batch_size',
|
||||
'ubatch_size',
|
||||
'ctx_size',
|
||||
'cache_type',
|
||||
'tensor_split',
|
||||
'extra_flags',
|
||||
'streaming_llm',
|
||||
'gpu_split',
|
||||
'alpha_value',
|
||||
'rope_freq_base',
|
||||
'compress_pos_emb',
|
||||
'compute_dtype',
|
||||
'quant_type',
|
||||
'load_in_8bit',
|
||||
'load_in_4bit',
|
||||
'attn_implementation',
|
||||
'cpu',
|
||||
'disk',
|
||||
'row_split',
|
||||
'no_kv_offload',
|
||||
'no_mmap',
|
||||
'mlock',
|
||||
'numa',
|
||||
'parallel',
|
||||
'use_double_quant',
|
||||
'bf16',
|
||||
'enable_tp',
|
||||
'tp_backend',
|
||||
'cfg_cache',
|
||||
'no_use_fast',
|
||||
'model_draft',
|
||||
'draft_max',
|
||||
'gpu_layers_draft',
|
||||
'device_draft',
|
||||
'ctx_size_draft',
|
||||
'spec_type',
|
||||
'spec_ngram_size_n',
|
||||
'spec_ngram_size_m',
|
||||
'spec_ngram_min_hits',
|
||||
'mmproj',
|
||||
]
|
||||
|
||||
return elements
|
||||
from modules.loaders import list_model_elements
|
||||
return list_model_elements()
|
||||
|
||||
|
||||
def list_interface_input_elements():
|
||||
|
|
|
|||
91
server.py
91
server.py
|
|
@ -1,58 +1,20 @@
|
|||
import os
|
||||
import shutil
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
|
||||
from modules import shared, ui # ui must be imported early to avoid circular imports
|
||||
from modules.image_models import load_image_model
|
||||
from modules.logging_colors import logger
|
||||
from modules.prompts import load_prompt
|
||||
|
||||
# Set up Gradio temp directory path
|
||||
gradio_temp_path = shared.user_data_dir / 'cache' / 'gradio'
|
||||
shutil.rmtree(gradio_temp_path, ignore_errors=True)
|
||||
gradio_temp_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Set environment variables
|
||||
os.environ.update({
|
||||
'GRADIO_ANALYTICS_ENABLED': 'False',
|
||||
'BITSANDBYTES_NOWELCOME': '1',
|
||||
'GRADIO_TEMP_DIR': str(gradio_temp_path)
|
||||
})
|
||||
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='Using the update method is deprecated')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_name" has conflict')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='The value passed into gr.Dropdown()')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_names" has conflict')
|
||||
|
||||
import gradio as gr
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import warnings
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from threading import Lock, Thread
|
||||
|
||||
import yaml
|
||||
|
||||
from modules import shared, utils
|
||||
from modules.image_models import load_image_model
|
||||
from modules.logging_colors import logger
|
||||
from modules.prompts import load_prompt
|
||||
|
||||
import modules.extensions as extensions_module
|
||||
from modules import (
|
||||
training,
|
||||
ui,
|
||||
ui_chat,
|
||||
ui_default,
|
||||
ui_file_saving,
|
||||
ui_image_generation,
|
||||
ui_model_menu,
|
||||
ui_notebook,
|
||||
ui_parameters,
|
||||
ui_session,
|
||||
utils
|
||||
)
|
||||
from modules.chat import generate_pfp_cache
|
||||
from modules.extensions import apply_extensions
|
||||
from modules.LoRA import add_lora_to_model
|
||||
from modules.models import load_model, unload_model_if_idle
|
||||
from modules.models_settings import (
|
||||
|
|
@ -61,7 +23,13 @@ from modules.models_settings import (
|
|||
update_model_parameters
|
||||
)
|
||||
from modules.shared import do_cmd_flags_warnings
|
||||
from modules.utils import gradio
|
||||
|
||||
os.environ['BITSANDBYTES_NOWELCOME'] = '1'
|
||||
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='TypedStorage is deprecated')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='Using the update method is deprecated')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_name" has conflict')
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='Field "model_names" has conflict')
|
||||
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
|
|
@ -83,6 +51,37 @@ signal.signal(signal.SIGTERM, signal_handler)
|
|||
|
||||
def create_interface():
|
||||
|
||||
import shutil
|
||||
|
||||
import gradio as gr
|
||||
|
||||
from modules import (
|
||||
training,
|
||||
ui,
|
||||
ui_chat,
|
||||
ui_default,
|
||||
ui_file_saving,
|
||||
ui_image_generation,
|
||||
ui_model_menu,
|
||||
ui_notebook,
|
||||
ui_parameters,
|
||||
ui_session,
|
||||
)
|
||||
from modules.chat import generate_pfp_cache
|
||||
from modules.extensions import apply_extensions
|
||||
from modules.utils import gradio
|
||||
|
||||
warnings.filterwarnings('ignore', category=UserWarning, message='The value passed into gr.Dropdown()')
|
||||
|
||||
# Set up Gradio temp directory path
|
||||
gradio_temp_path = shared.user_data_dir / 'cache' / 'gradio'
|
||||
shutil.rmtree(gradio_temp_path, ignore_errors=True)
|
||||
gradio_temp_path.mkdir(parents=True, exist_ok=True)
|
||||
os.environ.update({
|
||||
'GRADIO_ANALYTICS_ENABLED': 'False',
|
||||
'GRADIO_TEMP_DIR': str(gradio_temp_path)
|
||||
})
|
||||
|
||||
title = 'Text Generation Web UI'
|
||||
|
||||
# Password authentication
|
||||
|
|
|
|||
Loading…
Reference in a new issue