Remove specialized code for gpt-4chan

modules/html_generator.py

@@ -16,8 +16,6 @@ image_cache = {}
 
 with open(Path(__file__).resolve().parent / '../css/html_readable_style.css', 'r') as f:
     readable_css = f.read()
-with open(Path(__file__).resolve().parent / '../css/html_4chan_style.css', 'r') as css_f:
-    _4chan_css = css_f.read()
 with open(Path(__file__).resolve().parent / '../css/html_instruct_style.css', 'r') as f:
     instruct_css = f.read()
 
@@ -118,63 +116,6 @@ def generate_basic_html(string):
     return string
 
 
-def process_post(post, c):
-    t = post.split('\n')
-    number = t[0].split(' ')[1]
-    if len(t) > 1:
-        src = '\n'.join(t[1:])
-    else:
-        src = ''
-    src = re.sub('>', '>', src)
-    src = re.sub('(&gt;&gt;[0-9]*)', '<span class="quote">\\1</span>', src)
-    src = re.sub('\n', '<br>\n', src)
-    src = f'<blockquote class="message_4chan">{src}\n'
-    src = f'<span class="name">Anonymous </span> <span class="number">No.{number}</span>\n{src}'
-    return src
-
-
-def generate_4chan_html(f):
-    posts = []
-    post = ''
-    c = -2
-    for line in f.splitlines():
-        line += "\n"
-        if line == '-----\n':
-            continue
-        elif line.startswith('--- '):
-            c += 1
-            if post != '':
-                src = process_post(post, c)
-                posts.append(src)
-            post = line
-        else:
-            post += line
-
-    if post != '':
-        src = process_post(post, c)
-        posts.append(src)
-
-    for i in range(len(posts)):
-        if i == 0:
-            posts[i] = f'<div class="op">{posts[i]}</div>\n'
-        else:
-            posts[i] = f'<div class="reply">{posts[i]}</div>\n'
-
-    output = ''
-    output += f'<style>{_4chan_css}</style><div id="parent"><div id="container">'
-    for post in posts:
-        output += post
-
-    output += '</div></div>'
-    output = output.split('\n')
-    for i in range(len(output)):
-        output[i] = re.sub(r'^(&gt;(.*?)(<br>|</div>))', r'<span class="greentext">\1</span>', output[i])
-        output[i] = re.sub(r'^<blockquote class="message_4chan">(&gt;(.*?)(<br>|</div>))', r'<blockquote class="message_4chan"><span class="greentext">\1</span>', output[i])
-
-    output = '\n'.join(output)
-    return output
-
-
 def make_thumbnail(image):
     image = image.resize((350, round(image.size[1] / image.size[0] * 350)), Image.Resampling.LANCZOS)
     if image.size[1] > 470:

modules/models.py

@@ -110,9 +110,7 @@ def load_model(model_name, loader=None):
 def load_tokenizer(model_name, model):
     tokenizer = None
     path_to_model = Path(f"{shared.args.model_dir}/{model_name}/")
-    if any(s in model_name.lower() for s in ['gpt-4chan', 'gpt4chan']) and Path(f"{shared.args.model_dir}/gpt-j-6B/").exists():
-        tokenizer = AutoTokenizer.from_pretrained(Path(f"{shared.args.model_dir}/gpt-j-6B/"))
-    elif path_to_model.exists():
+    if path_to_model.exists():
         if shared.args.no_use_fast:
             logger.info('Loading the tokenizer with use_fast=False.')
 

modules/text_generation.py

@@ -22,7 +22,7 @@ from modules.callbacks import (
 from modules.extensions import apply_extensions
 from modules.grammar.grammar_utils import initialize_grammar
 from modules.grammar.logits_process import GrammarConstrainedLogitsProcessor
-from modules.html_generator import generate_4chan_html, generate_basic_html
+from modules.html_generator import generate_basic_html
 from modules.logging_colors import logger
 from modules.models import clear_torch_cache, local_rank
 
@@ -186,23 +186,7 @@ def generate_reply_wrapper(question, state, stopping_strings=None):
 
 
 def formatted_outputs(reply, model_name):
-    if any(s in model_name for s in ['gpt-4chan', 'gpt4chan']):
-        reply = fix_gpt4chan(reply)
-        return html.unescape(reply), generate_4chan_html(reply)
-    else:
-        return html.unescape(reply), generate_basic_html(reply)
-
-
-def fix_gpt4chan(s):
-    """
-    Removes empty replies from gpt4chan outputs
-    """
-    for i in range(10):
-        s = re.sub("--- [0-9]*\n>>[0-9]*\n---", "---", s)
-        s = re.sub("--- [0-9]*\n *\n---", "---", s)
-        s = re.sub("--- [0-9]*\n\n\n---", "---", s)
-
-    return s
+    return html.unescape(reply), generate_basic_html(reply)
 
 
 def fix_galactica(s):