oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-11 09:33:40 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add back my llama-cpp-python wheels, bump to 0.2.65 (#5964)

Browse source
This commit is contained in:
oobabooga 2024-04-30 09:11:31 -03:00 • committed by GitHub
parent 5770e06c48
commit 51fb766bea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 330 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

18
modules/llama_cpp_python_hijack.py
View file

@ -1,11 +1,25 @@
from typing import Sequence
import llama_cpp
from tqdm import tqdm
from modules import shared
from modules.cache_utils import process_llamacpp_cache
try:
import llama_cpp
except:
llama_cpp = None
try:
import llama_cpp_cuda
except:
llama_cpp_cuda = None
try:
import llama_cpp_cuda_tensorcores
except:
llama_cpp_cuda_tensorcores = None
def eval_with_progress(self, tokens: Sequence[int]):
"""
@ -67,7 +81,7 @@ def monkey_patch_generate(lib):
lib.Llama.generate = my_generate
for lib in [llama_cpp]:
for lib in [llama_cpp, llama_cpp_cuda, llama_cpp_cuda_tensorcores]:
if lib is not None:
lib.Llama.eval = eval_with_progress
monkey_patch_generate(lib)