oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-10 17:13:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Bump llama-cpp-python to 0.2.64, use official wheels (#5921)

Browse source
This commit is contained in:
oobabooga 2024-04-23 23:17:05 -03:00 • committed by GitHub
parent 0877741b03
commit 9b623b8a78
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 53 additions and 325 deletions
Download patch file Download diff file Expand all files Collapse all files

18
modules/llama_cpp_python_hijack.py
View file

@ -1,25 +1,11 @@
from typing import Sequence
import llama_cpp
from tqdm import tqdm
from modules import shared
from modules.cache_utils import process_llamacpp_cache
try:
import llama_cpp
except:
llama_cpp = None
try:
import llama_cpp_cuda
except:
llama_cpp_cuda = None
try:
import llama_cpp_cuda_tensorcores
except:
llama_cpp_cuda_tensorcores = None
def eval_with_progress(self, tokens: Sequence[int]):
"""
@ -81,7 +67,7 @@ def monkey_patch_generate(lib):
lib.Llama.generate = my_generate
for lib in [llama_cpp, llama_cpp_cuda, llama_cpp_cuda_tensorcores]:
for lib in [llama_cpp]:
if lib is not None:
lib.Llama.eval = eval_with_progress
monkey_patch_generate(lib)