oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-01-05 08:10:07 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

llama.cpp: new attempt at optimizing the llama-server connection

Browse source
This commit is contained in:
oobabooga 2025-04-18 19:30:53 -07:00
parent f727b4a2cc
commit 9f07a1f5d7
1 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
modules/llama_cpp_server.py
View file

@ -147,19 +147,18 @@ class LlamaServer:
full_text = ""
# Process the streaming response
for line in response.iter_lines():
for line in response.iter_lines(decode_unicode=True):
if shared.stop_everything:
break
if line:
try:
# Check if the line starts with "data: " and remove it
line_str = line.decode('utf-8')
if line_str.startswith('data: '):
line_str = line_str[6:] # Remove the "data: " prefix
if line.startswith('data: '):
line = line[6:] # Remove the "data: " prefix
# Parse the JSON data
data = json.loads(line_str)
data = json.loads(line)
# Extract the token content
if 'content' in data: