oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-01-30 20:34:32 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix exllamav2 generating eos randomly after previous fix

Browse source
This commit is contained in:
oobabooga 2025-04-18 05:42:38 -07:00
parent 2fc58ad935
commit 5c2f8d828e
2 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
modules/exllamav2_hf.py
View file

@ -157,6 +157,9 @@ class Exllamav2HF(PreTrainedModel):
else:
self.past_seq = seq_tensor
if torch.cuda.is_available():
torch.cuda.synchronize()
loss = None
if labels is not None:
# Shift so that tokens < n predict n

3
modules/text_generation.py
View file

@ -263,9 +263,6 @@ def apply_stopping_strings(reply, all_stop_strings):
def get_reply_from_output_ids(output_ids, state=None, starting_from=0):
if torch.cuda.is_available():
torch.cuda.synchronize()
reply = decode(output_ids[starting_from:], state['skip_special_tokens'] if state else True)
# Handle tokenizers that do not add the leading space for the first token