oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-06 15:13:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Suppress EOS token at logit level for ExLlamav3 when ban_eos_token is set

Browse source
This commit is contained in:
oobabooga 2026-03-28 06:17:57 -07:00
parent bda95172bd
commit 9dd04b86ce
1 changed files with 9 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

9
modules/exllamav3.py
View file

@ -423,6 +423,15 @@ class Exllamav3Model:
if logit_bias:
filters.append(LogitBiasFilter(self.tokenizer, logit_bias))
# Suppress EOS tokens via logit bias so they are never sampled
if state['ban_eos_token']:
eos_bias = {}
for eos_id in self.config.eos_token_id_list:
if eos_id is not None:
eos_bias[str(eos_id)] = float('-inf')
if eos_bias:
filters.append(LogitBiasFilter(self.tokenizer, eos_bias))
# Logprobs support (OpenAI API)
logprobs = state.get('logprobs', 0) or 0
return_top_tokens = logprobs if logprobs > 0 else 0