Merge branch 'oobabooga:dev' into dev

This commit is contained in:
Underscore 2026-01-08 09:01:06 -05:00 committed by GitHub
commit d1c3fba1df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 80 additions and 57 deletions

View file

@ -83,7 +83,11 @@ def get_model_metadata(model):
if 'tokenizer.chat_template' in metadata:
template = metadata['tokenizer.chat_template']
eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']]
if 'tokenizer.ggml.eos_token_id' in metadata:
eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']]
else:
eos_token = ""
if 'tokenizer.ggml.bos_token_id' in metadata:
bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']]
else: