courage17340 hmellor HF Staff commited on
Commit
ac6c49f
·
verified ·
1 Parent(s): 12d9c7c

Transformers v5 support (#21)

Browse files

- Transformers v5 support (227ab582b3b2617d10be2f8bfa78707c3c38e4af)


Co-authored-by: Harry Mellor <hmellor@users.noreply.huggingface.co>

Files changed (1) hide show
  1. tokenization_kimi.py +1 -1
tokenization_kimi.py CHANGED
@@ -16,7 +16,7 @@ from shutil import copyfile
16
  from tiktoken.load import load_tiktoken_bpe
17
  from tokenizers import AddedToken, pre_tokenizers, Regex
18
  from transformers.tokenization_utils import PreTrainedTokenizer
19
- from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode
20
  from typing import Any
21
 
22
 
 
16
  from tiktoken.load import load_tiktoken_bpe
17
  from tokenizers import AddedToken, pre_tokenizers, Regex
18
  from transformers.tokenization_utils import PreTrainedTokenizer
19
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
20
  from typing import Any
21
 
22