7 lines
220 B
Python
7 lines
220 B
Python
from transformers import AutoTokenizer
|
|
|
|
model_name = "Qwen/Qwen3-1.7B"
|
|
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
|
text = "This is a great blog about Tokenizers."
|
|
tokens = tokenizer.tokenize(text)
|
|
print(tokens) |