from transformers import AutoTokenizer

model_name = "Qwen/Qwen3-1.7B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
text = "This is a great blog about Tokenizers."
tokens = tokenizer.tokenize(text)
print(tokens)