Erste Tests

2026-03-30 14:57:42 +02:00
parent c59ae0ad82
commit 81ea3521a8
2 changed files with 55 additions and 0 deletions
@@ -0,0 +1,7 @@
+from transformers import AutoTokenizer
+
+model_name = "Qwen/Qwen3-1.7B"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+text = "This is a great blog about Tokenizers."
+tokens = tokenizer.tokenize(text)
+print(tokens)