Update README.md

README.md

@@ -1,20 +1,22 @@
+```python
 from transformers import LlamaTokenizer
 
-tokenizer = LlamaTokenizer.from_pretrained(
-        'ocisd4/llama_tokenizer_ext_zhtw',
-        pad_token='<unk>',
-        add_bos_token=True,
-        add_eos_token=False,
-        use_auth_token=True,
-)
+tokenizer = LlamaTokenizer.from_pretrained(  
+        'ocisd4/llama_tokenizer_ext_zhtw',  
+        pad_token='<unk>',  
+        add_bos_token=True,  
+        add_eos_token=False,  
+        use_auth_token=True,  
+)  
 
-#vocab size: 36128   
+#vocab size: 36128     
 
-print(tokenizer.tokenize('今天天氣真好！'))
+print(tokenizer.tokenize('今天天氣真好！'))   
 #['▁', '今', '天', '天', '氣', '真', '好', '！']
 
-print(tokenizer.encode('今天天氣真好！'))
+print(tokenizer.encode('今天天氣真好！'))  
 #[1, 29871, 31482, 30408, 30408, 32045, 30848, 31076, 30584]
 
-print(tokenizer.decode(tokenizer.encode('今天天氣真好！')))
-# <s>今天天氣真好！
+print(tokenizer.decode(tokenizer.encode('今天天氣真好！')))  
+# <s>今天天氣真好！
+```