from transformers import LlamaTokenizer

tokenizer = LlamaTokenizer.from_pretrained(
        'ocisd4/llama_tokenizer_ext_zh',
        pad_token='<unk>',
        add_bos_token=True,
        add_eos_token=False,
        use_auth_token=True,
)

print('vocab size:',tokenizer.vocab_size)
#vocab size: 51904   

print(tokenizer.tokenize('今天天氣真好！'))
#['▁', '今天', '天氣', '真', '好', '！']

print(tokenizer.encode('今天天氣真好！'))
#[1, 29871, 32096, 32917, 30848, 31076, 30584]

print(tokenizer.decode(tokenizer.encode('今天天氣真好！')))
# <s>今天天氣真好！

** note: The first token might be a whitespace in LLamaTokenizer. **

updated

• 2023-08-07
  • remove special tokens and use <unk> as pad to align with llama 2 sft.
• 2023-06-02
  • add special tokens: <|output|>, <|input|>, <|sep|>, <|emb|>, <|rwd|>, <|ctx|>