[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-1)from transformers import AutoTokenizer
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-2)tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-7b-chat-hf")
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-3)
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-4)chat = [
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-5)   {"role": "system", "content": "lorem"},
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-6)   {"role": "user", "content": "abc"},
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-7)   {"role": "assistant", "content": "ipsum"},
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-8)   {"role": "user", "content": "123"},
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-9)   {"role": "assistant", "content": "sit"},
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-10)]
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-11)
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-12)ids = tokenizer.apply_chat_template(chat)
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb11-13)print(tokenizer.decode(ids))