[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb1-1)from transformers import AutoTokenizer
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb1-2)from functools import partial
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb1-3)model_id = 'Open-Orca/Mistral-7B-OpenOrca'
[](https://hamel.dev/notes/llm/finetuning/tokenizer_gotchas.html#cb1-4)tok = AutoTokenizer.from_pretrained(model_id)