from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("google/gemma-3-270m-it")

print(f"{tokenizer._tokenizer.normalizer=}")
# Replace(...)

print(f"{tokenizer._tokenizer.pre_tokenizer=}")
# Split(...)

print(f"{tokenizer._tokenizer.model=}")
# BPE(...)

print(f"{tokenizer._tokenizer.post_processor=}")
# TemplateProcessing(...)

print(f"{tokenizer._tokenizer.decoder=}")
# Sequence(decoders=[Replace(...), ByteFallback(), Fuse()])