# Numerical encoding of individual character

token2idx = {char: idx for idx, char in enumerate(sorted(set(tokens)))}
print(token2idx)

👉 {' ': 0, '!': 1, 'L': 2, 'N': 3, 'P': 4, 'W': 5, 'e': 6, 'l': 7, 'o': 8, 'v': 9}

# Using token2idx to map our tokenized text to integers

integer_tokens = [token2idx[token] for token in tokens]
print(integer_tokens)

👉 [5, 6, 0, 7, 8, 9, 6, 0, 3, 2, 4, 1]