BLOCK_SIZE = 16 def block_hashes(token_ids, salt=None): """Chain-hash a token sequence into per-block keys.""" hashes, parent = [], hash(salt) # Only complete blocks are hashed. A partial tail block is skipped. for start in range(0, len(token_ids) - BLOCK_SIZE + 1, BLOCK_SIZE): block = tuple(token_ids[start : start + BLOCK_SIZE]) parent = hash((parent, block)) hashes.append(parent) return hashes