[](https://hamel.dev/notes/llm/inference/inference.html#cb6-1)import time
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-2)import ctranslate2
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-3)import transformers
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-4)import sys
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-5)sys.path.append('../common/')
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-6)from questions import questions
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-7)import pandas as pd
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-8)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-9)generator = ctranslate2.Generator("llama-2-7b-ct2", device="cuda")
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-10)tokenizer = transformers.AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-11)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-12)def predict(prompt:str):
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-13)    "Generate text give a prompt"
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-14)    start = time.perf_counter()
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-15)    tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-16)    results = generator.generate_batch([tokens], sampling_topk=1, max_length=200, include_prompt_in_result=False)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-17)    tokens = results[0].sequences_ids[0]
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-18)    output = tokenizer.decode(tokens)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-19)    request_time = time.perf_counter() - start
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-20)    return {'tok_count': len(tokens),
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-21)            'time': request_time,
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-22)            'question': prompt,
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-23)            'answer': output,
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-24)            'note': 'CTranslate2 int8 quantization'}
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-25)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-26)if __name__ == '__main__':
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-27)    counter = 1
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-28)    responses = []
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-29)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-30)    for q in questions:
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-31)        if counter >= 2: responses.append(predict(q))
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-32)        counter += 1
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-33)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-34)    df = pd.DataFrame(responses)
[](https://hamel.dev/notes/llm/inference/inference.html#cb6-35)    df.to_csv('bench-ctranslate-int8.csv', index=False)