Sample
Browse files- sample/ct2-bloom.py +24 -0
sample/ct2-bloom.py
ADDED
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import ctranslate2
|
2 |
+
import transformers
|
3 |
+
import datetime
|
4 |
+
|
5 |
+
for prompt in ["Hello, I am Joan and I am from Barcelona.", "Translate to Catalan: how are you today?", "Translate to Catalan: It is a British public service broadcaster, Headquartered in London, and it is the world's oldest national broadcaster.", "Parafraseja en català: Avui serà un dia molt llarg perquè farem moltes coses junts."]:
|
6 |
+
for model_name in [#"bloom-560m",
|
7 |
+
"bloomz-7b1-mt"]:
|
8 |
+
|
9 |
+
generator = ctranslate2.Generator(model_name, compute_type="int8")
|
10 |
+
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
|
11 |
+
|
12 |
+
start_time = datetime.datetime.now()
|
13 |
+
|
14 |
+
start_tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))
|
15 |
+
results = generator.generate_batch([start_tokens], max_length=90)
|
16 |
+
result = tokenizer.decode(results[0].sequences_ids[0])
|
17 |
+
t = str(datetime.datetime.now() - start_time)
|
18 |
+
print(f"Model: {model_name}")
|
19 |
+
print(f"Prompt: {prompt}")
|
20 |
+
print(f"Result: {result}")
|
21 |
+
print(f"Time: {t}")
|
22 |
+
print("")
|
23 |
+
|
24 |
+
del generator, tokenizer
|