StarRing2022
commited on
Commit
•
fd7d871
1
Parent(s):
0caf375
Upload 2 files
Browse files- alpacatest.py +63 -0
- alpacatrain.py +59 -0
alpacatest.py
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from datasets import load_dataset
|
2 |
+
from transformers import RwkvForCausalLM, GPTNeoXTokenizerFast,GPT2Config,pipeline,GenerationConfig
|
3 |
+
import torch
|
4 |
+
import numpy as np
|
5 |
+
import gradio as gr
|
6 |
+
|
7 |
+
if torch.cuda.is_available():
|
8 |
+
device = "cuda"
|
9 |
+
else:
|
10 |
+
device = "cpu"
|
11 |
+
|
12 |
+
model = RwkvForCausalLM.from_pretrained("rwkv-alpaca",device_map='auto') #仅500MB,自训练,使用alpaca
|
13 |
+
|
14 |
+
tokenizer = GPTNeoXTokenizerFast.from_pretrained("rwkv-alpaca", add_special_tokens=True)
|
15 |
+
|
16 |
+
|
17 |
+
|
18 |
+
#rwkv with alpaca
|
19 |
+
def generate_prompt(instruction, input=None):
|
20 |
+
|
21 |
+
return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
|
22 |
+
|
23 |
+
### Instruction:
|
24 |
+
{instruction}
|
25 |
+
|
26 |
+
### Response:"""
|
27 |
+
|
28 |
+
def evaluate(
|
29 |
+
instruction,
|
30 |
+
temperature=0.1,
|
31 |
+
top_p=0.75,
|
32 |
+
top_k=40,
|
33 |
+
max_new_tokens=128,
|
34 |
+
):
|
35 |
+
prompt = generate_prompt(instruction)
|
36 |
+
input_ids = tokenizer.encode(prompt, return_tensors='pt')
|
37 |
+
out = model.generate(input_ids=input_ids,temperature=temperature,top_p=top_p,top_k=top_k,max_new_tokens=max_new_tokens)
|
38 |
+
answer = tokenizer.decode(out[0])
|
39 |
+
return answer.split("### Response:")[1].strip()
|
40 |
+
|
41 |
+
|
42 |
+
gr.Interface(
|
43 |
+
fn=evaluate,#接口函数
|
44 |
+
inputs=[
|
45 |
+
gr.components.Textbox(
|
46 |
+
lines=2, label="Instruction", placeholder="Tell me about alpacas."
|
47 |
+
),
|
48 |
+
gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature"),
|
49 |
+
gr.components.Slider(minimum=0, maximum=1, value=0.75, label="Top p"),
|
50 |
+
gr.components.Slider(minimum=0, maximum=100, step=1, value=40, label="Top k"),
|
51 |
+
gr.components.Slider(
|
52 |
+
minimum=1, maximum=2000, step=1, value=128, label="Max tokens"
|
53 |
+
),
|
54 |
+
],
|
55 |
+
outputs=[
|
56 |
+
gr.inputs.Textbox(
|
57 |
+
lines=5,
|
58 |
+
label="Output",
|
59 |
+
)
|
60 |
+
],
|
61 |
+
title="RWKV-Alpaca",
|
62 |
+
description="RWKV,easy in HF.",
|
63 |
+
).launch()
|
alpacatrain.py
ADDED
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from datasets import load_dataset
|
2 |
+
from transformers import RwkvForCausalLM, GPTNeoXTokenizerFast, Trainer, TrainingArguments,DataCollatorForLanguageModeling
|
3 |
+
|
4 |
+
MICRO_BATCH_SIZE = 8
|
5 |
+
BATCH_SIZE = 128
|
6 |
+
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
|
7 |
+
EPOCHS = 100
|
8 |
+
LEARNING_RATE = 2e-5
|
9 |
+
CUTOFF_LEN = 256
|
10 |
+
|
11 |
+
model = RwkvForCausalLM.from_pretrained("rwkv-430M-pile")
|
12 |
+
tokenizer = GPTNeoXTokenizerFast.from_pretrained("rwkv-430M-pile", add_special_tokens=True)
|
13 |
+
# model = RwkvForCausalLM.from_pretrained("rwkv-7b-pile")
|
14 |
+
# tokenizer = GPTNeoXTokenizerFast.from_pretrained("rwkv-7b-pile", add_special_tokens=True)
|
15 |
+
tokenizer.pad_token = tokenizer.eos_token
|
16 |
+
tokenizer.pad_token_id = tokenizer.eos_token_id
|
17 |
+
|
18 |
+
data = load_dataset("json", data_files="test.json")
|
19 |
+
|
20 |
+
def generate_prompt(data_point):
|
21 |
+
|
22 |
+
return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
|
23 |
+
|
24 |
+
### Instruction:
|
25 |
+
{data_point["instruction"]}
|
26 |
+
|
27 |
+
### Response:
|
28 |
+
{data_point["output"]}"""
|
29 |
+
|
30 |
+
|
31 |
+
data = data.shuffle().map(
|
32 |
+
lambda data_point: tokenizer(
|
33 |
+
generate_prompt(data_point),
|
34 |
+
truncation=True,
|
35 |
+
max_length=CUTOFF_LEN,
|
36 |
+
padding="max_length",
|
37 |
+
)
|
38 |
+
)
|
39 |
+
|
40 |
+
trainer = Trainer(
|
41 |
+
model=model,
|
42 |
+
train_dataset=data["train"],
|
43 |
+
args=TrainingArguments(
|
44 |
+
per_device_train_batch_size=MICRO_BATCH_SIZE,
|
45 |
+
gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS,
|
46 |
+
warmup_steps=100,
|
47 |
+
num_train_epochs=EPOCHS,
|
48 |
+
learning_rate=LEARNING_RATE,
|
49 |
+
fp16=True,
|
50 |
+
logging_steps=1,
|
51 |
+
output_dir="rwkv-alpaca",
|
52 |
+
save_total_limit=3,
|
53 |
+
),
|
54 |
+
data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False),
|
55 |
+
)
|
56 |
+
model.config.use_cache = False
|
57 |
+
trainer.train(resume_from_checkpoint=False)
|
58 |
+
|
59 |
+
model.save_pretrained("rwkv-alpaca")
|