import datetime import json import os import shutil from typing import Optional from typing import Tuple from typing import Union import gradio as gr import requests import torch from fastchat.conversation import Conversation from fastchat.conversation import SeparatorStyle from fastchat.conversation import get_conv_template from fastchat.conversation import register_conv_template from fastchat.model.model_adapter import BaseAdapter from fastchat.model.model_adapter import load_model from fastchat.model.model_adapter import model_adapters from fastchat.serve.cli import SimpleChatIO from fastchat.serve.inference import generate_stream from huggingface_hub import Repository from huggingface_hub import snapshot_download from peft import LoraConfig from peft import PeftModel from peft import get_peft_model from peft import set_peft_model_state_dict from transformers import AutoModelForCausalLM from transformers import AutoTokenizer from transformers import PreTrainedModel from transformers import PreTrainedTokenizerBase class FastTokenizerAvailableBaseAdapter(BaseAdapter): def load_model(self, model_path: str, from_pretrained_kwargs: dict): try: tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) except ValueError: tokenizer = AutoTokenizer.from_pretrained(model_path) model = AutoModelForCausalLM.from_pretrained( model_path, low_cpu_mem_usage=True, **from_pretrained_kwargs ) return model, tokenizer model_adapters[-1] = FastTokenizerAvailableBaseAdapter() def load_lora_model( model_path: str, lora_weight: str, device: str, num_gpus: int, max_gpu_memory: Optional[str] = None, load_8bit: bool = False, cpu_offloading: bool = False, debug: bool = False, ) -> Tuple[Union[PreTrainedModel, PeftModel], PreTrainedTokenizerBase]: model: Union[PreTrainedModel, PeftModel] tokenizer: PreTrainedTokenizerBase model, tokenizer = load_model( model_path=model_path, device=device, num_gpus=num_gpus, max_gpu_memory=max_gpu_memory, load_8bit=load_8bit, cpu_offloading=cpu_offloading, debug=debug, ) if lora_weight is not None: # model = PeftModelForCausalLM.from_pretrained(model, model_path, **kwargs) config = LoraConfig.from_pretrained(lora_weight) model = get_peft_model(model, config) # Check the available weights and load them checkpoint_name = os.path.join( lora_weight, "pytorch_model.bin" ) # Full checkpoint if not os.path.exists(checkpoint_name): checkpoint_name = os.path.join( lora_weight, "adapter_model.bin" ) # only LoRA model - LoRA config above has to fit # The two files above have a different name depending on how they were saved, # but are actually the same. if os.path.exists(checkpoint_name): adapters_weights = torch.load(checkpoint_name) set_peft_model_state_dict(model, adapters_weights) else: raise IOError(f"Checkpoint {checkpoint_name} not found") if debug: print(model) return model, tokenizer print(datetime.datetime.now()) NUM_THREADS = 1 print(NUM_THREADS) print("starting server ...") BASE_MODEL = "cyberagent/open-calm-7b" LORA_WEIGHTS_HF = "izumi-lab/stormy-7b-10ep" HF_TOKEN = os.environ.get("HF_TOKEN", None) DATASET_REPOSITORY = os.environ.get("DATASET_REPOSITORY", None) SLACK_WEBHOOK = os.environ.get("SLACK_WEBHOOK", None) LORA_WEIGHTS = snapshot_download(LORA_WEIGHTS_HF) repo = None LOCAL_DIR = "/home/user/data/" if HF_TOKEN and DATASET_REPOSITORY: try: shutil.rmtree(LOCAL_DIR) except Exception: pass repo = Repository( local_dir=LOCAL_DIR, clone_from=DATASET_REPOSITORY, use_auth_token=HF_TOKEN, repo_type="dataset", ) repo.git_pull() if torch.cuda.is_available(): device = "cuda" else: device = "cpu" model, tokenizer = load_lora_model( model_path=BASE_MODEL, lora_weight=LORA_WEIGHTS, device=device, num_gpus=1, max_gpu_memory="16GiB", load_8bit=False, cpu_offloading=False, debug=False, ) register_conv_template( Conversation( name="japanese", system="以下はタスクを説明する指示です。要求を適切に満たすような返答を書いてください。\n\n", roles=("### 指示", "### 返答"), messages=(), offset=0, sep_style=SeparatorStyle.ADD_COLON_SINGLE, sep="\n###", stop_str="###", ) ) Conversation._get_prompt = Conversation.get_prompt Conversation._append_message = Conversation.append_message def conversation_append_message(cls, role: str, message: str): cls.offset = -2 return cls._append_message(role, message) def conversation_get_prompt_overrider(cls: Conversation) -> str: cls.messages = cls.messages[-2:] return cls._get_prompt() def save_inputs_and_outputs(now, inputs, outputs, generate_kwargs): current_hour = now.strftime("%Y-%m-%d_%H") file_name = f"prompts_{LORA_WEIGHTS_HF.split('/')[-1]}_{current_hour}.jsonl" if repo is not None: repo.git_pull(rebase=True) with open(os.path.join(LOCAL_DIR, file_name), "a", encoding="utf-8") as f: json.dump( { "inputs": inputs, "outputs": outputs, "generate_kwargs": generate_kwargs, }, f, ensure_ascii=False, ) f.write("\n") repo.push_to_hub() # we cant add typing now # https://github.com/gradio-app/gradio/issues/3514 def evaluate( instruction, temperature=0.7, max_tokens=256, repetition_penalty=1.0, ): try: conv_template = "japanese" inputs = tokenizer(instruction, return_tensors="pt") if len(inputs["input_ids"][0]) > max_tokens - 40: if HF_TOKEN and DATASET_REPOSITORY: try: now = datetime.datetime.now() current_time = now.strftime("%Y-%m-%d %H:%M:%S") print(f"[{current_time}] Pushing prompt and completion to the Hub") save_inputs_and_outputs( now, instruction, "", { "temperature": temperature, "max_tokens": max_tokens, "repetition_penalty": repetition_penalty, }, ) except Exception as e: print(e) return ( f"please reduce the input length. Currently, {len(inputs['input_ids'][0])} ( > {max_tokens - 40}) tokens are used.", gr.update(interactive=True), gr.update(interactive=True), ) conv = get_conv_template(conv_template) conv.append_message(conv.roles[0], instruction) conv.append_message(conv.roles[1], None) generate_stream_func = generate_stream prompt = conv.get_prompt() gen_params = { "model": BASE_MODEL, "prompt": prompt, "temperature": temperature, "max_new_tokens": max_tokens - len(inputs["input_ids"][0]) - 30, "stop": conv.stop_str, "stop_token_ids": conv.stop_token_ids, "echo": False, "repetition_penalty": repetition_penalty, } chatio = SimpleChatIO() chatio.prompt_for_output(conv.roles[1]) output_stream = generate_stream_func(model, tokenizer, gen_params, device) output = chatio.stream_output(output_stream) if HF_TOKEN and DATASET_REPOSITORY: try: now = datetime.datetime.now() current_time = now.strftime("%Y-%m-%d %H:%M:%S") print(f"[{current_time}] Pushing prompt and completion to the Hub") save_inputs_and_outputs( now, prompt, output, { "temperature": temperature, "max_tokens": max_tokens, "repetition_penalty": repetition_penalty, }, ) except Exception as e: print(e) return output, gr.update(interactive=True), gr.update(interactive=True) except Exception as e: print(e) import traceback if SLACK_WEBHOOK: payload_dic = { "text": f"BASE_MODEL: {BASE_MODEL}\n LORA_WEIGHTS: {LORA_WEIGHTS_HF}\n" + f"instruction: {instruction}\ninput: {input}\ntemperature: {temperature}\n" + f"max_tokens: {max_tokens}\nrepetition_penalty: {repetition_penalty}\n\n" + str(traceback.format_exc()), "username": "Hugging Face Space", "channel": "#monitor", } try: requests.post(SLACK_WEBHOOK, data=json.dumps(payload_dic)) except Exception: pass return ( "Error happend. Please return later.", gr.update(interactive=True), gr.update(interactive=True), ) def reset_textbox(): return gr.update(value=""), gr.update(value=""), gr.update(value="") def no_interactive() -> Tuple[gr.Request, gr.Request]: return gr.update(interactive=False), gr.update(interactive=False) title = """

stormy 7B 10epochs

""" theme = gr.themes.Default(primary_hue="green") description = ( "The official demo for **[izumi-lab/stormy-7b-10ep](https://huggingface.co/izumi-lab/stormy-7b-10ep)**. " "It is a 7B-parameter CALM model finetuned to follow instructions. " "It is trained on the dataset specially extracted from [izumi-lab/llm-japanese-dataset](https://huggingface.co/datasets/izumi-lab/llm-japanese-dataset) dataset. " "For more information, please visit [the project's website](https://llm.msuzuki.me). " "This model can output up to 256 tokens. " "It takes about **1 minute** to output. When access is concentrated, the operation may become slow." ) with gr.Blocks( css="""#col_container { margin-left: auto; margin-right: auto;}""", theme=theme, ) as demo: gr.HTML(title) gr.Markdown(description) with gr.Column(elem_id="col_container", visible=False) as main_block: with gr.Row(): with gr.Column(): instruction = gr.Textbox( lines=3, label="Instruction", placeholder="こんにちは" ) with gr.Row(): with gr.Column(scale=3): clear_button = gr.Button("Clear").style(full_width=True) with gr.Column(scale=5): submit_button = gr.Button("Submit").style(full_width=True) outputs = gr.Textbox(lines=4, label="Output") # inputs, top_p, temperature, top_k, repetition_penalty with gr.Accordion("Parameters", open=True): temperature = gr.Slider( minimum=0, maximum=1.0, value=0.0, step=0.05, interactive=True, label="Temperature", ) max_tokens = gr.Slider( minimum=20, maximum=256, value=128, step=1, interactive=True, label="Max length (Pre-prompt + instruction + input + output)", ) repetition_penalty = gr.Slider( minimum=0.0, maximum=5.0, value=1.05, step=0.05, interactive=True, label="Repetition penalty", ) with gr.Column(elem_id="user_consent_container") as user_consent_block: # Get user consent gr.Markdown( """ ## User Consent for Data Collection, Use, and Sharing: By using our app, you acknowledge and agree to the following terms regarding the data you provide: - **Collection**: We may collect inputs you type into our app. - **Use**: We may use the collected data for research purposes, to improve our services, and to develop new products or services, including commercial applications. - **Sharing and Publication**: Your input data may be published, shared with third parties, or used for analysis and reporting purposes. - **Data Retention**: We may retain your input data for as long as necessary. By continuing to use our app, you provide your explicit consent to the collection, use, and potential sharing of your data as described above. If you do not agree with our data collection, use, and sharing practices, please do not use our app. ## データ収集、利用、共有に関するユーザーの同意: 本アプリを使用することにより、提供するデータに関する以下の条件に同意するものとします: - **収集**: 本アプリに入力されるテキストデータは収集される場合があります。 - **利用**: 収集されたデータは研究や、商用アプリケーションを含むサービスの開発に使用される場合があります。 - **共有および公開**: 入力データは第三者と共有されたり、分析や公開の目的で使用される場合があります。 - **データ保持**: 入力データは必要な限り保持されます。 本アプリを引き続き使用することにより、上記のようにデータの収集・利用・共有について同意します。データの利用方法に同意しない場合は、本アプリを使用しないでください。 """ ) accept_button = gr.Button("I Agree") def enable_inputs(): return user_consent_block.update(visible=False), main_block.update( visible=True ) accept_button.click( fn=enable_inputs, inputs=[], outputs=[user_consent_block, main_block], queue=False, ) submit_button.click(no_interactive, [], [submit_button, clear_button]) submit_button.click( evaluate, [instruction, temperature, max_tokens, repetition_penalty], [outputs, submit_button, clear_button], ) clear_button.click(reset_textbox, [], [instruction, outputs], queue=False) demo.queue(max_size=20, concurrency_count=NUM_THREADS, api_open=False).launch( server_name="0.0.0.0", server_port=7860 )