svjack / Qwen2-7B_Function_Call_tiny_lora

Install some dependency

pip install peft transformers bitsandbytes

Inference

import json
import re
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Literal, Optional, Sequence, Set, Tuple, Union

def calculate_gpa(grades: Sequence[str], hours: Sequence[int]) -> float:
    grade_to_score = {"A": 4, "B": 3, "C": 2}
    total_score, total_hour = 0, 0
    for grade, hour in zip(grades, hours):
        total_score += grade_to_score[grade] * hour
        total_hour += hour
    return round(total_score / total_hour, 2)

tool_map = {"calculate_gpa": calculate_gpa}

from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
from peft import PeftModel

tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-7B-Instruct")
model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-7B-Instruct", 
                                             torch_dtype="auto", device_map="auto", load_in_8bit = True)

model = PeftModel.from_pretrained(model, "svjack/Qwen2-7B_Function_Call_tiny_lora")
streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)

SLOTS = Sequence[Union[str, Set[str], Dict[str, str]]]


DEFAULT_TOOL_PROMPT = (
    "You have access to the following tools:\n{tool_text}"
    "Use the following format if using a tool:\n"
    "```\n"
    "Action: tool name (one of [{tool_names}]).\n"
    "Action Input: the input to the tool, in a JSON format representing the kwargs "
    """(e.g. ```{{"input": "hello world", "num_beams": 5}}```).\n"""
    "```\n"
)

def default_tool_formatter(tools: List[Dict[str, Any]]) -> str:
    tool_text = ""
    tool_names = []
    for tool in tools:
        param_text = ""
        for name, param in tool["parameters"]["properties"].items():
            required = ", required" if name in tool["parameters"].get("required", []) else ""
            enum = ", should be one of [{}]".format(", ".join(param["enum"])) if param.get("enum", None) else ""
            items = (
                ", where each item should be {}".format(param["items"].get("type", "")) if param.get("items") else ""
            )
            param_text += "  - {name} ({type}{required}): {desc}{enum}{items}\n".format(
                name=name,
                type=param.get("type", ""),
                required=required,
                desc=param.get("description", ""),
                enum=enum,
                items=items,
            )

        tool_text += "> Tool Name: {name}\nTool Description: {desc}\nTool Args:\n{args}\n".format(
            name=tool["name"], desc=tool.get("description", ""), args=param_text
        )
        tool_names.append(tool["name"])

    return DEFAULT_TOOL_PROMPT.format(tool_text=tool_text, tool_names=", ".join(tool_names))

def default_tool_extractor(content: str) -> Union[str, List[Tuple[str, str]]]:
    regex = re.compile(r"Action:\s*([a-zA-Z0-9_]+)\s*Action Input:\s*(.+?)(?=\s*Action:|\s*$)", re.DOTALL)
    action_match: List[Tuple[str, str]] = re.findall(regex, content)
    if not action_match:
        return content

    results = []
    for match in action_match:
        tool_name = match[0].strip()
        tool_input = match[1].strip().strip('"').strip("```")
        try:
            arguments = json.loads(tool_input)
            results.append((tool_name, json.dumps(arguments, ensure_ascii=False)))
        except json.JSONDecodeError:
            return content

    return results

#### Function tool defination
tools = [
        {
            "type": "function",
            "function": {
                "name": "calculate_gpa",
                "description": "Calculate the Grade Point Average (GPA) based on grades and credit hours",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "grades": {"type": "array", "items": {"type": "string"}, "description": "The grades"},
                        "hours": {"type": "array", "items": {"type": "integer"}, "description": "The credit hours"},
                    },
                    "required": ["grades", "hours"],
                },
            },
        }
    ]

tools_input = list(map(lambda x: x["function"], tools))
system_tool_prompt = default_tool_formatter(tools_input)
#print(system_tool_prompt)

def qwen_hf_predict(messages, qw_model = model,
    tokenizer = tokenizer, streamer = streamer,
    do_sample = True,
    top_p = 0.95,
    top_k = 40,
    max_new_tokens = 512,
    max_input_length = 3500,
    temperature = 0.9,
    repetition_penalty = 1.0,
    device = "cuda"):

    encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt",
        add_generation_prompt=True
    )
    model_inputs = encodeds.to(device)

    generated_ids = qw_model.generate(model_inputs, max_new_tokens=max_new_tokens,
                                do_sample=do_sample,
                                  streamer = streamer,
                                  top_p = top_p,
                                  top_k = top_k,
                                  temperature = temperature,
                                  repetition_penalty = repetition_penalty,
                                  )
    out = tokenizer.batch_decode(generated_ids)[0].split("<|im_start|>assistant")[-1].replace("<|im_end|>", "").strip()
    return out

messages = [
    {
        "role" :"system",
        "content": system_tool_prompt
    },
    {"role": "user", "content": "My grades are A, A, B, and C. The credit hours are 3, 4, 3, and 2."}
]

out = qwen_hf_predict(messages)
tool_out = default_tool_extractor(out)
print(tool_out)

name, arguments = tool_out[0][0], json.loads(tool_out[0][1])
tool_result = tool_map[name](**arguments)
print(tool_result)

messages.append(
    {
        "role" :"assistant",
        "content": out
    }
)

messages.append({"role": "tool", "content": json.dumps({"gpa": tool_result}, ensure_ascii=False)})

final_out = qwen_hf_predict(messages)
print(final_out)

Output

Action: calculate_gpa
Action Input: {"grades": ["A", "A", "B", "C"], "hours": [3, 4, 3, 2]}
[('calculate_gpa', '{"grades": ["A", "A", "B", "C"], "hours": [3, 4, 3, 2]}')]
3.42
Based on the grades and credit hours you provided, your Grade Point Average (GPA) is 3.42.

Inference

messages = [
    {
        "role" :"system",
        "content": system_tool_prompt
    },
    {"role": "user", "content": "我的成绩分别是A,A,B,C学分分别是3, 4, 3,和2"}
]

out = qwen_hf_predict(messages)
tool_out = default_tool_extractor(out)
print(tool_out)

name, arguments = tool_out[0][0], json.loads(tool_out[0][1])
tool_result = tool_map[name](**arguments)
print(tool_result)

messages.append(
    {
        "role" :"assistant",
        "content": out
    }
)

messages.append({"role": "tool", "content": json.dumps({"gpa": tool_result}, ensure_ascii=False)})

final_out = qwen_hf_predict(messages)
print(final_out)

Output

Action: calculate_gpa
Action Input: {"grades": ["A", "A", "B", "C"], "hours": [3, 4, 3, 2]}
[('calculate_gpa', '{"grades": ["A", "A", "B", "C"], "hours": [3, 4, 3, 2]}')]
3.42
您的绩点（GPA）是3.42。

train_2024-06-17-19-49-05

This model is a fine-tuned version of Qwen/Qwen2-7B-Instruct on the glaive_toolcall_zh and the glaive_toolcall_en datasets.

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

• learning_rate: 5e-05
• train_batch_size: 1
• eval_batch_size: 8
• seed: 42
• distributed_type: multi-GPU
• num_devices: 2
• gradient_accumulation_steps: 8
• total_train_batch_size: 16
• total_eval_batch_size: 16
• optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
• lr_scheduler_type: cosine
• num_epochs: 3.0
• mixed_precision_training: Native AMP

Training results

Framework versions

• PEFT 0.11.1
• Transformers 4.41.2
• Pytorch 2.3.1+cu121
• Datasets 2.20.0
• Tokenizers 0.19.1