essobi/functiongemma-mobile-actions-v5-16bit

pip install vllm

from vllm import LLM, SamplingParams
from datetime import datetime

# Load model
llm = LLM(
    model="essobi/functiongemma-mobile-actions-v5-16bit",
    trust_remote_code=True,
    max_model_len=4096,
)

# Define available tools
tools = [
    {
        "function": {
            "name": "set_alarm",
            "description": "Sets an alarm for a specific time.",
            "parameters": {
                "type": "OBJECT",
                "properties": {
                    "datetime": {"type": "STRING", "description": "The time for the alarm."},
                    "title": {"type": "STRING", "description": "Optional label for the alarm."},
                },
                "required": ["datetime"]
            }
        }
    },
    {
        "function": {
            "name": "create_reminder",
            "description": "Creates a reminder with text and optional time.",
            "parameters": {
                "type": "OBJECT",
                "properties": {
                    "body": {"type": "STRING", "description": "The reminder text."},
                    "datetime": {"type": "STRING", "description": "When to remind."},
                },
                "required": ["body"]
            }
        }
    },
    {
        "function": {
            "name": "send_message",
            "description": "Sends a text message to a contact.",
            "parameters": {
                "type": "OBJECT",
                "properties": {
                    "to": {"type": "STRING", "description": "Contact name or phone number."},
                    "body": {"type": "STRING", "description": "Message content."},
                },
                "required": ["to", "body"]
            }
        }
    },
    # Add more tools as needed...
]

# Build prompt using the training format
def build_prompt(user_input: str, tools: list) -> str:
    now = datetime.now()
    dt_str = now.strftime("%Y-%m-%dT%H:%M:%S")
    day = now.strftime("%A")
    
    # Build function declarations
    func_decls = ""
    for tool in tools:
        func = tool["function"]
        props = func["parameters"].get("properties", {})
        required = func["parameters"].get("required", [])
        
        props_str = ""
        for pname, pinfo in props.items():
            desc = pinfo.get("description", "")
            ptype = pinfo.get("type", "STRING")
            props_str += f"{pname}:{{description:<escape>{desc}<escape>,type:<escape>{ptype}<escape>}},"
        props_str = props_str.rstrip(",")
        
        req_str = ",".join([f"<escape>{r}<escape>" for r in required])
        
        func_decls += f"<start_function_declaration>declaration:{func['name']}{{description:<escape>{func['description']}<escape>,parameters:{{properties:{{{props_str}}},required:[{req_str}],type:<escape>OBJECT<escape>}}}}<end_function_declaration>"
    
    return f"""<start_of_turn>developer
Current date and time given in YYYY-MM-DDTHH:MM:SS format: {dt_str}
Day of week is {day}
You are a model that can do function calling with the following functions{func_decls}<end_of_turn>
<start_of_turn>user
{user_input}<end_of_turn>
<start_of_turn>model
"""

# Generate
prompt = build_prompt("Set an alarm for 7am tomorrow", tools)
sampling_params = SamplingParams(temperature=0.1, max_tokens=150)
outputs = llm.generate([prompt], sampling_params)

print(outputs[0].outputs[0].text)
# Output: <start_function_call>call:set_alarm{datetime:<escape>7am tomorrow<escape>}<end_function_call>

# Start the server
python -m vllm.entrypoints.openai.api_server \
    --model essobi/functiongemma-mobile-actions-v5-16bit \
    --port 8000

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="dummy")

response = client.chat.completions.create(
    model="essobi/functiongemma-mobile-actions-v5-16bit",
    messages=[
        {"role": "user", "content": "Remind me to call the dentist tomorrow"}
    ],
    max_tokens=150,
    temperature=0.1,
)

print(response.choices[0].message.content)

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "essobi/functiongemma-mobile-actions-v5-16bit"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
    device_map="auto",
)

# Use the same prompt building function as above
prompt = build_prompt("What's the weather like?", tools)

inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=150, temperature=0.1, do_sample=True)
response = tokenizer.decode(outputs[0], skip_special_tokens=False)

print(response)

<start_function_call>call:function_name{param1:<escape>value1<escape>,param2:<escape>value2<escape>}<end_function_call>

import re

def parse_function_call(text: str) -> dict | None:
    """Parse function call from model output."""
    match = re.search(
        r'<start_function_call>call:(\w+)\{([^}]*)\}<end_function_call>', 
        text
    )
    if not match:
        return None
    
    func_name = match.group(1)
    args_str = match.group(2)
    
    # Parse arguments
    args = {}
    for param_match in re.finditer(r'(\w+):<escape>([^<]*)<escape>', args_str):
        args[param_match.group(1)] = param_match.group(2)
    
    return {"name": func_name, "arguments": args}

# Example
output = "<start_function_call>call:set_alarm{datetime:<escape>7am<escape>,title:<escape>Wake up<escape>}<end_function_call>"
parsed = parse_function_call(output)
print(parsed)
# {'name': 'set_alarm', 'arguments': {'datetime': '7am', 'title': 'Wake up'}}