lime-nlp/ExeVRM-8B

pip install transformers torch accelerate

pip install av pillow

from transformers import Qwen3VLForConditionalGeneration, AutoProcessor
import torch

model_name = "lime-nlp/ExeVRM-8B"  # Replace with the actual HF repo name

# Load model
model = Qwen3VLForConditionalGeneration.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    trust_remote_code=True,
)

# Load processor
processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)

import av
import numpy as np
import math


def sample_frames_from_video(video_path, fps=1.0, max_frames=50):
    """
    Sample frames from a video at the specified FPS, up to max_frames.

    Args:
        video_path: Path to the video file (mp4, avi, etc.)
        fps: Target frames per second for sampling (default: 1.0)
        max_frames: Maximum number of frames to sample (default: 50)

    Returns:
        List of PIL Images
    """
    container = av.open(video_path)
    stream = container.streams.video[0]

    total_frames = stream.frames
    duration_seconds = float(stream.duration * stream.time_base)

    # Compute number of frames to sample
    sample_frames = max(1, math.floor(duration_seconds * fps))
    sample_frames = min(total_frames, max_frames, sample_frames)

    # Uniformly sample frame indices
    indices = np.linspace(0, total_frames - 1, sample_frames).astype(np.int32)
    indices_set = set(indices.tolist())

    frames = []
    for i, frame in enumerate(container.decode(video=0)):
        if i in indices_set:
            frames.append(frame.to_image())  # Convert to PIL Image
        if i > max(indices):
            break

    container.close()
    return frames

from qwen_vl_utils import process_vision_info

# Prepare the input message
messages = [
    {
        "role": "user",
        "content": [
            {"type": "video", "video": "path/to/agent_trajectory.mp4", "fps": 1.0},
            {"type": "text", "text": (
                "Given a user task and a computer-using video recording, "
                "evaluate whether the user completes the task or not. "
                "Reply your judgement in the \\box{}.\n"
                "If the video correctly completes the task, reply \\box{correct}. "
                "Otherwise, reply \\box{incorrect}.\n\n"
                "# User Task\n"
                "Open Google Chrome and search for 'weather today'\n"
            )},
        ],
    }
]

# Process inputs
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
image_inputs, video_inputs = process_vision_info(messages)
inputs = processor(
    text=[text],
    images=image_inputs,
    videos=video_inputs,
    padding=True,
    return_tensors="pt",
).to(model.device)

# Generate
with torch.no_grad():
    output_ids = model.generate(**inputs, max_new_tokens=128)

# Decode
generated_ids = output_ids[:, inputs.input_ids.shape[1]:]
response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(response)
# Expected output: \box{correct} or \box{incorrect}

pip install qwen-vl-utils

# 1. Sample frames manually
frames = sample_frames_from_video("path/to/video.mp4", fps=1.0, max_frames=50)

# 2. Build message with individual frames as images
content = [{"type": "video", "video": frames}]
content.append({
    "type": "text",
    "text": (
        "Given a user task and a computer-using video recording, "
        "evaluate whether the user completes the task or not. "
        "Reply your judgement in the \\box{}.\n"
        "If the video correctly completes the task, reply \\box{correct}. "
        "Otherwise, reply \\box{incorrect}.\n\n"
        "# User Task\n"
        "Your task description here\n"
    ),
})

messages = [{"role": "user", "content": content}]

# 3. Process and run inference (same as above)
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
image_inputs, video_inputs = process_vision_info(messages)
inputs = processor(
    text=[text],
    images=image_inputs,
    videos=video_inputs,
    padding=True,
    return_tensors="pt",
).to(model.device)

with torch.no_grad():
    output_ids = model.generate(**inputs, max_new_tokens=256)

generated_ids = output_ids[:, inputs.input_ids.shape[1]:]
response = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(response)

Given a user task and a computer-using video recording, evaluate whether the user completes the task or not. Reply your judgement in the \box{}.
If the video correctly completes the task, reply \box{correct}. Otherwise, reply \box{incorrect}.

# User Task
<task description>

Given a user task and a computer-using video recording, evaluate whether the user completes the task or not. Reply your judgement in the \box{}.
If the video correctly completes the task, reply \box{correct}. Otherwise, reply \box{incorrect}.
If the video does not complete the task (i.e., incorrect), please provide the timestamp range, i.e., from <[time_start] seconds> to <[time_end] seconds>, of the video that deviates from the user's instruction.

# User Task
<task description>

@misc{song2026videobasedrewardmodelingcomputeruse,
      title={Video-Based Reward Modeling for Computer-Use Agents}, 
      author={Linxin Song and Jieyu Zhang and Huanxin Sheng and Taiwei Shi and Gupta Rahul and Yang Liu and Ranjay Krishna and Jian Kang and Jieyu Zhao},
      year={2026},
      eprint={2603.10178},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2603.10178}, 
}