lorashen/Auto-SLURP
Auto-SLURP: A Benchmark Dataset for Evaluating Multi-Agent Frameworks in Smart Personal Assistant Repository for the paper Auto-SLURP: A Benchmark Dataset for Evaluating Multi-Agent Frameworks in Smart Personal Assistant requirements To test the multi-agent frameworks, you need to first install the framework according to the instruction of the framework. We have tested CamelAI, Langgraph, AgentLite, and AutoGEN. 1. start simulated servers cd server… See the full description on the dataset page: https://huggingface.co/datasets/lorashen/Auto-SLURP.
0108
1{"id": "7499", "query": "make new schedule for tomorrow meeting", "intent": "calendar_set", "slots": [{"date": "tomorrow", "event_name": "meeting"}], "domain": "calendar", "action": "set"}2{"id": "10999", "query": "say the list", "intent": "lists_query", "slots": [{}], "domain": "lists", "action": "query"}3{"id": "7890", "query": "clear haircut today", "intent": "calendar_remove", "slots": [{"event_name": "haircut", "date": "today"}], "domain": "calendar", "action": "remove"}4{"id": "614", "query": "start playlist", "intent": "play_music", "slots": [{"player_setting": "playlist"}], "domain": "music", "action": "music"}5{"id": "1593", "query": "set notification from world news", "intent": "news_subscription", "slots": [{"news_topic": "world news"}], "domain": "news", "action": "query"}6{"id": "5635", "query": "ensure i have to get up at six in morning", "intent": "alarm_set", "slots": [{"time": "six", "timeofday": "morning"}], "domain": "alarm", "action": "set"}7{"id": "712", "query": "i want to hear the last news from cnn", "intent": "news_query", "slots": [{"media_type": "cnn", "descriptor": "last"}], "domain": "news", "action": "query"}8{"id": "8100", "query": "email my appointments to reschedule", "intent": "email_sendemail", "slots": [{"content": "reschedule appointments"}], "domain": "email", "action": "sendemail"}9{"id": "2816", "query": "set the living room lights to fifty percent", "intent": "iot_hue_lightother", "slots": [{"house_place": "living room", "change_to": "fifty percent"}], "domain": "iot", "action": "hue_lightchange"}10{"id": "103", "query": "tell me the weather this week", "intent": "weather_query", "slots": [{"date": "this week"}], "domain": "weather", "action": "query"}11{"id": "6118", "query": "hey what time is it in dallas texas", "intent": "datetime_query", "slots": [{"place_name": "dallas texas"}], "domain": "datetime", "action": "query"}12{"id": "2753", "query": "i need a cup of coffee", "intent": "iot_coffee", "slots": [{}], "domain": "iot", "action": "coffee"}13{"id": "13636", "query": "what is arnold schwarzenegger's birthday", "intent": "qa_factoid", "slots": [{"content": "arnold schwarzenegger's birthday"}], "domain": "qa", "action": "factoid"}14{"id": "3146", "query": "play track one from my david bowie playlist", "intent": "play_music", "slots": [{"playlist_name": "david bowie", "index": "1"}], "domain": "music", "action": "music"}15{"id": "5678", "query": "august fifteenth is what day of the week", "intent": "datetime_query", "slots": [{"date": "august fifteenth"}], "domain": "datetime", "action": "query"}16{"id": "13847", "query": "what is the exchange rate for mexico money", "intent": "qa_currency", "slots": [{"currency_name": "mexico"}], "domain": "currency", "action": "currency"}17{"id": "12905", "query": "please reserve the next taxi a nine am", "intent": "transport_taxi", "slots": [{"transport_type": "taxi", "time": "nine am"}], "domain": "transport", "action": "taxi"}18{"id": "2016", "query": "lower lights in kitchen", "intent": "iot_hue_lightdim", "slots": [{"house_place": "kitchen"}], "domain": "iot", "action": "hue_lightdim"}19{"id": "14933", "query": "tell me the info about india's geography", "intent": "qa_factoid", "slots": [{"content": "india's geography"}], "domain": "qa", "action": "factoid"}20{"id": "13348", "query": "i need to ask you how many angels can dance on a pinhead", "intent": "general_quirky", "slots": [{"content": "how many angels can dance on a pinhead"}], "domain": "qa", "action": "quirky"}21{"id": "15032", "query": "how much is one dollar in pounds", "intent": "qa_currency", "slots": [{"currency_name": "pounds"}], "domain": "currency", "action": "currency"}22{"id": "3401", "query": "play the most popular elton john song", "intent": "play_music", "slots": [{"artist_name": "elton john", "descriptor": "most popular"}], "domain": "music", "action": "music"}23{"id": "3407", "query": "are there any forecasts of rain or snow this week", "intent": "weather_query", "slots": [{"weather_descriptor": "rain or snow", "date": "this week"}], "domain": "weather", "action": "query"}24{"id": "7817", "query": "is this true about this date", "intent": "calendar_query", "slots": [{"date": "this date"}], "domain": "calendar", "action": "query"}25{"id": "6621", "query": "latest movies in theatres", "intent": "recommendation_movies", "slots": [{"descriptor": "latest"}], "domain": "recommendation", "action": "recommendation"}26{"id": "13619", "query": "are there any free events on in my area today", "intent": "recommendation_events", "slots": [{"date": "today", "descriptor": "free"}], "domain": "recommendation", "action": "events"}27{"id": "4798", "query": "i want coffee everyday", "intent": "iot_coffee", "slots": [{"setting": "set", "date": "everyday"}], "domain": "iot", "action": "coffee"}28{"id": "2082", "query": "raise the light", "intent": "iot_hue_lightup", "slots": [{}], "domain": "iot", "action": "hue_lightup"}29{"id": "6856", "query": "give me a reminder notification on tomorrow's meeting at ten am", "intent": "calendar_set", "slots": [{"date": "tomorrow", "event_name": "meeting", "time": "ten am"}], "domain": "calendar", "action": "set"}30{"id": "4928", "query": "play my soundtrack", "intent": "play_music", "slots": [{"playlist_name": "my soundtrack"}], "domain": "music", "action": "music"}31{"id": "17138", "query": "please archive my read messages", "intent": "email_query", "slots": [{"setting": "archive", "descriptor": "read messages"}], "domain": "email", "action": "query"}32{"id": "11409", "query": "please skip to the next podcast episode", "intent": "play_podcasts", "slots": [{"player_setting": "next episode"}], "domain": "podcasts", "action": "podcasts"}33{"id": "4680", "query": "should i wear sandals or shoes with socks", "intent": "weather_query", "slots": [{"weather_descriptor": "sandals or shoes with socks"}], "domain": "weather", "action": "query"}34{"id": "8856", "query": "delete all events in calendar", "intent": "calendar_remove", "slots": [{"descriptor": "all"}], "domain": "calendar", "action": "remove"}35{"id": "13457", "query": "tell me when i should leave for a scheduled event so that i am on time", "intent": "calendar_set", "slots": [{}], "domain": "calendar", "action": "set"}36{"id": "9700", "query": "play biography of jackie kennedy", "intent": "play_audiobook", "slots": [{"audiobook_name": "biography of jackie kennedy"}], "domain": "audiobook", "action": "audiobook"}37{"id": "9524", "query": "turn program on xmtune", "intent": "play_radio", "slots": [{"radio_name": "xmtune"}], "domain": "radio", "action": "radio"}38{"id": "11133", "query": "remove the guest list i created last week", "intent": "lists_remove", "slots": [{"list_name": "guest", "date": "last week"}], "domain": "lists", "action": "remove"}39{"id": "8182", "query": "can you delete the dentist appointment", "intent": "calendar_remove", "slots": [{"event_name": "dentist appointment"}], "domain": "calendar", "action": "remove"}40{"id": "12127", "query": "i need to go to dundee at five am on sunday by train", "intent": "transport_ticket", "slots": [{"place_name": "dundee", "time": "five am", "date": "sunday", "transport_type": "train"}], "domain": "transport", "action": "ticket"}41{"id": "1413", "query": "can you make me some coffee olly", "intent": "iot_coffee", "slots": [{}], "domain": "iot", "action": "coffee"}42{"id": "14763", "query": "how much is the dollar worth compared to the euro", "intent": "qa_currency", "slots": [{"currency_name": "euro"}], "domain": "currency", "action": "currency"}43{"id": "3562", "query": "play one more time", "intent": "play_music", "slots": [{"player_setting": "replay"}], "domain": "music", "action": "music"}44{"id": "5411", "query": "what are the top stories on cnn", "intent": "news_query", "slots": [{"media_type": "cnn"}], "domain": "news", "action": "query"}45{"id": "9768", "query": "keep reading the audiobook to me", "intent": "play_audiobook", "slots": [{"player_setting": "resume"}], "domain": "audiobook", "action": "audiobook"}46{"id": "13181", "query": "check for all the movie theater prices and availability for a newly released movie in my location", "intent": "recommendation_movies", "slots": [{"descriptor": "new release"}], "domain": "recommendation", "action": "movies"}47{"id": "4048", "query": "please find out of charley's steakhouse delivers", "intent": "takeaway_query", "slots": [{"business_name": "charley's steakhouse"}], "domain": "takeaway", "action": "query"}48{"id": "10815", "query": "add toothpaste to my shopping list", "intent": "lists_createoradd", "slots": [{"item_name": "toothpaste", "list_name": "shopping"}], "domain": "lists", "action": "createoradd"}49{"id": "6533", "query": "are you happy", "intent": "general_quirky", "slots": [{"content": "are you happy"}], "domain": "qa", "action": "quirky"}50{"id": "11563", "query": "what can i do in berlin", "intent": "recommendation_events", "slots": [{"place_name": "berlin"}], "domain": "recommendation", "action": "events"}51{"id": "4112", "query": "convert current time from g. m. t. to est", "intent": "datetime_convert", "slots": [{"time_zone": "g. m. t.", "time_zone2": "est"}], "domain": "datetime", "action": "convert"}52{"id": "7496", "query": "do i have appointments today", "intent": "calendar_query", "slots": [{"event_name": "appointments", "date": "today"}], "domain": "calendar", "action": "query"}53{"id": "17051", "query": "get me new email", "intent": "email_query", "slots": [{"descriptor": "new"}], "domain": "email", "action": "query"}54{"id": "10577", "query": "give me all my lists", "intent": "lists_query", "slots": [{"descriptor": "all"}], "domain": "lists", "action": "query"}55{"id": "4268", "query": "move to the next song in the list", "intent": "music_settings", "slots": [{"player_setting": "next song"}], "domain": "music", "action": "settings"}56{"id": "9321", "query": "make the radio turn on now", "intent": "play_radio", "slots": [{}], "domain": "radio", "action": "turn on"}57{"id": "15970", "query": "what was the subject of the last email from mom", "intent": "email_query", "slots": [{"descriptor": "last email", "from_relation": "mom"}], "domain": "email", "action": "query"}58{"id": "8904", "query": "information of recent events", "intent": "calendar_query", "slots": [{"time": "recent"}], "domain": "calendar", "action": "query"}59{"id": "8782", "query": "do i have a meeting this afternoon", "intent": "calendar_query", "slots": [{"event_name": "meeting", "timeofday": "this afternoon"}], "domain": "calendar", "action": "query"}60{"id": "3096", "query": "how's the weather like in new jersey", "intent": "weather_query", "slots": [{"place_name": "new jersey"}], "domain": "weather", "action": "query"}61{"id": "5122", "query": "check if default alarm is set", "intent": "alarm_query", "slots": [{"descriptor": "default"}], "domain": "alarm", "action": "query"}62{"id": "6964", "query": "can you remind my next meeting with my boss one hour before it will happen", "intent": "calendar_set", "slots": [{"time": "one hour before", "event_name": "meeting", "relation": "boss"}], "domain": "calendar", "action": "set"}63{"id": "1266", "query": "if it's five am in berlin what time is it here", "intent": "datetime_convert", "slots": [{"time": "five am", "place_name": "berlin"}], "domain": "datetime", "action": "convert"}64{"id": "9030", "query": "remove from calendar my medical appointment for today", "intent": "calendar_remove", "slots": [{"event_name": "medical appointment", "date": "today"}], "domain": "calendar", "action": "remove"}65{"id": "2497", "query": "today is what date", "intent": "datetime_query", "slots": [{"date": "today"}], "domain": "datetime", "action": "query"}66{"id": "9738", "query": "could you please play the classical music for me", "intent": "play_music", "slots": [{"music_genre": "classical"}], "domain": "music", "action": "music"}67{"id": "8084", "query": "what is the event agenda", "intent": "calendar_query", "slots": [{}], "domain": "calendar", "action": "query"}68{"id": "14465", "query": "directions to disney land", "intent": "transport_query", "slots": [{"place_name": "disney land"}], "domain": "transport", "action": "query"}69{"id": "4973", "query": "olly play i can't feel my face", "intent": "play_music", "slots": [{"song_name": "i can't feel my face"}], "domain": "music", "action": "music"}70{"id": "3568", "query": "set my coffee machine", "intent": "iot_coffee", "slots": [{"setting": "set"}], "domain": "iot", "action": "set"}71{"id": "10776", "query": "do i have a grocery list ready", "intent": "lists_query", "slots": [{"list_name": "grocery"}], "domain": "lists", "action": "query"}72{"id": "9955", "query": "show me a video on cooking fried chicken", "intent": "cooking_recipe", "slots": [{"media_type": "video", "food_type": "fried chicken"}], "domain": "cooking", "action": "recipe"}73{"id": "7262", "query": "create a notification reminder for meeting tomorrow", "intent": "calendar_set", "slots": [{"event_name": "meeting", "date": "tomorrow"}], "domain": "calendar", "action": "set"}74{"id": "14272", "query": "hey what's a book", "intent": "qa_definition", "slots": [{"definition_word": "book"}], "domain": "qa", "action": "definition"}75{"id": "16989", "query": "please say to the email that sent me george i need the money now it's urgent", "intent": "email_sendemail", "slots": [{"content": "i need the money now it's urgent", "to_person": "george"}], "domain": "email", "action": "sendemail"}76{"id": "14342", "query": "what is the stock price of hdfc", "intent": "qa_stock", "slots": [{"business_name": "hdfc"}], "domain": "stock", "action": "stock"}77{"id": "9541", "query": "play the sweet song on radio", "intent": "play_radio", "slots": [{"radio_name": "sweet song"}], "domain": "radio", "action": "radio"}78{"id": "7161", "query": "remind this event in future", "intent": "calendar_set", "slots": [{"event_name": "this", "time": "in future"}], "domain": "calendar", "action": "set"}79{"id": "16801", "query": "has cassie sent me any emails about tom", "intent": "email_query", "slots": [{"from_person": "cassie", "content": "tom"}], "domain": "email", "action": "query"}80{"id": "3118", "query": "what is eastern standard time right noe", "intent": "datetime_query", "slots": [{"time_zone": "eastern"}], "domain": "datetime", "action": "query"}81{"id": "15311", "query": "tell me what's going on with facebook", "intent": "social_query", "slots": [{"media_type": "facebook"}], "domain": "social", "action": "query"}82{"id": "263", "query": "let's have the lights blue", "intent": "iot_hue_lightcolor", "slots": [{"color_type": "blue"}], "domain": "iot", "action": "hue_lightchange"}83{"id": "4031", "query": "play pandora nineties dance party", "intent": "play_music", "slots": [{"song_name": "pandora nineties dance party"}], "domain": "music", "action": "music"}84{"id": "4254", "query": "what time is it on the east coast of the united states", "intent": "datetime_query", "slots": [{"time_zone": "east coast of the united states"}], "domain": "datetime", "action": "query"}85{"id": "6909", "query": "what is happening after one and before three pm", "intent": "calendar_query", "slots": [{"time": "one", "time2": "three pm"}], "domain": "calendar", "action": "query"}86{"id": "11455", "query": "move to next section", "intent": "play_podcasts", "slots": [{"player_setting": "next section"}], "domain": "podcasts", "action": "podcasts"}87{"id": "3321", "query": "please make me a cup of coffee", "intent": "iot_coffee", "slots": [{}], "domain": "iot", "action": "coffee"}88{"id": "3175", "query": "turn off the speakers", "intent": "audio_volume_mute", "slots": [{"device_type": "speakers"}], "domain": "iot", "action": "volume_mute"}89{"id": "4088", "query": "turn off overhead light", "intent": "iot_hue_lightoff", "slots": [{"device_type": "overhead light"}], "domain": "iot", "action": "hue_lightoff"}90{"id": "16717", "query": "could you please email john saying i'm on leave", "intent": "email_sendemail", "slots": [{"to_person": "john", "content": "i'm on leave"}], "domain": "email", "action": "sendemail"}91{"id": "11999", "query": "tell me all events for today in my city", "intent": "recommendation_events", "slots": [{"date": "today", "place_name": "my city"}], "domain": "recommendation", "action": "events"}92{"id": "2679", "query": "will the temperature stay about the same or is it going to get colder", "intent": "weather_query", "slots": [{"weather_descriptor": "temperature"}], "domain": "weather", "action": "query"}93{"id": "11514", "query": "olly can we listen to reply all podcast", "intent": "play_podcasts", "slots": [{"podcast_name": "reply all"}], "domain": "podcasts", "action": "podcasts"}94{"id": "9726", "query": "stop audiobook", "intent": "play_audiobook", "slots": [{"player_setting": "stop"}], "domain": "audiobook", "action": "audiobook"}95{"id": "6134", "query": "find out what song this is and tell me the name", "intent": "music_query", "slots": [{"query": "song name"}], "domain": "music", "action": "query"}96{"id": "7295", "query": "please clear my calendar", "intent": "calendar_remove", "slots": [{}], "domain": "calendar", "action": "remove"}97{"id": "6025", "query": "will i need sunscreen this afternoon", "intent": "weather_query", "slots": [{"weather_descriptor": "sunscreen", "timeofday": "this afternoon"}], "domain": "weather", "action": "query"}98{"id": "11384", "query": "can you please download my podcast file available in internet and play it once", "intent": "play_podcasts", "slots": [{"player_setting": "play"}], "domain": "podcasts", "action": "podcasts"}99{"id": "1428", "query": "i'd like some red lighting", "intent": "iot_hue_lightcolor", "slots": [{"color_type": "red"}], "domain": "iot", "action": "hue_lightchange"}100{"id": "3408", "query": "is the weather going to be nice this weekend", "intent": "weather_query", "slots": [{"weather_descriptor": "nice", "date": "this weekend"}], "domain": "weather", "action": "query"}101 