{ "language": "en", "experiments": [ { "model": "logistic", "name": "Logistic Regression", "language": "en", "accuracy": 0.6267150928167877, "precision": 0.5968396594066977, "recall": 0.6267150928167877, "f1": 0.6035602970368393, "macro_f1": 0.37247177314487734, "labels": [ "Alarm_1::AddAlarm", "Alarm_1::GetAlarms", "Banks::CheckBalance", "Banks::TransferMoney", "Buses::BuyBusTicket", "Buses::FindBus", "Calendar_1::AddEvent", "Calendar_1::GetAvailableTime", "Calendar_1::GetEvents", "Events::BuyEventTickets", "Events::FindEvents", "Events::GetEventDates", "Flights::ReserveOnewayFlight", "Flights::ReserveRoundtripFlights", "Flights::SearchOnewayFlight", "Flights::SearchRoundtripFlights", "Homes::FindApartment", "Homes::FindHomeByArea", "Homes::ScheduleVisit", "Hotels::BookHouse", "Hotels::ReserveHotel", "Hotels::SearchHotel", "Hotels::SearchHouse", "Media::FindMovies", "Media::PlayMovie", "Messaging_1::ShareLocation", "Movies::BuyMovieTickets", "Movies::FindMovies", "Movies::GetTimesForMovie", "Music::LookupMusic", "Music::LookupSong", "Music::PlayMedia", "Music::PlaySong", "Payment_1::MakePayment", "Payment_1::RequestPayment", "RentalCars::GetCarsAvailable", "RentalCars::ReserveCar", "Restaurants::FindRestaurants", "Restaurants::ReserveRestaurant", "RideSharing::GetRide", "Services_1::BookAppointment", "Services_1::FindProvider", "Services_2::BookAppointment", "Services_2::FindProvider", "Services_3::BookAppointment", "Services_3::FindProvider", "Services_4::BookAppointment", "Services_4::FindProvider", "Trains_1::FindTrains", "Trains_1::GetTrainTickets", "Travel_1::FindAttractions", "Weather_1::GetWeather" ], "confusion_matrix": [ [ 0, 0, 0, 0, 1, 0, 82, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 1, 2, 2, 0, 4, 0, 2, 0, 4, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 12, 0, 0, 0, 1, 74, 15, 0, 1, 1, 1, 0, 0, 0, 0, 0, 6, 0, 0, 10, 6, 1, 4, 0, 0, 12, 4, 0, 3, 2, 2, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 1 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 142, 24, 3, 4, 0, 38, 0, 0, 5, 13, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 12, 351, 0, 5, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 13, 0, 5, 3, 0, 271, 6, 4, 2, 4, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 5, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 2, 0, 6, 0, 6, 344, 5, 2, 0, 1, 1, 0, 0, 0, 0, 0, 3, 2, 10, 5, 0, 0, 13, 3, 10, 5, 1, 4, 0, 0, 1, 0, 3, 1, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 5, 14, 0, 2, 0, 1, 0, 1, 3, 2, 99, 35, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 2, 12, 0, 5, 0, 2, 0, 0, 1, 1, 8, 247, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 33, 0, 0, 1, 0, 0, 19, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 1, 0, 1, 0, 3, 2, 0, 0, 0, 0, 0, 1, 1, 0, 7, 0, 178, 2, 0, 3, 6, 0, 0, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 1, 1, 1, 1, 6, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 97, 20, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 7, 0, 6, 1, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 3, 1, 3, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 1, 164, 25, 3, 0, 1, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 1, 3, 1, 6, 1, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 1, 0, 2, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 391, 17, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0 ], [ 0, 0, 0, 0, 1, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 2, 0, 9, 347, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 1, 2, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 3, 0, 0, 23, 1, 1, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 1, 0, 0, 0, 7, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 183, 0, 0, 10, 0, 0, 0, 7, 5, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 2, 7, 0, 3, 16, 3, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 17, 3, 1, 4, 0, 0, 0, 3, 0, 0, 1, 1, 0, 0, 1, 17, 1, 2, 149, 4, 0, 3, 1, 2, 0, 0, 0, 0, 0, 6, 0 ], [ 0, 0, 0, 0, 2, 0, 0, 0, 0, 33, 0, 0, 2, 14, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 17, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 14, 107, 14, 0, 0, 261, 4, 1, 3, 2, 0, 0, 0, 0, 0, 6, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 1, 0, 0, 0, 3, 80, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1 ], [ 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 6, 0, 0, 0, 0, 0, 0, 96, 141, 4, 9, 0, 0, 0, 0, 1, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 1, 0, 2, 0, 0, 0, 0, 7, 17, 134, 104, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 4, 116, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 64, 2, 1, 6, 3, 0, 1, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 1, 0, 7, 3, 15, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 23, 1, 3, 1, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 1, 0, 4, 4, 1, 1, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 3, 5, 4, 3, 0, 3, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 74, 215, 6, 6, 7, 2, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 310, 2, 0, 0, 1, 3, 4, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 6, 311, 1, 2, 0, 6, 0, 4, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 204, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 20, 1, 164, 5, 9, 0, 31, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 8, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 3, 1, 13, 3, 137, 0, 23, 0, 43, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 6, 1, 0, 0, 1, 16, 1, 0, 1, 2, 3, 0, 0, 1, 0, 0, 6, 3, 1, 4, 0, 0, 19, 1, 6, 2, 2, 0, 0, 0, 1, 0, 10, 0, 3, 1, 2, 2, 66, 5, 89, 0, 0, 0, 0, 8, 1 ], [ 0, 0, 0, 0, 25, 100, 0, 7, 0, 30, 15, 4, 5, 23, 9, 37, 0, 0, 1, 0, 0, 3, 1, 0, 8, 0, 0, 2, 0, 12, 0, 0, 0, 0, 0, 0, 0, 7, 1, 13, 0, 0, 0, 4, 0, 8, 0, 0, 0, 0, 1, 1 ], [ 0, 0, 0, 0, 26, 3, 6, 0, 0, 12, 0, 0, 21, 53, 2, 3, 0, 0, 1, 1, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 1, 9, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 6, 0, 0, 1, 6, 3, 0, 0, 13, 5, 0, 0, 0, 1, 0, 0, 0, 0, 3, 7, 13, 0, 1, 0, 0, 1, 0, 0, 2, 1, 0, 0, 0, 0, 2, 2, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 444, 1 ], [ 0, 0, 1, 0, 1, 0, 0, 7, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 6, 358 ] ], "report": { "Alarm_1::AddAlarm": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 106.0 }, "Alarm_1::GetAlarms": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 184.0 }, "Banks::CheckBalance": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Banks::TransferMoney": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Buses::BuyBusTicket": { "precision": 0.5612648221343873, "recall": 0.5892116182572614, "f1-score": 0.5748987854251012, "support": 241.0 }, "Buses::FindBus": { "precision": 0.6698473282442748, "recall": 0.9212598425196851, "f1-score": 0.7756906077348066, "support": 381.0 }, "Calendar_1::AddEvent": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Calendar_1::GetAvailableTime": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Calendar_1::GetEvents": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Events::BuyEventTickets": { "precision": 0.6674876847290641, "recall": 0.8287461773700305, "f1-score": 0.7394270122783083, "support": 327.0 }, "Events::FindEvents": { "precision": 0.8329297820823245, "recall": 0.7782805429864253, "f1-score": 0.8046783625730994, "support": 442.0 }, "Events::GetEventDates": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::ReserveOnewayFlight": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::ReserveRoundtripFlights": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::SearchOnewayFlight": { "precision": 0.7983870967741935, "recall": 0.5755813953488372, "f1-score": 0.668918918918919, "support": 172.0 }, "Flights::SearchRoundtripFlights": { "precision": 0.7286135693215339, "recall": 0.8821428571428571, "f1-score": 0.7980613893376414, "support": 280.0 }, "Homes::FindApartment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Homes::FindHomeByArea": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 139.0 }, "Homes::ScheduleVisit": { "precision": 0.8356807511737089, "recall": 0.7911111111111111, "f1-score": 0.8127853881278538, "support": 225.0 }, "Hotels::BookHouse": { "precision": 0.8151260504201681, "recall": 0.6689655172413793, "f1-score": 0.7348484848484849, "support": 145.0 }, "Hotels::ReserveHotel": { "precision": 0.7288888888888889, "recall": 0.7068965517241379, "f1-score": 0.7177242888402626, "support": 232.0 }, "Hotels::SearchHotel": { "precision": 0.7562862669245648, "recall": 0.8786516853932584, "f1-score": 0.8128898128898129, "support": 445.0 }, "Hotels::SearchHouse": { "precision": 0.7609649122807017, "recall": 0.9083769633507853, "f1-score": 0.8281622911694511, "support": 382.0 }, "Media::FindMovies": { "precision": 0.6621983914209115, "recall": 0.8636363636363636, "f1-score": 0.7496206373292867, "support": 286.0 }, "Media::PlayMovie": { "precision": 0.7721518987341772, "recall": 0.8280542986425339, "f1-score": 0.7991266375545851, "support": 221.0 }, "Messaging_1::ShareLocation": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 255.0 }, "Movies::BuyMovieTickets": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 80.0 }, "Movies::FindMovies": { "precision": 0.7054054054054054, "recall": 0.6184834123222749, "f1-score": 0.6590909090909091, "support": 422.0 }, "Movies::GetTimesForMovie": { "precision": 0.7547169811320755, "recall": 0.7619047619047619, "f1-score": 0.7582938388625592, "support": 105.0 }, "Music::LookupMusic": { "precision": 0.6906474820143885, "recall": 0.35555555555555557, "f1-score": 0.46943765281173594, "support": 270.0 }, "Music::LookupSong": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Music::PlayMedia": { "precision": 0.7701149425287356, "recall": 0.46689895470383275, "f1-score": 0.5813449023861171, "support": 287.0 }, "Music::PlaySong": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Payment_1::MakePayment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 134.0 }, "Payment_1::RequestPayment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 122.0 }, "RentalCars::GetCarsAvailable": { "precision": 0.7949367088607595, "recall": 0.8626373626373627, "f1-score": 0.8274044795783926, "support": 364.0 }, "RentalCars::ReserveCar": { "precision": 0.7413793103448276, "recall": 0.6160458452722063, "f1-score": 0.672926447574335, "support": 349.0 }, "Restaurants::FindRestaurants": { "precision": 0.6724511930585684, "recall": 0.9011627906976745, "f1-score": 0.7701863354037267, "support": 344.0 }, "Restaurants::ReserveRestaurant": { "precision": 0.783375314861461, "recall": 0.9014492753623189, "f1-score": 0.8382749326145552, "support": 345.0 }, "RideSharing::GetRide": { "precision": 0.477751756440281, "recall": 0.9532710280373832, "f1-score": 0.6365054602184087, "support": 214.0 }, "Services_1::BookAppointment": { "precision": 0.5, "recall": 0.6332046332046332, "f1-score": 0.5587734241908007, "support": 259.0 }, "Services_1::FindProvider": { "precision": 0.9629629629629629, "recall": 0.9961685823754789, "f1-score": 0.9792843691148776, "support": 261.0 }, "Services_2::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_2::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_3::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_3::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_4::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 248.0 }, "Services_4::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 265.0 }, "Trains_1::FindTrains": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 317.0 }, "Trains_1::GetTrainTickets": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 157.0 }, "Travel_1::FindAttractions": { "precision": 0.8393194706994329, "recall": 0.8621359223300971, "f1-score": 0.8505747126436781, "support": 515.0 }, "Weather_1::GetWeather": { "precision": 0.9862258953168044, "recall": 0.9156010230179028, "f1-score": 0.9496021220159151, "support": 391.0 }, "accuracy": 0.6267150928167877, "macro avg": { "precision": 0.3705599012837423, "recall": 0.3858737321566567, "f1-score": 0.37247177314487734, "support": 9912.0 }, "weighted avg": { "precision": 0.5968396594066977, "recall": 0.6267150928167877, "f1-score": 0.6035602970368393, "support": 9912.0 } }, "train_samples": 37201, "dev_samples": 5324, "test_samples": 9912, "unseen_test_classes": [ "Alarm_1::AddAlarm", "Alarm_1::GetAlarms", "Homes::FindHomeByArea", "Messaging_1::ShareLocation", "Movies::BuyMovieTickets", "Payment_1::MakePayment", "Payment_1::RequestPayment", "Services_4::BookAppointment", "Services_4::FindProvider", "Trains_1::FindTrains", "Trains_1::GetTrainTickets" ], "errors": [ { "text": "My boss from headquarters is coming to town and I would like to treat him and his wife to dinner. Can you help me find a restaurant in San Francisco with available seating for four people, next Tuesday?", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "I would like to take this girl I like out for a casual meal on the 3rd of this month, when I get paid so something around eleven am would be perfect. Can you help me locate a restaurant in Palo Alto?", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "I need to make a hotel reservation. Please help me find a 4 star hotel in London.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "I would like to book a hotel in Washington D.C.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "I need to book a hotel please.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "That works for me. I'd like to make a reservation.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Alright make the arrival time for 4th of March.", "actual": "Hotels::ReserveHotel", "predicted": "Calendar_1::GetAvailableTime" }, { "text": "Yes. I want to make a reservation please.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes, I need to reserve it now and do it for me.", "actual": "Hotels::ReserveHotel", "predicted": "RentalCars::ReserveCar" }, { "text": "Yes. I wish to reserve it.", "actual": "Hotels::ReserveHotel", "predicted": "RentalCars::ReserveCar" }, { "text": "yes I am looking to book", "actual": "Hotels::ReserveHotel", "predicted": "Hotels::BookHouse" }, { "text": "Sounds nice. Please make me a reservation there.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes if you would please.", "actual": "Music::PlayMedia", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Can you recommend a song to listen to?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes, I'd like to listen to it now.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Please find me any of the songs that I've played the most this week.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Can you search for some songs from 2015?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Oh yes, that is the one I wanted to hear. Please play the song on any device.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Hey, I feel like listening to some tunes right now. Can you find me something from two years ago?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Please do. Cast it on the living room device.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "I'd like to look for music right now.", "actual": "Music::LookupMusic", "predicted": "Music::PlaySong" }, { "text": "Yup. Play it on patio speaker.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Hello, I want to find songs please, ones like on the album The New Classic. Or tunes from, say, six years back.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes, and please do it in the Living room.", "actual": "Music::PlayMedia", "predicted": "Hotels::ReserveHotel" }, { "text": "I would like to search for songs at this time.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" } ], "protocol": "SGD official train/dev/test, all shards; only user turns that start or change the active intent (confirmations, slot answers and thanks excluded); schema-reviewed domain-qualified intents, API instance omitted for equivalent tasks; Services subtypes preserved; unseen test classes retained; exact-text deduplication", "trained_at": "2026-10-25T10:17:58.845520+00:00", "seed": 42, "dataset_sha256": "4580ee54b4981f2026e4ea9d21e63eb6855c62c6327366927d13b181be77593c", "best_c": 2.0, "dev_macro_f1": 0.4053650869655293, "confidence_kind": "uncalibrated_probability", "slot_metrics": { "available": true, "evaluated_samples": 30397, "precision": 0.024566790962930466, "recall": 0.019817163078737834, "f1": 0.02193784277879342, "true_positive": 336, "false_positive": 13341, "false_negative": 16619, "protocol": "Train-only unambiguous slot-value lexicon; exact type + start + end micro score; non-categorical slots only" }, "pilot": false, "seen_accuracy": 0.785831752055661, "seen_macro_f1": 0.7449435462897548, "seen_test_samples": 7905, "unseen_test_samples": 2007, "unseen_class_count": 11 }, { "model": "svm", "name": "Linear SVM", "language": "en", "accuracy": 0.6257062146892656, "precision": 0.5928969768949391, "recall": 0.6257062146892656, "f1": 0.6015392744084022, "macro_f1": 0.37064300556159274, "labels": [ "Alarm_1::AddAlarm", "Alarm_1::GetAlarms", "Banks::CheckBalance", "Banks::TransferMoney", "Buses::BuyBusTicket", "Buses::FindBus", "Calendar_1::AddEvent", "Calendar_1::GetAvailableTime", "Calendar_1::GetEvents", "Events::BuyEventTickets", "Events::FindEvents", "Events::GetEventDates", "Flights::ReserveOnewayFlight", "Flights::ReserveRoundtripFlights", "Flights::SearchOnewayFlight", "Flights::SearchRoundtripFlights", "Homes::FindApartment", "Homes::FindHomeByArea", "Homes::ScheduleVisit", "Hotels::BookHouse", "Hotels::ReserveHotel", "Hotels::SearchHotel", "Hotels::SearchHouse", "Media::FindMovies", "Media::PlayMovie", "Messaging_1::ShareLocation", "Movies::BuyMovieTickets", "Movies::FindMovies", "Movies::GetTimesForMovie", "Music::LookupMusic", "Music::LookupSong", "Music::PlayMedia", "Music::PlaySong", "Payment_1::MakePayment", "Payment_1::RequestPayment", "RentalCars::GetCarsAvailable", "RentalCars::ReserveCar", "Restaurants::FindRestaurants", "Restaurants::ReserveRestaurant", "RideSharing::GetRide", "Services_1::BookAppointment", "Services_1::FindProvider", "Services_2::BookAppointment", "Services_2::FindProvider", "Services_3::BookAppointment", "Services_3::FindProvider", "Services_4::BookAppointment", "Services_4::FindProvider", "Trains_1::FindTrains", "Trains_1::GetTrainTickets", "Travel_1::FindAttractions", "Weather_1::GetWeather" ], "confusion_matrix": [ [ 0, 0, 0, 0, 1, 0, 84, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 4, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 16, 0, 1, 1, 1, 68, 19, 0, 2, 4, 1, 0, 0, 0, 0, 0, 3, 0, 0, 12, 4, 0, 4, 0, 0, 3, 0, 2, 3, 2, 8, 0, 0, 0, 0, 3, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 23, 1 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 137, 26, 2, 4, 0, 40, 0, 0, 4, 13, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 3, 0, 1, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 7, 348, 0, 3, 0, 1, 0, 0, 0, 0, 5, 6, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 10, 0, 3, 3, 0, 275, 7, 5, 2, 4, 0, 0, 1, 0, 1, 0, 1, 3, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 1, 0, 4, 0, 7, 336, 6, 0, 0, 2, 2, 0, 0, 0, 0, 0, 2, 1, 13, 2, 0, 0, 10, 5, 11, 3, 1, 10, 0, 0, 2, 0, 3, 1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 17, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 4, 14, 0, 2, 0, 0, 0, 0, 2, 2, 101, 40, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 2, 13, 0, 5, 0, 1, 0, 0, 1, 3, 9, 244, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 49, 0, 0, 1, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 1, 0, 1, 0, 3, 2, 0, 0, 0, 0, 0, 1, 1, 0, 9, 0, 172, 2, 0, 1, 7, 0, 1, 0, 0, 2, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 6, 0, 2, 0, 4, 0, 0, 0, 0, 0, 4, 0 ], [ 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 93, 21, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 6, 0, 8, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 4, 1, 3, 2, 0, 1, 0, 0, 2, 1, 0, 1, 0, 0, 0, 1, 165, 25, 3, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 1, 3, 1, 7, 2, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 1, 0, 2, 7, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 9, 391, 18, 0, 0, 0, 0, 0, 0, 2, 2, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0 ], [ 0, 0, 0, 0, 0, 1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 2, 2, 0, 9, 345, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 1, 3, 0, 1, 0, 0, 0, 0, 0, 5, 0 ], [ 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 248, 4, 0, 0, 21, 1, 1, 2, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 2, 181, 0, 0, 10, 3, 0, 0, 7, 5, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 1, 9, 0, 2, 9, 2, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 2, 1, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 19, 0, 2, 176, 4, 0, 2, 0, 1, 0, 0, 0, 0, 0, 7, 0 ], [ 0, 0, 0, 0, 2, 0, 0, 0, 0, 29, 0, 0, 2, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 25, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 11, 108, 10, 0, 0, 258, 7, 1, 4, 2, 0, 0, 0, 0, 0, 7, 0, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0 ], [ 0, 0, 0, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 4, 0, 0, 0, 0, 2, 81, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 1 ], [ 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 98, 135, 3, 14, 0, 0, 0, 0, 1, 1, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 0, 2, 2, 0, 1, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 6, 1, 2, 0, 3, 0, 0, 1, 0, 7, 15, 140, 92, 0, 0, 0, 2, 0, 2, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 3, 125, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 95, 2, 1, 3, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 20, 0, 3, 1, 3, 0, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 4, 3, 0, 1, 0, 3, 0, 0, 2, 3, 1, 1, 0, 0, 4, 5, 6, 2, 3, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 75, 217, 7, 3, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 311, 2, 0, 0, 1, 2, 4, 1, 1, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 1, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6, 9, 307, 1, 2, 0, 6, 0, 3, 0, 0, 0, 0, 0, 0, 1 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 9, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 27, 1, 160, 9, 8, 1, 27, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 259, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 0, 1, 1, 0, 2, 0, 0, 2, 2, 1, 0, 0, 0, 8, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 2, 22, 0, 142, 0, 21, 0, 33, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 10, 0, 0, 0, 3, 11, 5, 1, 2, 1, 3, 0, 0, 1, 0, 0, 10, 3, 3, 3, 0, 0, 28, 3, 4, 1, 2, 0, 0, 0, 2, 0, 11, 1, 3, 2, 6, 0, 38, 2, 100, 0, 0, 0, 0, 4, 1 ], [ 0, 0, 0, 0, 14, 115, 0, 5, 0, 30, 8, 5, 3, 18, 12, 36, 0, 0, 1, 0, 2, 1, 0, 0, 9, 0, 0, 1, 0, 31, 0, 0, 0, 0, 0, 3, 0, 4, 1, 10, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 1 ], [ 0, 0, 0, 0, 20, 4, 6, 0, 0, 20, 0, 0, 18, 44, 11, 2, 0, 0, 1, 4, 6, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 6, 0, 7, 2, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 5, 0, 1, 2, 5, 2, 0, 0, 15, 2, 0, 0, 0, 2, 0, 0, 1, 0, 2, 5, 12, 0, 1, 0, 0, 1, 0, 0, 3, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1 ], [ 0, 0, 2, 0, 0, 1, 0, 5, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 0, 1, 0, 2, 1, 0, 0, 0, 0, 0, 9, 357 ] ], "report": { "Alarm_1::AddAlarm": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 106.0 }, "Alarm_1::GetAlarms": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 184.0 }, "Banks::CheckBalance": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Banks::TransferMoney": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Buses::BuyBusTicket": { "precision": 0.6255707762557078, "recall": 0.5684647302904564, "f1-score": 0.5956521739130435, "support": 241.0 }, "Buses::FindBus": { "precision": 0.6373626373626373, "recall": 0.9133858267716536, "f1-score": 0.7508090614886731, "support": 381.0 }, "Calendar_1::AddEvent": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Calendar_1::GetAvailableTime": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Calendar_1::GetEvents": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Events::BuyEventTickets": { "precision": 0.6395348837209303, "recall": 0.8409785932721713, "f1-score": 0.726552179656539, "support": 327.0 }, "Events::FindEvents": { "precision": 0.8615384615384616, "recall": 0.7601809954751131, "f1-score": 0.8076923076923077, "support": 442.0 }, "Events::GetEventDates": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::ReserveOnewayFlight": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::ReserveRoundtripFlights": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::SearchOnewayFlight": { "precision": 0.6917808219178082, "recall": 0.5872093023255814, "f1-score": 0.6352201257861635, "support": 172.0 }, "Flights::SearchRoundtripFlights": { "precision": 0.7176470588235294, "recall": 0.8714285714285714, "f1-score": 0.7870967741935484, "support": 280.0 }, "Homes::FindApartment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Homes::FindHomeByArea": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 139.0 }, "Homes::ScheduleVisit": { "precision": 0.8431372549019608, "recall": 0.7644444444444445, "f1-score": 0.8018648018648019, "support": 225.0 }, "Hotels::BookHouse": { "precision": 0.7815126050420168, "recall": 0.6413793103448275, "f1-score": 0.7045454545454546, "support": 145.0 }, "Hotels::ReserveHotel": { "precision": 0.7112068965517241, "recall": 0.7112068965517241, "f1-score": 0.7112068965517241, "support": 232.0 }, "Hotels::SearchHotel": { "precision": 0.791497975708502, "recall": 0.8786516853932584, "f1-score": 0.832800851970181, "support": 445.0 }, "Hotels::SearchHouse": { "precision": 0.756578947368421, "recall": 0.9031413612565445, "f1-score": 0.8233890214797136, "support": 382.0 }, "Media::FindMovies": { "precision": 0.656084656084656, "recall": 0.8671328671328671, "f1-score": 0.7469879518072289, "support": 286.0 }, "Media::PlayMovie": { "precision": 0.8080357142857143, "recall": 0.8190045248868778, "f1-score": 0.8134831460674158, "support": 221.0 }, "Messaging_1::ShareLocation": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 255.0 }, "Movies::BuyMovieTickets": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 80.0 }, "Movies::FindMovies": { "precision": 0.7010869565217391, "recall": 0.6113744075829384, "f1-score": 0.6531645569620254, "support": 422.0 }, "Movies::GetTimesForMovie": { "precision": 0.75, "recall": 0.7714285714285715, "f1-score": 0.7605633802816901, "support": 105.0 }, "Music::LookupMusic": { "precision": 0.6125, "recall": 0.362962962962963, "f1-score": 0.4558139534883721, "support": 270.0 }, "Music::LookupSong": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Music::PlayMedia": { "precision": 0.7777777777777778, "recall": 0.4878048780487805, "f1-score": 0.5995717344753747, "support": 287.0 }, "Music::PlaySong": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Payment_1::MakePayment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 134.0 }, "Payment_1::RequestPayment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 122.0 }, "RentalCars::GetCarsAvailable": { "precision": 0.7785888077858881, "recall": 0.8791208791208791, "f1-score": 0.8258064516129032, "support": 364.0 }, "RentalCars::ReserveCar": { "precision": 0.7508650519031141, "recall": 0.6217765042979942, "f1-score": 0.6802507836990596, "support": 349.0 }, "Restaurants::FindRestaurants": { "precision": 0.7266355140186916, "recall": 0.9040697674418605, "f1-score": 0.805699481865285, "support": 344.0 }, "Restaurants::ReserveRestaurant": { "precision": 0.7542997542997543, "recall": 0.8898550724637682, "f1-score": 0.8164893617021277, "support": 345.0 }, "RideSharing::GetRide": { "precision": 0.4793577981651376, "recall": 0.9766355140186916, "f1-score": 0.6430769230769231, "support": 214.0 }, "Services_1::BookAppointment": { "precision": 0.4804804804804805, "recall": 0.6177606177606177, "f1-score": 0.5405405405405406, "support": 259.0 }, "Services_1::FindProvider": { "precision": 0.9316546762589928, "recall": 0.9923371647509579, "f1-score": 0.961038961038961, "support": 261.0 }, "Services_2::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_2::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_3::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_3::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_4::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 248.0 }, "Services_4::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 265.0 }, "Trains_1::FindTrains": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 317.0 }, "Trains_1::GetTrainTickets": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 157.0 }, "Travel_1::FindAttractions": { "precision": 0.8238532110091743, "recall": 0.8718446601941747, "f1-score": 0.8471698113207548, "support": 515.0 }, "Weather_1::GetWeather": { "precision": 0.9834710743801653, "recall": 0.9130434782608695, "f1-score": 0.946949602122016, "support": 391.0 }, "accuracy": 0.6257062146892656, "macro avg": { "precision": 0.36677038061851897, "recall": 0.3851273766905223, "f1-score": 0.37064300556159274, "support": 9912.0 }, "weighted avg": { "precision": 0.5928969768949391, "recall": 0.6257062146892656, "f1-score": 0.6015392744084022, "support": 9912.0 } }, "train_samples": 37201, "dev_samples": 5324, "test_samples": 9912, "unseen_test_classes": [ "Alarm_1::AddAlarm", "Alarm_1::GetAlarms", "Homes::FindHomeByArea", "Messaging_1::ShareLocation", "Movies::BuyMovieTickets", "Payment_1::MakePayment", "Payment_1::RequestPayment", "Services_4::BookAppointment", "Services_4::FindProvider", "Trains_1::FindTrains", "Trains_1::GetTrainTickets" ], "errors": [ { "text": "Hi, could you get me a restaurant booking on the 8th please?", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "Can you check restaurants in Pacifica, I'm looking to make a reservation.", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "My boss from headquarters is coming to town and I would like to treat him and his wife to dinner. Can you help me find a restaurant in San Francisco with available seating for four people, next Tuesday?", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "I would like to take this girl I like out for a casual meal on the 3rd of this month, when I get paid so something around eleven am would be perfect. Can you help me locate a restaurant in Palo Alto?", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "I would like to book a hotel in Washington D.C.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "I need to book a hotel please.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "That works for me. I'd like to make a reservation.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Sure, could you make a reservation on the 4th of March please?", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes please. I need 3 rooms and we're staying 2 night", "actual": "Hotels::ReserveHotel", "predicted": "Hotels::SearchHotel" }, { "text": "Yes. I want to make a reservation please.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes, I need to reserve it now and do it for me.", "actual": "Hotels::ReserveHotel", "predicted": "RentalCars::ReserveCar" }, { "text": "Yes. I wish to reserve it.", "actual": "Hotels::ReserveHotel", "predicted": "RentalCars::ReserveCar" }, { "text": "yes I am looking to book", "actual": "Hotels::ReserveHotel", "predicted": "Hotels::BookHouse" }, { "text": "Sounds nice. Please make me a reservation there.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes if you would please.", "actual": "Music::PlayMedia", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Can you recommend a song to listen to?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes, I'd like to listen to it now.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Can you search for some songs from 2015?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Hey, I feel like listening to some tunes right now. Can you find me something from two years ago?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Please do. Cast it on the living room device.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "I'd like to look for music right now.", "actual": "Music::LookupMusic", "predicted": "Music::PlaySong" }, { "text": "Yup. Play it on patio speaker.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Hello, I want to find songs please, ones like on the album The New Classic. Or tunes from, say, six years back.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes, and please do it in the Living room.", "actual": "Music::PlayMedia", "predicted": "Hotels::ReserveHotel" }, { "text": "I would like to search for songs at this time.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" } ], "protocol": "SGD official train/dev/test, all shards; only user turns that start or change the active intent (confirmations, slot answers and thanks excluded); schema-reviewed domain-qualified intents, API instance omitted for equivalent tasks; Services subtypes preserved; unseen test classes retained; exact-text deduplication", "trained_at": "2026-10-25T10:18:14.921096+00:00", "seed": 42, "dataset_sha256": "4580ee54b4981f2026e4ea9d21e63eb6855c62c6327366927d13b181be77593c", "best_c": 0.5, "dev_macro_f1": 0.4026634471734219, "confidence_kind": null, "slot_metrics": { "available": true, "evaluated_samples": 30397, "precision": 0.024566790962930466, "recall": 0.019817163078737834, "f1": 0.02193784277879342, "true_positive": 336, "false_positive": 13341, "false_negative": 16619, "protocol": "Train-only unambiguous slot-value lexicon; exact type + start + end micro score; non-categorical slots only" }, "pilot": false, "seen_accuracy": 0.7845667299177735, "seen_macro_f1": 0.7412860111231857, "seen_test_samples": 7905, "unseen_test_samples": 2007, "unseen_class_count": 11 }, { "model": "mbert", "name": "Multilingual BERT", "language": "en", "accuracy": 0.6251008878127522, "precision": 0.6097753649101566, "recall": 0.6251008878127522, "f1": 0.6006828056612206, "macro_f1": 0.37912842142342584, "labels": [ "Alarm_1::AddAlarm", "Alarm_1::GetAlarms", "Banks::CheckBalance", "Banks::TransferMoney", "Buses::BuyBusTicket", "Buses::FindBus", "Calendar_1::AddEvent", "Calendar_1::GetAvailableTime", "Events::BuyEventTickets", "Events::FindEvents", "Events::GetEventDates", "Flights::ReserveOnewayFlight", "Flights::ReserveRoundtripFlights", "Flights::SearchOnewayFlight", "Flights::SearchRoundtripFlights", "Homes::FindApartment", "Homes::FindHomeByArea", "Homes::ScheduleVisit", "Hotels::BookHouse", "Hotels::ReserveHotel", "Hotels::SearchHotel", "Hotels::SearchHouse", "Media::FindMovies", "Media::PlayMovie", "Messaging_1::ShareLocation", "Movies::BuyMovieTickets", "Movies::FindMovies", "Movies::GetTimesForMovie", "Music::LookupMusic", "Music::LookupSong", "Music::PlayMedia", "Music::PlaySong", "Payment_1::MakePayment", "Payment_1::RequestPayment", "RentalCars::GetCarsAvailable", "RentalCars::ReserveCar", "Restaurants::FindRestaurants", "Restaurants::ReserveRestaurant", "RideSharing::GetRide", "Services_1::BookAppointment", "Services_1::FindProvider", "Services_2::BookAppointment", "Services_2::FindProvider", "Services_3::BookAppointment", "Services_3::FindProvider", "Services_4::BookAppointment", "Services_4::FindProvider", "Trains_1::FindTrains", "Trains_1::GetTrainTickets", "Travel_1::FindAttractions", "Weather_1::GetWeather" ], "confusion_matrix": [ [ 0, 0, 0, 0, 0, 0, 66, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2, 0, 7, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 1, 0, 0, 8, 0, 7, 0, 0, 0, 0, 0, 0, 1 ], [ 0, 0, 1, 0, 0, 5, 1, 63, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 72, 4, 0, 1, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 139, 24, 1, 2, 38, 0, 0, 4, 14, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 7, 1, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 367, 0, 1, 1, 2, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 13, 2, 0, 4, 272, 8, 1, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 6, 0, 1, 2, 354, 0, 1, 1, 2, 1, 0, 0, 0, 0, 1, 4, 0, 9, 0, 0, 0, 27, 9, 1, 6, 1, 1, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 20, 1, 2, 0, 2, 0, 0, 1, 101, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 0, 21, 0, 3, 1, 1, 0, 0, 0, 9, 242, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 37, 0, 0, 1, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 3, 5, 0, 172, 3, 3, 0, 5, 0, 0, 0, 0, 7, 1, 0, 1, 0, 0, 0, 0, 0, 3, 0, 1, 5, 3, 0, 3, 0, 3, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 89, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 14, 0, 9, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 1, 3, 5, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 173, 11, 0, 0, 1, 0, 0, 0, 0, 0, 1, 2, 2, 0, 0, 3, 6, 0, 5, 1, 1, 0, 3, 0, 4, 0, 0, 0, 0, 0, 1, 0 ], [ 0, 0, 0, 0, 1, 2, 0, 11, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 391, 12, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0 ], [ 0, 0, 0, 0, 1, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 12, 324, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 2, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 23, 1 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 2, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 179, 0, 0, 21, 0, 0, 1, 1, 4, 0, 0, 0, 2, 1, 0, 3, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 13, 0, 0, 5, 1, 9, 0, 1, 0, 0, 2, 0, 0, 0, 3, 0, 2, 16, 10, 8, 10, 0, 0, 8, 2, 0, 1, 0, 24, 0, 0, 1, 14, 6, 1, 66, 2, 1, 46, 1, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 1, 0, 0, 0, 27, 0, 0, 0, 6, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 31, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 2, 1, 118, 2, 0, 0, 279, 0, 0, 0, 1, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 3, 2, 0, 0, 0, 4, 78, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0 ], [ 0, 0, 0, 0, 0, 0, 1, 1, 3, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 2, 0, 52, 183, 2, 12, 0, 0, 0, 3, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 4, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4, 1, 0, 0, 3, 0, 0, 0, 0, 0, 10, 109, 135, 0, 0, 0, 6, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 8, 120, 0, 1, 0, 0, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 114, 0, 0, 5, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 341, 2, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 1, 2, 1, 2, 5, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 1, 1, 5, 5, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 86, 218, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 316, 2, 4, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 7, 308, 3, 2, 0, 4, 0, 4, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 6, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3, 3, 0, 10, 3, 163, 6, 13, 0, 39, 0, 0, 0, 0, 0, 2, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 258, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 2, 0, 2, 0, 4, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 7, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 5, 1, 157, 0, 34, 0, 25, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 48, 1, 205, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 18, 269, 0, 1, 10, 0, 1, 0, 1, 2, 5, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 40, 7, 7, 0, 8, 0, 0, 11, 32, 8, 18, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 2, 0, 0, 2, 1, 5, 3, 27, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 9, 5, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 450, 0 ], [ 0, 0, 1, 0, 1, 0, 0, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 1, 0, 2, 2, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 2, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 9, 361 ] ], "report": { "Alarm_1::AddAlarm": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 106.0 }, "Alarm_1::GetAlarms": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 184.0 }, "Banks::CheckBalance": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Banks::TransferMoney": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Buses::BuyBusTicket": { "precision": 0.6205357142857143, "recall": 0.5767634854771784, "f1-score": 0.5978494623655914, "support": 241.0 }, "Buses::FindBus": { "precision": 0.4986413043478261, "recall": 0.963254593175853, "f1-score": 0.657117278424351, "support": 381.0 }, "Calendar_1::AddEvent": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Calendar_1::GetAvailableTime": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Events::BuyEventTickets": { "precision": 0.6956521739130435, "recall": 0.8318042813455657, "f1-score": 0.7576601671309192, "support": 327.0 }, "Events::FindEvents": { "precision": 0.8634146341463415, "recall": 0.8009049773755657, "f1-score": 0.8309859154929577, "support": 442.0 }, "Events::GetEventDates": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::ReserveOnewayFlight": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::ReserveRoundtripFlights": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Flights::SearchOnewayFlight": { "precision": 0.7709923664122137, "recall": 0.5872093023255814, "f1-score": 0.6666666666666666, "support": 172.0 }, "Flights::SearchRoundtripFlights": { "precision": 0.7610062893081762, "recall": 0.8642857142857143, "f1-score": 0.8093645484949833, "support": 280.0 }, "Homes::FindApartment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Homes::FindHomeByArea": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 139.0 }, "Homes::ScheduleVisit": { "precision": 0.8349514563106796, "recall": 0.7644444444444445, "f1-score": 0.7981438515081206, "support": 225.0 }, "Hotels::BookHouse": { "precision": 0.9270833333333334, "recall": 0.6137931034482759, "f1-score": 0.7385892116182573, "support": 145.0 }, "Hotels::ReserveHotel": { "precision": 0.6947791164658634, "recall": 0.7456896551724138, "f1-score": 0.7193347193347194, "support": 232.0 }, "Hotels::SearchHotel": { "precision": 0.7122040072859745, "recall": 0.8786516853932584, "f1-score": 0.7867203219315896, "support": 445.0 }, "Hotels::SearchHouse": { "precision": 0.7941176470588235, "recall": 0.8481675392670157, "f1-score": 0.8202531645569621, "support": 382.0 }, "Media::FindMovies": { "precision": 0.6377551020408163, "recall": 0.8741258741258742, "f1-score": 0.7374631268436578, "support": 286.0 }, "Media::PlayMovie": { "precision": 0.8861386138613861, "recall": 0.8099547511312217, "f1-score": 0.8463356973995272, "support": 221.0 }, "Messaging_1::ShareLocation": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 255.0 }, "Movies::BuyMovieTickets": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 80.0 }, "Movies::FindMovies": { "precision": 0.6227678571428571, "recall": 0.6611374407582938, "f1-score": 0.6413793103448275, "support": 422.0 }, "Movies::GetTimesForMovie": { "precision": 0.7722772277227723, "recall": 0.7428571428571429, "f1-score": 0.7572815533980582, "support": 105.0 }, "Music::LookupMusic": { "precision": 0.9811320754716981, "recall": 0.1925925925925926, "f1-score": 0.3219814241486068, "support": 270.0 }, "Music::LookupSong": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Music::PlayMedia": { "precision": 0.8650793650793651, "recall": 0.3797909407665505, "f1-score": 0.5278450363196125, "support": 287.0 }, "Music::PlaySong": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Payment_1::MakePayment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 134.0 }, "Payment_1::RequestPayment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 122.0 }, "RentalCars::GetCarsAvailable": { "precision": 0.7560975609756098, "recall": 0.9368131868131868, "f1-score": 0.8368098159509203, "support": 364.0 }, "RentalCars::ReserveCar": { "precision": 0.7147540983606557, "recall": 0.6246418338108882, "f1-score": 0.6666666666666666, "support": 349.0 }, "Restaurants::FindRestaurants": { "precision": 0.7726161369193154, "recall": 0.9186046511627907, "f1-score": 0.8393094289508632, "support": 344.0 }, "Restaurants::ReserveRestaurant": { "precision": 0.875, "recall": 0.8927536231884058, "f1-score": 0.8837876614060258, "support": 345.0 }, "RideSharing::GetRide": { "precision": 0.6344410876132931, "recall": 0.9813084112149533, "f1-score": 0.7706422018348624, "support": 214.0 }, "Services_1::BookAppointment": { "precision": 0.49096385542168675, "recall": 0.6293436293436293, "f1-score": 0.5516074450084603, "support": 259.0 }, "Services_1::FindProvider": { "precision": 0.9626865671641791, "recall": 0.9885057471264368, "f1-score": 0.9754253308128544, "support": 261.0 }, "Services_2::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_2::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_3::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_3::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 0.0 }, "Services_4::BookAppointment": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 248.0 }, "Services_4::FindProvider": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 265.0 }, "Trains_1::FindTrains": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 317.0 }, "Trains_1::GetTrainTickets": { "precision": 0.0, "recall": 0.0, "f1-score": 0.0, "support": 157.0 }, "Travel_1::FindAttractions": { "precision": 0.8064516129032258, "recall": 0.8737864077669902, "f1-score": 0.8387698042870456, "support": 515.0 }, "Weather_1::GetWeather": { "precision": 0.9944903581267218, "recall": 0.9232736572890026, "f1-score": 0.9575596816976127, "support": 391.0 }, "accuracy": 0.6251008878127522, "macro avg": { "precision": 0.39109861885630526, "recall": 0.39028350336585926, "f1-score": 0.37912842142342584, "support": 9912.0 }, "weighted avg": { "precision": 0.6097753649101566, "recall": 0.6251008878127522, "f1-score": 0.6006828056612206, "support": 9912.0 } }, "train_samples": 37201, "dev_samples": 5324, "test_samples": 9912, "unseen_test_classes": [ "Alarm_1::AddAlarm", "Alarm_1::GetAlarms", "Homes::FindHomeByArea", "Messaging_1::ShareLocation", "Movies::BuyMovieTickets", "Payment_1::MakePayment", "Payment_1::RequestPayment", "Services_4::BookAppointment", "Services_4::FindProvider", "Trains_1::FindTrains", "Trains_1::GetTrainTickets" ], "errors": [ { "text": "I would like to take this girl I like out for a casual meal on the 3rd of this month, when I get paid so something around eleven am would be perfect. Can you help me locate a restaurant in Palo Alto?", "actual": "Restaurants::ReserveRestaurant", "predicted": "Restaurants::FindRestaurants" }, { "text": "I would like to book a hotel in Washington D.C.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "I need to book a hotel please.", "actual": "Hotels::SearchHotel", "predicted": "Hotels::ReserveHotel" }, { "text": "That works for me. I'd like to make a reservation.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Alright make the arrival time for 4th of March.", "actual": "Hotels::ReserveHotel", "predicted": "Homes::ScheduleVisit" }, { "text": "Sure, could you make a reservation on the 4th of March please?", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes. I want to make a reservation please.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Yes, I need to reserve it now and do it for me.", "actual": "Hotels::ReserveHotel", "predicted": "RentalCars::ReserveCar" }, { "text": "Yes. I wish to reserve it.", "actual": "Hotels::ReserveHotel", "predicted": "RentalCars::ReserveCar" }, { "text": "Sounds nice. Please make me a reservation there.", "actual": "Hotels::ReserveHotel", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "I am in a nice mood and I like to listen some nice songs. Can you search for me the best one?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes if you would please.", "actual": "Music::PlayMedia", "predicted": "Restaurants::ReserveRestaurant" }, { "text": "Can you recommend a song to listen to?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes, I'd like to listen to it now.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Please find me any of the songs that I've played the most this week.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Can you search for some songs from 2015?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "I am looking for some nice songs.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Hey, I feel like listening to some tunes right now. Can you find me something from two years ago?", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Please do. Cast it on the living room device.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "I'd like to look for music right now.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yup. Play it on patio speaker.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Okay, give me that. I want it to play on the patio speaker.", "actual": "Music::PlayMedia", "predicted": "Music::PlaySong" }, { "text": "Hello, I want to find songs please, ones like on the album The New Classic. Or tunes from, say, six years back.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" }, { "text": "Yes, and please do it in the Living room.", "actual": "Music::PlayMedia", "predicted": "Hotels::ReserveHotel" }, { "text": "I would like to search for songs at this time.", "actual": "Music::LookupMusic", "predicted": "Music::LookupSong" } ], "protocol": "SGD official train/dev/test, all shards; only user turns that start or change the active intent (confirmations, slot answers and thanks excluded); schema-reviewed domain-qualified intents, API instance omitted for equivalent tasks; Services subtypes preserved; unseen test classes retained; exact-text deduplication", "trained_at": "2026-10-25T16:33:15.574042+00:00", "seed": 42, "dataset_sha256": "4580ee54b4981f2026e4ea9d21e63eb6855c62c6327366927d13b181be77593c", "checkpoint": "google-bert/bert-base-multilingual-cased", "epochs": 4, "dev_macro_f1": 0.4211468062833233, "confidence_kind": "uncalibrated_softmax", "pilot": false, "max_train": 0, "max_eval": 0, "seen_accuracy": 0.7838077166350411, "seen_macro_f1": 0.7436749804844123, "seen_test_samples": 7905, "unseen_test_samples": 2007, "unseen_class_count": 11 } ] }