diff --git "a/all_test_qa.json" "b/all_test_qa.json" new file mode 100644--- /dev/null +++ "b/all_test_qa.json" @@ -0,0 +1,60998 @@ +[ + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3191,0.1138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138]" + }, + { + "from": "gpt", + "value": "CLICK[0.5556,0.9655]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655]" + }, + { + "from": "gpt", + "value": "CLICK[0.3911,0.1556]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556]" + }, + { + "from": "gpt", + "value": "TYPE[startups]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4007,0.3161]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news about \"Startups\", then read the second news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.1138],CLICK[0.5556,0.9655],CLICK[0.3911,0.1556],TYPE[startups],PRESS_ENTER,CLICK[0.4007,0.3161],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_25_995c006a845b43218882161c9bbfae85-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3164,0.0918]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.0918]" + }, + { + "from": "gpt", + "value": "CLICK[0.5632,0.9737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.0918],CLICK[0.5632,0.9737]" + }, + { + "from": "gpt", + "value": "CLICK[0.3668,0.1395]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.0918],CLICK[0.5632,0.9737],CLICK[0.3668,0.1395]" + }, + { + "from": "gpt", + "value": "TYPE[eurppe]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.0918],CLICK[0.5632,0.9737],CLICK[0.3668,0.1395],TYPE[eurppe]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.0918],CLICK[0.5632,0.9737],CLICK[0.3668,0.1395],TYPE[eurppe],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3711,0.4717]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search news about Europe and read the third news item on the list.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.0918],CLICK[0.5632,0.9737],CLICK[0.3668,0.1395],TYPE[eurppe],PRESS_ENTER,CLICK[0.3711,0.4717]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_29_7831d67b90824b36bff7221050c03411-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3280,0.1003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003]" + }, + { + "from": "gpt", + "value": "CLICK[0.5516,0.9681]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681]" + }, + { + "from": "gpt", + "value": "CLICK[0.3513,0.1280]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681],CLICK[0.3513,0.1280]" + }, + { + "from": "gpt", + "value": "TYPE[science]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681],CLICK[0.3513,0.1280],TYPE[science]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681],CLICK[0.3513,0.1280],TYPE[science],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4188,0.8059]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681],CLICK[0.3513,0.1280],TYPE[science],PRESS_ENTER,CLICK[0.4188,0.8059]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681],CLICK[0.3513,0.1280],TYPE[science],PRESS_ENTER,CLICK[0.4188,0.8059],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for \"Science\" tab and read the fifth news articles.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3280,0.1003],CLICK[0.5516,0.9681],CLICK[0.3513,0.1280],TYPE[science],PRESS_ENTER,CLICK[0.4188,0.8059],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_31_2ac62903d1a94346be9df604cf8058d9-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3158,0.1079]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079]" + }, + { + "from": "gpt", + "value": "CLICK[0.5467,0.9724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724]" + }, + { + "from": "gpt", + "value": "CLICK[0.3339,0.1342]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342]" + }, + { + "from": "gpt", + "value": "TYPE[paste]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4000,0.1632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER,CLICK[0.4000,0.1632]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER,CLICK[0.4000,0.1632],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER,CLICK[0.4000,0.1632],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER,CLICK[0.4000,0.1632],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER,CLICK[0.4000,0.1632],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Go to \"Entertainment\" section, search for \"Hollywood\", click on the top news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3158,0.1079],CLICK[0.5467,0.9724],CLICK[0.3339,0.1342],TYPE[paste],PRESS_ENTER,CLICK[0.4000,0.1632],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_34_e1c42b24b391461db929f51f0d8d9a56-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3125,0.1171]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.4059,0.1349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4059,0.1349]" + }, + { + "from": "gpt", + "value": "CLICK[0.4674,0.7184]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4059,0.1349],CLICK[0.4674,0.7184]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4059,0.1349],CLICK[0.4674,0.7184],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4059,0.1349],CLICK[0.4674,0.7184],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"NBC Select\" section, open the second item.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3125,0.1171],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4059,0.1349],CLICK[0.4674,0.7184],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_38_e6e6cf400af14b60822a5183f0e43f37-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.1138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.5043,0.1266]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266]" + }, + { + "from": "gpt", + "value": "CLICK[0.5523,0.4280]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. From the \"Health\" section, find an article related to \"nutrition\", read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.1138],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5043,0.1266],CLICK[0.5523,0.4280],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_40_1bee0e1c2f4d44039d39ab81b8f41cd3-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3217,0.1092]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092]" + }, + { + "from": "gpt", + "value": "CLICK[0.5704,0.9803]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5704,0.9803]" + }, + { + "from": "gpt", + "value": "CLICK[0.3674,0.1461]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5704,0.9803],CLICK[0.3674,0.1461]" + }, + { + "from": "gpt", + "value": "TYPE[paste]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5704,0.9803],CLICK[0.3674,0.1461],TYPE[paste]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5704,0.9803],CLICK[0.3674,0.1461],TYPE[paste],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3743,0.3260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search for news related to \"Cybersecurity\", and read the second article from the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5704,0.9803],CLICK[0.3674,0.1461],TYPE[paste],PRESS_ENTER,CLICK[0.3743,0.3260]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_47_59f4e7543a9a492493319d0350e6ac59-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3217,0.1092]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092]" + }, + { + "from": "gpt", + "value": "CLICK[0.5434,0.9609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5434,0.9609]" + }, + { + "from": "gpt", + "value": "CLICK[0.3697,0.1530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5434,0.9609],CLICK[0.3697,0.1530]" + }, + { + "from": "gpt", + "value": "TYPE[South America]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5434,0.9609],CLICK[0.3697,0.1530],TYPE[South America]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5434,0.9609],CLICK[0.3697,0.1530],TYPE[South America],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4072,0.3477]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about \"South America\", and read the second news on the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3217,0.1092],CLICK[0.5434,0.9609],CLICK[0.3697,0.1530],TYPE[South America],PRESS_ENTER,CLICK[0.4072,0.3477]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_51_656a661aa49c41ae9d4731ec34ca17e8-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3204,0.1112]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.4102,0.1313]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313]" + }, + { + "from": "gpt", + "value": "CLICK[0.4780,0.5674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. On the homepage, open any news piece from the \"Politics\" category to read.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3204,0.1112],SWIPE[LEFT],CLICK[0.4102,0.1313],CLICK[0.4780,0.5674],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_53_f13debee535d43c5a0087b756e7f4558-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3237,0.1043]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.4786,0.1349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4816,0.2533]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4816,0.2533]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4816,0.2533],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4816,0.2533],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4816,0.2533],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Open the \"Sports\" tab, find and read about the latest \"Basketball\" match.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3237,0.1043],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.4786,0.1349],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4816,0.2533],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_55_6aa4fe0cbe6f4b6baa39332e5654f1b1-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3164,0.1112]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112]" + }, + { + "from": "gpt", + "value": "CLICK[0.5589,0.9661]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661]" + }, + { + "from": "gpt", + "value": "CLICK[0.3428,0.1428]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428]" + }, + { + "from": "gpt", + "value": "TYPE[global warming]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4707,0.2885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP],CLICK[0.4707,0.2885]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP],CLICK[0.4707,0.2885],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP],CLICK[0.4707,0.2885],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP],CLICK[0.4707,0.2885],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP],CLICK[0.4707,0.2885],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Find an article about \"Global Warming\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3164,0.1112],CLICK[0.5589,0.9661],CLICK[0.3428,0.1428],TYPE[global warming],PRESS_ENTER,SWIPE[UP],CLICK[0.4707,0.2885],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_1_59_44aa2b208d4c44a981b85dbfd08f73c8-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3257,0.0977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977]" + }, + { + "from": "gpt", + "value": "CLICK[0.5615,0.9549]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549]" + }, + { + "from": "gpt", + "value": "CLICK[0.3911,0.1428]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428]" + }, + { + "from": "gpt", + "value": "TYPE[asia]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4276,0.3408]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Search the news about Asia, I want to read the first news of the topic.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3257,0.0977],CLICK[0.5615,0.9549],CLICK[0.3911,0.1428],TYPE[asia],PRESS_ENTER,CLICK[0.4276,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_10_ba5c277724c14369b3154eba61e65722-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3089,0.0970]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970]" + }, + { + "from": "gpt", + "value": "CLICK[0.5576,0.9724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724]" + }, + { + "from": "gpt", + "value": "CLICK[0.3783,0.1503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503]" + }, + { + "from": "gpt", + "value": "TYPE[aritificial intelligence]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4020,0.3408]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP],CLICK[0.4020,0.3408]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP],CLICK[0.4020,0.3408],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP],CLICK[0.4020,0.3408],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP],CLICK[0.4020,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP],CLICK[0.4020,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. find an articles about \"Artificial Intelligence\" and read it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3089,0.0970],CLICK[0.5576,0.9724],CLICK[0.3783,0.1503],TYPE[aritificial intelligence],PRESS_ENTER,SWIPE[UP],CLICK[0.4020,0.3408],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_1_19f23dbca5e14a6b8aaa4cf0bce3b849-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3138,0.1178]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3023,0.1355]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4944,0.6711]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Navigate to the \"Sports\" section, look for an \"Swim\" news.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.1178],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.1355],SWIPE[UP],SWIPE[UP],CLICK[0.4944,0.6711],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_4_3093605da94746a7a61b7df2ec47aa34-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3243,0.1220]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220]" + }, + { + "from": "gpt", + "value": "CLICK[0.3750,0.7263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open NBC News. Randomly open a piece of news in the home page.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.1220],CLICK[0.3750,0.7263],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_4_28_2_8_f9bdef9ae1f54ac8b044b77f24311626-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6587,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.3894,0.2183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183]" + }, + { + "from": "gpt", + "value": "TYPE[Tokyo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.1343]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343]" + }, + { + "from": "gpt", + "value": "CLICK[0.3692,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6631,0.5481]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481]" + }, + { + "from": "gpt", + "value": "CLICK[0.3154,0.7163]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163]" + }, + { + "from": "gpt", + "value": "CLICK[0.5074,0.9250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407]" + }, + { + "from": "gpt", + "value": "CLICK[0.5724,0.6885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.7590]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590]" + }, + { + "from": "gpt", + "value": "CLICK[0.5151,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.3955]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955]" + }, + { + "from": "gpt", + "value": "CLICK[0.4904,0.1522]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.4606]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.8119]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.4215]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.5324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.4292]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4292]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.9083]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4292],CLICK[0.4478,0.9083]" + }, + { + "from": "gpt", + "value": "CLICK[0.3436,0.1442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4292],CLICK[0.4478,0.9083],CLICK[0.3436,0.1442]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.6234]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Tokyo, from 5 October to 20 October. 2 guests and 1 child in 1 room. Queen-sized bed and 1 single bed. Filters include Wi-Fi, 24-hour front desk, and parking. Sort by proximity to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2587],CLICK[0.3894,0.2183],TYPE[Tokyo],CLICK[0.3962,0.1343],CLICK[0.3692,0.2766],SWIPE[UP],CLICK[0.6631,0.5481],CLICK[0.3154,0.7163],CLICK[0.5074,0.9250],CLICK[0.4522,0.3407],CLICK[0.5724,0.6885],CLICK[0.6824,0.7590],CLICK[0.5151,0.9330],CLICK[0.4724,0.3955],CLICK[0.4904,0.1522],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4606],CLICK[0.7013,0.8119],CLICK[0.7003,0.4215],SWIPE[UP],CLICK[0.6913,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4292],CLICK[0.4478,0.9083],CLICK[0.3436,0.1442],CLICK[0.6913,0.6234]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_28_041e1eb6a4194806ac2d38702c44603e-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6756,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.6026,0.1208]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.6154]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154]" + }, + { + "from": "gpt", + "value": "CLICK[0.5532,0.6683]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.3429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429]" + }, + { + "from": "gpt", + "value": "CLICK[0.3580,0.3542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542]" + }, + { + "from": "gpt", + "value": "CLICK[0.6532,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.5503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.5269]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503],CLICK[0.4747,0.5269]" + }, + { + "from": "gpt", + "value": "CLICK[0.6106,0.1208]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503],CLICK[0.4747,0.5269],CLICK[0.6106,0.1208]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.3888]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503],CLICK[0.4747,0.5269],CLICK[0.6106,0.1208],CLICK[0.4119,0.3888]" + }, + { + "from": "gpt", + "value": "CLICK[0.6218,0.6144]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503],CLICK[0.4747,0.5269],CLICK[0.6106,0.1208],CLICK[0.4119,0.3888],CLICK[0.6218,0.6144]" + }, + { + "from": "gpt", + "value": "CLICK[0.3683,0.7971]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503],CLICK[0.4747,0.5269],CLICK[0.6106,0.1208],CLICK[0.4119,0.3888],CLICK[0.6218,0.6144],CLICK[0.3683,0.7971]" + }, + { + "from": "gpt", + "value": "CLICK[0.4926,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in CHicago from Oct 1 8 am to Oct 10 10 pm. Filter by meidum type, Sixt Car as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2542],CLICK[0.6026,0.1208],CLICK[0.3939,0.3349],SWIPE[UP],CLICK[0.4423,0.6154],CLICK[0.5532,0.6683],CLICK[0.4567,0.9218],CLICK[0.6487,0.3429],CLICK[0.3580,0.3542],CLICK[0.6532,0.4022],CLICK[0.3728,0.5503],CLICK[0.4747,0.5269],CLICK[0.6106,0.1208],CLICK[0.4119,0.3888],CLICK[0.6218,0.6144],CLICK[0.3683,0.7971],CLICK[0.4926,0.9160]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_32_7ad146353c9e49ac951805cc5546e750-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6542,0.2699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699]" + }, + { + "from": "gpt", + "value": "CLICK[0.4006,0.2173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173]" + }, + { + "from": "gpt", + "value": "TYPE[Shnhanghai]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.2667]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667]" + }, + { + "from": "gpt", + "value": "CLICK[0.5635,0.6535]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.7074]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074]" + }, + { + "from": "gpt", + "value": "CLICK[0.5163,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.4045]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045]" + }, + { + "from": "gpt", + "value": "CLICK[0.5119,0.1487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5494]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3279,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.4080]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080]" + }, + { + "from": "gpt", + "value": "CLICK[0.7048,0.5795]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.4942]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6862]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942],CLICK[0.6968,0.6862]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942],CLICK[0.6968,0.6862],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942],CLICK[0.6968,0.6862],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942],CLICK[0.6968,0.6862],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.5324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942],CLICK[0.6968,0.6862],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7003,0.5324]" + }, + { + "from": "gpt", + "value": "CLICK[0.4837,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 25 July to 30 July. 2 guests in 1 double room. US$100 to US$400. Filter with very good rating, cofffee machine and breakfast, non-smoking rooms and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2699],CLICK[0.4006,0.2173],TYPE[Shnhanghai],CLICK[0.4042,0.1375],CLICK[0.3827,0.2667],CLICK[0.5635,0.6535],CLICK[0.4288,0.7074],CLICK[0.5163,0.9160],CLICK[0.4567,0.4045],CLICK[0.5119,0.1487],SWIPE[LEFT],SWIPE[UP],CLICK[0.6946,0.5494],SWIPE[UP],CLICK[0.3279,0.5122],SWIPE[UP],CLICK[0.6981,0.4080],CLICK[0.7048,0.5795],SWIPE[UP],CLICK[0.7003,0.4942],CLICK[0.6968,0.6862],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7003,0.5324],CLICK[0.4837,0.9183]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_39_938665979ac7460eb3f545603ecb9928-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6654,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2779]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779]" + }, + { + "from": "gpt", + "value": "TYPE[JFK]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.3228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228]" + }, + { + "from": "gpt", + "value": "TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square]" + }, + { + "from": "gpt", + "value": "CLICK[0.4199,0.3385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square],CLICK[0.4199,0.3385]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.3933]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square],CLICK[0.4199,0.3385],CLICK[0.4042,0.3933]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square],CLICK[0.4199,0.3385],CLICK[0.4042,0.3933],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square],CLICK[0.4199,0.3385],CLICK[0.4042,0.3933],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5253,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square],CLICK[0.4199,0.3385],CLICK[0.4042,0.3933],SWIPE[UP],SWIPE[UP],CLICK[0.5253,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.4782,0.4561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in New York for today at 8 pm. Pick-up location is JFK Airport and destination is Pod Times Square.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2442],CLICK[0.6878,0.1253],CLICK[0.3885,0.2779],TYPE[JFK],CLICK[0.4356,0.3228],TYPE[Times SquareLeftLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaltLeftRightaltLeftaPod Times Square],CLICK[0.4199,0.3385],CLICK[0.4042,0.3933],SWIPE[UP],SWIPE[UP],CLICK[0.5253,0.9183],CLICK[0.4782,0.4561]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_42_495c0e679842451795ed70d5fec3a425-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6497,0.2577]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577]" + }, + { + "from": "gpt", + "value": "CLICK[0.4131,0.2196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.5000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000]" + }, + { + "from": "gpt", + "value": "CLICK[0.3288,0.5449]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.3933]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.1510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.5090]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.3978]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978]" + }, + { + "from": "gpt", + "value": "CLICK[0.3154,0.8240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.2946]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.2981]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.2981]" + }, + { + "from": "gpt", + "value": "CLICK[0.5128,0.9317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.2981],CLICK[0.5128,0.9317]" + }, + { + "from": "gpt", + "value": "CLICK[0.3446,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.2981],CLICK[0.5128,0.9317],CLICK[0.3446,0.1465]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.7872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Rome, from 1 July to 7 July. 2 guests in 1 room. $400 to $800. Filter with Pleasant rating, free WiFi, gym and breakfast included. Sort by lowest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2577],CLICK[0.4131,0.2196],TYPE[Rome],CLICK[0.3827,0.1375],CLICK[0.4221,0.2734],CLICK[0.3849,0.5000],CLICK[0.3288,0.5449],CLICK[0.4724,0.9218],CLICK[0.4378,0.3933],CLICK[0.5029,0.1510],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],CLICK[0.6990,0.5090],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.3978],CLICK[0.3154,0.8240],CLICK[0.6968,0.6849],SWIPE[UP],CLICK[0.7026,0.2946],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.2981],CLICK[0.5128,0.9317],CLICK[0.3446,0.1465],CLICK[0.6968,0.7872]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_46_b0f5c858e91543abbef8dceca94249d9-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6712,0.2407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.2016]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.3984,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410]" + }, + { + "from": "gpt", + "value": "CLICK[0.3737,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6721,0.4933]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.5426]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426]" + }, + { + "from": "gpt", + "value": "CLICK[0.4894,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.7647]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647]" + }, + { + "from": "gpt", + "value": "CLICK[0.4782,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.5196,0.4125]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125]" + }, + { + "from": "gpt", + "value": "CLICK[0.3558,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.5696]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696]" + }, + { + "from": "gpt", + "value": "CLICK[0.5141,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.4192]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.4224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192],SWIPE[UP],CLICK[0.6923,0.4224]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192],SWIPE[UP],CLICK[0.6923,0.4224],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.3183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192],SWIPE[UP],CLICK[0.6923,0.4224],SWIPE[UP],CLICK[0.7003,0.3183]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.7859]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192],SWIPE[UP],CLICK[0.6923,0.4224],SWIPE[UP],CLICK[0.7003,0.3183],CLICK[0.6981,0.7859]" + }, + { + "from": "gpt", + "value": "CLICK[0.5330,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in Sydney from August 10 to August 17 for 2 Adults and 1 Child. Sort by distance from Downtown. Filter options should include free Wi-Fi, included breakfast, and a gym.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2407],CLICK[0.4051,0.2016],TYPE[Sydney],CLICK[0.3984,0.1410],CLICK[0.3737,0.2788],SWIPE[UP],CLICK[0.6721,0.4933],CLICK[0.6824,0.5426],CLICK[0.4894,0.9263],CLICK[0.4423,0.3362],CLICK[0.6891,0.7647],CLICK[0.4782,0.9183],CLICK[0.5196,0.4125],CLICK[0.3558,0.1410],CLICK[0.6891,0.5696],CLICK[0.5141,0.1375],SWIPE[UP],CLICK[0.6936,0.4192],SWIPE[UP],CLICK[0.6923,0.4224],SWIPE[UP],CLICK[0.7003,0.3183],CLICK[0.6981,0.7859],CLICK[0.5330,0.9263]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_50_26434514d5e045c1ac2cdf11c776ec24-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6599,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.1231]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231]" + }, + { + "from": "gpt", + "value": "CLICK[0.4917,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.1657]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.3513,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654]" + }, + { + "from": "gpt", + "value": "CLICK[0.3615,0.3385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385]" + }, + { + "from": "gpt", + "value": "TYPE[Los Angeles]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles]" + }, + { + "from": "gpt", + "value": "CLICK[0.3660,0.2689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.3955]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.4247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247]" + }, + { + "from": "gpt", + "value": "CLICK[0.6138,0.7446]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446]" + }, + { + "from": "gpt", + "value": "CLICK[0.5106,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4590,0.4574]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574]" + }, + { + "from": "gpt", + "value": "CLICK[0.6846,0.2183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574],CLICK[0.6846,0.2183]" + }, + { + "from": "gpt", + "value": "CLICK[0.3099,0.5054]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574],CLICK[0.6846,0.2183],CLICK[0.3099,0.5054]" + }, + { + "from": "gpt", + "value": "CLICK[0.6205,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574],CLICK[0.6846,0.2183],CLICK[0.3099,0.5054],CLICK[0.6205,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.5067]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574],CLICK[0.6846,0.2183],CLICK[0.3099,0.5054],CLICK[0.6205,0.9330],CLICK[0.4522,0.5067]" + }, + { + "from": "gpt", + "value": "CLICK[0.3615,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574],CLICK[0.6846,0.2183],CLICK[0.3099,0.5054],CLICK[0.6205,0.9330],CLICK[0.4522,0.5067],CLICK[0.3615,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.3446,0.8926]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Sydney to Los Angeles. Departure date October 1, return date November 1. 2 Adults in business class. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2442],CLICK[0.4635,0.1231],CLICK[0.4917,0.2756],CLICK[0.4423,0.1657],TYPE[Sydney],CLICK[0.3513,0.2654],CLICK[0.3615,0.3385],TYPE[Los Angeles],CLICK[0.3660,0.2689],CLICK[0.4378,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.4365,0.4247],CLICK[0.6138,0.7446],CLICK[0.5106,0.9340],CLICK[0.4590,0.4574],CLICK[0.6846,0.2183],CLICK[0.3099,0.5054],CLICK[0.6205,0.9330],CLICK[0.4522,0.5067],CLICK[0.3615,0.1263],CLICK[0.3446,0.8926]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_52_4dbd738bc8b147d089ff6e563ba621f4-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6564,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.1106]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.2824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824]" + }, + { + "from": "gpt", + "value": "TYPE[LAX]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX]" + }, + { + "from": "gpt", + "value": "CLICK[0.4176,0.3282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282]" + }, + { + "from": "gpt", + "value": "TYPE[Universal Studios Hollywood]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.3260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood],CLICK[0.3728,0.3260]" + }, + { + "from": "gpt", + "value": "CLICK[0.4737,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood],CLICK[0.3728,0.3260],CLICK[0.4737,0.4058]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood],CLICK[0.3728,0.3260],CLICK[0.4737,0.4058],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood],CLICK[0.3728,0.3260],CLICK[0.4737,0.4058],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4827,0.9115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood],CLICK[0.3728,0.3260],CLICK[0.4737,0.4058],SWIPE[UP],SWIPE[DOWN],CLICK[0.4827,0.9115]" + }, + { + "from": "gpt", + "value": "CLICK[0.5163,0.4696]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in LA for today 9 pm. Pick-up location LAX, and destination is Universal Studios Hollywood.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2442],CLICK[0.7013,0.1106],CLICK[0.4388,0.2824],TYPE[LAX],CLICK[0.4176,0.3282],TYPE[Universal Studios Hollywood],CLICK[0.3728,0.3260],CLICK[0.4737,0.4058],SWIPE[UP],SWIPE[DOWN],CLICK[0.4827,0.9115],CLICK[0.5163,0.4696]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_13_55_7cc36f4171cb44bcb632d1bf0d0c84b5-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6699,0.2429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429]" + }, + { + "from": "gpt", + "value": "CLICK[0.4580,0.1208]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.2811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.3295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.3910]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910]" + }, + { + "from": "gpt", + "value": "CLICK[0.4971,0.6984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984]" + }, + { + "from": "gpt", + "value": "CLICK[0.6744,0.6974]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984],CLICK[0.6744,0.6974]" + }, + { + "from": "gpt", + "value": "CLICK[0.4782,0.9285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984],CLICK[0.6744,0.6974],CLICK[0.4782,0.9285]" + }, + { + "from": "gpt", + "value": "CLICK[0.4176,0.4641]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984],CLICK[0.6744,0.6974],CLICK[0.4782,0.9285],CLICK[0.4176,0.4641]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.5077]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984],CLICK[0.6744,0.6974],CLICK[0.4782,0.9285],CLICK[0.4176,0.4641],CLICK[0.3064,0.5077]" + }, + { + "from": "gpt", + "value": "CLICK[0.6218,0.9250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984],CLICK[0.6744,0.6974],CLICK[0.4782,0.9285],CLICK[0.4176,0.4641],CLICK[0.3064,0.5077],CLICK[0.6218,0.9250]" + }, + { + "from": "gpt", + "value": "CLICK[0.4792,0.5301]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from New York to Rome departing 12 June and returning 15 June. Adjust for 1 Adult in business class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2429],CLICK[0.4580,0.1208],CLICK[0.4478,0.2811],CLICK[0.4455,0.1689],TYPE[New York],CLICK[0.4410,0.2721],CLICK[0.4119,0.3295],TYPE[Rome],CLICK[0.3849,0.2721],CLICK[0.3974,0.3910],CLICK[0.4971,0.6984],CLICK[0.6744,0.6974],CLICK[0.4782,0.9285],CLICK[0.4176,0.4641],CLICK[0.3064,0.5077],CLICK[0.6218,0.9250],CLICK[0.4792,0.5301]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_0_35edb0690faa4631aae5aa6cf1f10d89-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6756,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.4904,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4006,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.1702]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702]" + }, + { + "from": "gpt", + "value": "TYPE[Tokyo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.3429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429]" + }, + { + "from": "gpt", + "value": "TYPE[Seattle]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle]" + }, + { + "from": "gpt", + "value": "CLICK[0.4019,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.3901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901]" + }, + { + "from": "gpt", + "value": "CLICK[0.6196,0.3093]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093]" + }, + { + "from": "gpt", + "value": "CLICK[0.3750,0.4224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4433,0.4551]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551]" + }, + { + "from": "gpt", + "value": "CLICK[0.6856,0.2833]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833]" + }, + { + "from": "gpt", + "value": "CLICK[0.3054,0.6391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391]" + }, + { + "from": "gpt", + "value": "CLICK[0.5926,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205]" + }, + { + "from": "gpt", + "value": "CLICK[0.4849,0.5301]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205],CLICK[0.4849,0.5301]" + }, + { + "from": "gpt", + "value": "CLICK[0.3446,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205],CLICK[0.4849,0.5301],CLICK[0.3446,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.3481,0.8846]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205],CLICK[0.4849,0.5301],CLICK[0.3446,0.1298],CLICK[0.3481,0.8846]" + }, + { + "from": "gpt", + "value": "CLICK[0.4603,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205],CLICK[0.4849,0.5301],CLICK[0.3446,0.1298],CLICK[0.3481,0.8846],CLICK[0.4603,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.7455]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205],CLICK[0.4849,0.5301],CLICK[0.3446,0.1298],CLICK[0.3481,0.8846],CLICK[0.4603,0.1276],CLICK[0.4141,0.7455]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Tokyo to Seattle. Departure date 10 May, return date 20 May. 1 adult, 1 child. Business class. Sort by total travel time. One stop is okay.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2465],CLICK[0.4904,0.1263],CLICK[0.4006,0.2756],CLICK[0.4423,0.1702],TYPE[Tokyo],CLICK[0.3997,0.2631],CLICK[0.3782,0.3429],TYPE[Seattle],CLICK[0.4019,0.2721],CLICK[0.4288,0.3901],CLICK[0.6196,0.3093],CLICK[0.3750,0.4224],CLICK[0.4522,0.9263],CLICK[0.4433,0.4551],CLICK[0.6856,0.2833],CLICK[0.3054,0.6391],CLICK[0.5926,0.9205],CLICK[0.4849,0.5301],CLICK[0.3446,0.1298],CLICK[0.3481,0.8846],CLICK[0.4603,0.1276],CLICK[0.4141,0.7455],CLICK[0.6644,0.9196]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_11_7eda2cc409064e8d8849d4506475814a-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6564,0.2497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.2061]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.1388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.2779]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6676,0.6333]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.7468]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.3304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304]" + }, + { + "from": "gpt", + "value": "CLICK[0.6833,0.6795]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.6907]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907]" + }, + { + "from": "gpt", + "value": "CLICK[0.4737,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4827,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000]" + }, + { + "from": "gpt", + "value": "CLICK[0.5006,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122]" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.5503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.4551]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7003,0.4551]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.9308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in Sydney from 15 June until 25 June. 4 guests in 2 rooms.Filter by 5-star rating, free breakfast, and swimming pool.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2497],CLICK[0.4388,0.2061],TYPE[Sydney],CLICK[0.4119,0.1388],CLICK[0.4154,0.2779],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6676,0.6333],CLICK[0.4478,0.7468],CLICK[0.4221,0.9160],CLICK[0.3997,0.3304],CLICK[0.6833,0.6795],CLICK[0.6869,0.6907],CLICK[0.4737,0.9218],CLICK[0.4827,0.4000],CLICK[0.5006,0.1465],SWIPE[UP],CLICK[0.6990,0.5122],CLICK[0.7071,0.5189],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5503],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7003,0.4551],CLICK[0.5029,0.9308]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_14_38077e608daa4128bca191118b79dfa5-33.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6712,0.2474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474]" + }, + { + "from": "gpt", + "value": "CLICK[0.4590,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4612,0.2128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.3282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282]" + }, + { + "from": "gpt", + "value": "TYPE[BBerlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.2599]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.3968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.4103]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968],SWIPE[UP],SWIPE[UP],CLICK[0.4984,0.4103]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968],SWIPE[UP],SWIPE[UP],CLICK[0.4984,0.4103],CLICK[0.4410,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.5920]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968],SWIPE[UP],SWIPE[UP],CLICK[0.4984,0.4103],CLICK[0.4410,0.9183],CLICK[0.6869,0.5920]" + }, + { + "from": "gpt", + "value": "CLICK[0.3625,0.5099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Berlin. Departure date 18 September. 1 adult in economy class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6712,0.2474],CLICK[0.4590,0.1263],CLICK[0.4612,0.2128],CLICK[0.4356,0.2721],CLICK[0.4401,0.1689],TYPE[London],CLICK[0.3885,0.2712],CLICK[0.4401,0.3282],TYPE[BBerlin],CLICK[0.3974,0.2599],CLICK[0.3872,0.3968],SWIPE[UP],SWIPE[UP],CLICK[0.4984,0.4103],CLICK[0.4410,0.9183],CLICK[0.6869,0.5920],CLICK[0.3625,0.5099]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_17_df69bb678d2746b0bf746393804e838b-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.2519]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519]" + }, + { + "from": "gpt", + "value": "CLICK[0.4176,0.2205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205]" + }, + { + "from": "gpt", + "value": "TYPE[Athens]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.1343]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5567,0.5010]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.7141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141]" + }, + { + "from": "gpt", + "value": "CLICK[0.5074,0.9250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407]" + }, + { + "from": "gpt", + "value": "CLICK[0.5779,0.6064]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064]" + }, + { + "from": "gpt", + "value": "CLICK[0.5747,0.6862]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862]" + }, + { + "from": "gpt", + "value": "CLICK[0.5792,0.6872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872]" + }, + { + "from": "gpt", + "value": "CLICK[0.5869,0.6894]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894]" + }, + { + "from": "gpt", + "value": "CLICK[0.5375,0.9151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151]" + }, + { + "from": "gpt", + "value": "CLICK[0.4994,0.3946]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946]" + }, + { + "from": "gpt", + "value": "CLICK[0.5038,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.5212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3279,0.7962]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.7971]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.4798]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.3215]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798],CLICK[0.6981,0.3215]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798],CLICK[0.6981,0.3215],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798],CLICK[0.6981,0.3215],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3526,0.6692]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798],CLICK[0.6981,0.3215],SWIPE[UP],SWIPE[UP],CLICK[0.3526,0.6692]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.4237]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798],CLICK[0.6981,0.3215],SWIPE[UP],SWIPE[UP],CLICK[0.3526,0.6692],CLICK[0.6968,0.4237]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.9106]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment in Athens for August. 1 adult. Apply filters for Wi-Fi, breakfast included, coffee machine and city center location.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2519],CLICK[0.4176,0.2205],TYPE[Athens],CLICK[0.3962,0.1343],CLICK[0.4029,0.2756],SWIPE[UP],CLICK[0.5567,0.5010],CLICK[0.6644,0.7141],CLICK[0.5074,0.9250],CLICK[0.4141,0.3407],CLICK[0.5779,0.6064],CLICK[0.5747,0.6862],CLICK[0.5792,0.6872],CLICK[0.5869,0.6894],CLICK[0.5375,0.9151],CLICK[0.4994,0.3946],CLICK[0.5038,0.1465],SWIPE[UP],CLICK[0.7003,0.5212],SWIPE[UP],CLICK[0.3279,0.7962],SWIPE[UP],CLICK[0.6923,0.7971],SWIPE[UP],CLICK[0.7003,0.4798],CLICK[0.6981,0.3215],SWIPE[UP],SWIPE[UP],CLICK[0.3526,0.6692],CLICK[0.6968,0.4237],CLICK[0.4401,0.9106]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_20_83a3cc21e7384a0aaa907033d9e77897-33.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6609,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.1958]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958]" + }, + { + "from": "gpt", + "value": "TYPE[MIami]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami]" + }, + { + "from": "gpt", + "value": "CLICK[0.4186,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.6689,0.5458]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.6526]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526]" + }, + { + "from": "gpt", + "value": "CLICK[0.4760,0.9308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308]" + }, + { + "from": "gpt", + "value": "CLICK[0.4208,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407]" + }, + { + "from": "gpt", + "value": "CLICK[0.6788,0.6872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.9173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.4090]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090]" + }, + { + "from": "gpt", + "value": "CLICK[0.4804,0.1365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.5324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.4282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6958,0.5494]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.4282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494],CLICK[0.6990,0.4282]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494],CLICK[0.6990,0.4282],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.2923]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494],CLICK[0.6990,0.4282],SWIPE[UP],CLICK[0.7003,0.2923]" + }, + { + "from": "gpt", + "value": "CLICK[0.4670,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494],CLICK[0.6990,0.4282],SWIPE[UP],CLICK[0.7003,0.2923],CLICK[0.4670,0.9138]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.1455]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494],CLICK[0.6990,0.4282],SWIPE[UP],CLICK[0.7003,0.2923],CLICK[0.4670,0.9138],CLICK[0.3593,0.1455]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.7356]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Miami from 10 August to 20 August for 2 adults. Want a room with a king bed and overlooking the beach. Include free parking and gym in filters. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2465],CLICK[0.4635,0.1958],TYPE[MIami],CLICK[0.4186,0.1375],CLICK[0.4119,0.2744],CLICK[0.6689,0.5458],CLICK[0.4455,0.6526],CLICK[0.4760,0.9308],CLICK[0.4208,0.3407],CLICK[0.6788,0.6872],CLICK[0.4356,0.9173],CLICK[0.4365,0.4090],CLICK[0.4804,0.1365],SWIPE[UP],CLICK[0.7026,0.5324],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4282],SWIPE[UP],CLICK[0.6958,0.5494],CLICK[0.6990,0.4282],SWIPE[UP],CLICK[0.7003,0.2923],CLICK[0.4670,0.9138],CLICK[0.3593,0.1455],CLICK[0.4500,0.7356]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_25_fb8f167baba74b06bb104ead0a2c3ef2-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6609,0.2577]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.2173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173]" + }, + { + "from": "gpt", + "value": "TYPE[Toronto]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto]" + }, + { + "from": "gpt", + "value": "CLICK[0.4186,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3189,0.5269]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269]" + }, + { + "from": "gpt", + "value": "CLICK[0.3907,0.7513]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513]" + }, + { + "from": "gpt", + "value": "CLICK[0.4702,0.9173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.4090]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090]" + }, + { + "from": "gpt", + "value": "CLICK[0.4881,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.4506]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.7567]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.4372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6692]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6692]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.9115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental home in Toronto for the month of September. Apply the filters \"3 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2577],CLICK[0.4500,0.2173],TYPE[Toronto],CLICK[0.4186,0.1298],CLICK[0.4231,0.2644],SWIPE[UP],CLICK[0.3189,0.5269],CLICK[0.3907,0.7513],CLICK[0.4702,0.9173],CLICK[0.4423,0.4090],CLICK[0.4881,0.1465],SWIPE[UP],CLICK[0.6981,0.4506],SWIPE[UP],CLICK[0.6923,0.7567],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.4372],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6692],CLICK[0.5016,0.9115]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_27_24ce2ad7b6144109a9664b9676ede5d4-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.2115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115]" + }, + { + "from": "gpt", + "value": "TYPE[SIngapore]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore]" + }, + { + "from": "gpt", + "value": "CLICK[0.4244,0.1353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6667,0.5997]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.7096]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096]" + }, + { + "from": "gpt", + "value": "CLICK[0.4535,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.3984,0.3394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.1837]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837]" + }, + { + "from": "gpt", + "value": "CLICK[0.5779,0.3192]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192]" + }, + { + "from": "gpt", + "value": "CLICK[0.4715,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353]" + }, + { + "from": "gpt", + "value": "CLICK[0.4859,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058]" + }, + { + "from": "gpt", + "value": "CLICK[0.4917,0.1420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.4304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3401,0.5673]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.6167]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673],CLICK[0.6913,0.6167]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673],CLICK[0.6913,0.6167],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673],CLICK[0.6913,0.6167],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.4538]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673],CLICK[0.6913,0.6167],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4538]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.5538]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673],CLICK[0.6913,0.6167],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4538],CLICK[0.7035,0.5538]" + }, + { + "from": "gpt", + "value": "CLICK[0.5375,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Singapore from 20th July to 30th July. 2 guests in 2 rooms. Filter features should include a bathtub, sea view, and fitness center\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2587],CLICK[0.3974,0.2115],TYPE[SIngapore],CLICK[0.4244,0.1353],CLICK[0.4567,0.2654],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5997],CLICK[0.4455,0.7096],CLICK[0.4535,0.9240],CLICK[0.3984,0.3394],CLICK[0.6824,0.1837],CLICK[0.5779,0.3192],CLICK[0.4715,0.9353],CLICK[0.4859,0.4058],CLICK[0.4917,0.1420],SWIPE[UP],CLICK[0.7035,0.4304],SWIPE[UP],CLICK[0.3401,0.5673],CLICK[0.6913,0.6167],SWIPE[UP],SWIPE[UP],CLICK[0.6981,0.4538],CLICK[0.7035,0.5538],CLICK[0.5375,0.9353]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_2_a19c4757377748539abd93297aa29795-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6721,0.2564]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564]" + }, + { + "from": "gpt", + "value": "CLICK[0.4814,0.1308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.2779]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[Los Angeles]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles]" + }, + { + "from": "gpt", + "value": "CLICK[0.3535,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349]" + }, + { + "from": "gpt", + "value": "TYPE[Vancouver]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.3856]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6173,0.5077]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077]" + }, + { + "from": "gpt", + "value": "CLICK[0.5455,0.5615]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615]" + }, + { + "from": "gpt", + "value": "CLICK[0.4468,0.9106]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.4529]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.2196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676]" + }, + { + "from": "gpt", + "value": "CLICK[0.3054,0.8218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676],CLICK[0.3054,0.8218]" + }, + { + "from": "gpt", + "value": "CLICK[0.5590,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676],CLICK[0.3054,0.8218],CLICK[0.5590,0.9196]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.5942]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676],CLICK[0.3054,0.8218],CLICK[0.5590,0.9196],CLICK[0.6913,0.5942]" + }, + { + "from": "gpt", + "value": "CLICK[0.4849,0.5292]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676],CLICK[0.3054,0.8218],CLICK[0.5590,0.9196],CLICK[0.6913,0.5942],CLICK[0.4849,0.5292]" + }, + { + "from": "gpt", + "value": "CLICK[0.3279,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676],CLICK[0.3054,0.8218],CLICK[0.5590,0.9196],CLICK[0.6913,0.5942],CLICK[0.4849,0.5292],CLICK[0.3279,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.3087,0.8837]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a round-trip flight from Los Angeles to Vancouver. Departing on August 2 and returning on August 8. For 2 adults and 2 children in first class. Sort by duration, non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2564],CLICK[0.4814,0.1308],CLICK[0.4401,0.2779],CLICK[0.4478,0.1689],TYPE[Los Angeles],CLICK[0.3535,0.2721],CLICK[0.3917,0.3349],TYPE[Vancouver],CLICK[0.3817,0.2712],CLICK[0.3962,0.3856],SWIPE[UP],CLICK[0.6173,0.5077],CLICK[0.5455,0.5615],CLICK[0.4468,0.9106],CLICK[0.3728,0.4529],CLICK[0.6811,0.2196],CLICK[0.6923,0.2756],CLICK[0.6811,0.2676],CLICK[0.3054,0.8218],CLICK[0.5590,0.9196],CLICK[0.6913,0.5942],CLICK[0.4849,0.5292],CLICK[0.3279,0.1253],CLICK[0.3087,0.8837]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_30_73dd5c56b68f45999975a5b9a6072972-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6587,0.2564]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564]" + }, + { + "from": "gpt", + "value": "CLICK[0.4715,0.1218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.2138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[Melbourne]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.3237]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237]" + }, + { + "from": "gpt", + "value": "TYPE[AUcuckland]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland]" + }, + { + "from": "gpt", + "value": "CLICK[0.4612,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022]" + }, + { + "from": "gpt", + "value": "CLICK[0.6654,0.7478]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478]" + }, + { + "from": "gpt", + "value": "CLICK[0.4894,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.4651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651]" + }, + { + "from": "gpt", + "value": "CLICK[0.6833,0.2205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.2218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205],CLICK[0.6824,0.2218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3087,0.5067]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205],CLICK[0.6824,0.2218],CLICK[0.3087,0.5067]" + }, + { + "from": "gpt", + "value": "CLICK[0.6038,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205],CLICK[0.6824,0.2218],CLICK[0.3087,0.5067],CLICK[0.6038,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.5256]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205],CLICK[0.6824,0.2218],CLICK[0.3087,0.5067],CLICK[0.6038,0.9263],CLICK[0.4119,0.5256]" + }, + { + "from": "gpt", + "value": "CLICK[0.3413,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205],CLICK[0.6824,0.2218],CLICK[0.3087,0.5067],CLICK[0.6038,0.9263],CLICK[0.4119,0.5256],CLICK[0.3413,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.3333,0.9362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look up a one-way flight from Melbourne to Auckland. Depart on June 22. 3 adults in business class. Stopovers are allowed. Sort by cheapest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2564],CLICK[0.4715,0.1218],CLICK[0.4522,0.2138],CLICK[0.4074,0.2654],CLICK[0.4478,0.1689],TYPE[Melbourne],CLICK[0.3817,0.2631],CLICK[0.3885,0.3237],TYPE[AUcuckland],CLICK[0.4612,0.2712],CLICK[0.3849,0.4022],CLICK[0.6654,0.7478],CLICK[0.4894,0.9228],CLICK[0.4388,0.4651],CLICK[0.6833,0.2205],CLICK[0.6824,0.2218],CLICK[0.3087,0.5067],CLICK[0.6038,0.9263],CLICK[0.4119,0.5256],CLICK[0.3413,0.1298],CLICK[0.3333,0.9362]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_33_3cf06ebca42141848696c220c04136e1-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6497,0.2317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.2609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.1702]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702]" + }, + { + "from": "gpt", + "value": "TYPE[Paris]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris]" + }, + { + "from": "gpt", + "value": "CLICK[0.3603,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.3647,0.3327]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327]" + }, + { + "from": "gpt", + "value": "TYPE[Tokyo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo]" + }, + { + "from": "gpt", + "value": "CLICK[0.4298,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4962,0.5404]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404]" + }, + { + "from": "gpt", + "value": "CLICK[0.5689,0.6391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404],CLICK[0.5689,0.6391]" + }, + { + "from": "gpt", + "value": "CLICK[0.4446,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404],CLICK[0.5689,0.6391],CLICK[0.4446,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404],CLICK[0.5689,0.6391],CLICK[0.4446,0.9340],CLICK[0.4109,0.5189]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404],CLICK[0.5689,0.6391],CLICK[0.4446,0.9340],CLICK[0.4109,0.5189],CLICK[0.4378,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.5567,0.6692]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404],CLICK[0.5689,0.6391],CLICK[0.4446,0.9340],CLICK[0.4109,0.5189],CLICK[0.4378,0.1263],CLICK[0.5567,0.6692]" + }, + { + "from": "gpt", + "value": "CLICK[0.6667,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a flight from Paris to Tokyo. Depart on July 10 and return on July 25. One adult traveller. Apply \"max 1 stops\" filters.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2317],CLICK[0.4522,0.1298],CLICK[0.4042,0.2609],CLICK[0.4356,0.1702],TYPE[Paris],CLICK[0.3603,0.2631],CLICK[0.3647,0.3327],TYPE[Tokyo],CLICK[0.4298,0.2766],CLICK[0.4042,0.4022],SWIPE[UP],CLICK[0.4962,0.5404],CLICK[0.5689,0.6391],CLICK[0.4446,0.9340],CLICK[0.4109,0.5189],CLICK[0.4378,0.1263],CLICK[0.5567,0.6692],CLICK[0.6667,0.9138]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_35_c0e376a416294310abcc2ba5ea245ceb-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6654,0.2510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510]" + }, + { + "from": "gpt", + "value": "CLICK[0.4433,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.1702]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702]" + }, + { + "from": "gpt", + "value": "TYPE[Milan]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.3705,0.3295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654]" + }, + { + "from": "gpt", + "value": "CLICK[0.3526,0.3910]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910]" + }, + { + "from": "gpt", + "value": "CLICK[0.6115,0.4224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.6401]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.4583]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.2196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583],CLICK[0.6913,0.2196]" + }, + { + "from": "gpt", + "value": "CLICK[0.6846,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583],CLICK[0.6913,0.2196],CLICK[0.6846,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.5724,0.9250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583],CLICK[0.6913,0.2196],CLICK[0.6846,0.2744],CLICK[0.5724,0.9250]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.5292]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583],CLICK[0.6913,0.2196],CLICK[0.6846,0.2744],CLICK[0.5724,0.9250],CLICK[0.4567,0.5292]" + }, + { + "from": "gpt", + "value": "CLICK[0.3301,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583],CLICK[0.6913,0.2196],CLICK[0.6846,0.2744],CLICK[0.5724,0.9250],CLICK[0.4567,0.5292],CLICK[0.3301,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3221,0.8926]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Milan to New York. Leave on May 24, come back on June 4. With 2 adults and 1 child in economy class. Sort by duration. A stopover is acceptable.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2510],CLICK[0.4433,0.1276],CLICK[0.4231,0.2721],CLICK[0.4478,0.1702],TYPE[Milan],CLICK[0.4141,0.2734],CLICK[0.3705,0.3295],TYPE[New York],CLICK[0.4109,0.2654],CLICK[0.3526,0.3910],CLICK[0.6115,0.4224],CLICK[0.4343,0.6401],CLICK[0.4051,0.9218],CLICK[0.4029,0.4583],CLICK[0.6913,0.2196],CLICK[0.6846,0.2744],CLICK[0.5724,0.9250],CLICK[0.4567,0.5292],CLICK[0.3301,0.1276],CLICK[0.3221,0.8926]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_37_017be8e3446b41a898cc60de869f7e16-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6554,0.2532]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532]" + }, + { + "from": "gpt", + "value": "CLICK[0.4333,0.2228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228]" + }, + { + "from": "gpt", + "value": "TYPE[Santorini]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.1343]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.2824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.9250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250]" + }, + { + "from": "gpt", + "value": "CLICK[0.4244,0.4080]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.1365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.6446]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3503,0.6647]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.3856]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5269]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5122]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5122],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.6087]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5122],SWIPE[UP],CLICK[0.3064,0.6087]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.6715]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5122],SWIPE[UP],CLICK[0.3064,0.6087],CLICK[0.6990,0.6715]" + }, + { + "from": "gpt", + "value": "CLICK[0.4962,0.9272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Santorini for the month of July. Include the filters \"non-smoking rooms\", \"sea views\", \"private pool\", \"breakfast included\", and \"located in Museum of Prehistoric Thera\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2532],CLICK[0.4333,0.2228],TYPE[Santorini],CLICK[0.3929,0.1343],CLICK[0.4401,0.2824],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4154,0.9250],CLICK[0.4244,0.4080],CLICK[0.4949,0.1365],SWIPE[UP],CLICK[0.6923,0.6446],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.5750],SWIPE[UP],CLICK[0.3503,0.6647],SWIPE[UP],CLICK[0.6990,0.3856],SWIPE[UP],CLICK[0.6946,0.5269],SWIPE[UP],SWIPE[UP],CLICK[0.7013,0.5122],SWIPE[UP],CLICK[0.3064,0.6087],CLICK[0.6990,0.6715],CLICK[0.4962,0.9272]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_39_975f7ef8607649d193a069d3db19d647-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6788,0.2510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.2048]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048]" + }, + { + "from": "gpt", + "value": "TYPE[Bali]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali]" + }, + { + "from": "gpt", + "value": "CLICK[0.4298,0.1330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.5590,0.5010]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010]" + }, + { + "from": "gpt", + "value": "CLICK[0.6599,0.7064]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.4135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.7131]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6958,0.5718]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.5718]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.5718],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.6750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.5718],SWIPE[UP],CLICK[0.6936,0.6750]" + }, + { + "from": "gpt", + "value": "CLICK[0.5510,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a holiday rental in Bali for the entire August. Filters include \"sea view\", \"breakfast included\" and \"private pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6788,0.2510],CLICK[0.3827,0.2048],TYPE[Bali],CLICK[0.4298,0.1330],CLICK[0.4388,0.2756],SWIPE[DOWN],CLICK[0.5590,0.5010],CLICK[0.6599,0.7064],CLICK[0.5016,0.9228],CLICK[0.4635,0.4135],CLICK[0.4872,0.1410],SWIPE[UP],CLICK[0.6946,0.5189],CLICK[0.7026,0.7131],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.5718],SWIPE[UP],CLICK[0.6936,0.6750],CLICK[0.5510,0.9228]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_43_da297fb0acec4aa8ac02e50b5c9bba38-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6667,0.2577]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577]" + }, + { + "from": "gpt", + "value": "CLICK[0.6218,0.1285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.2779]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.0635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635]" + }, + { + "from": "gpt", + "value": "TYPE[San Francisco]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.4135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135]" + }, + { + "from": "gpt", + "value": "CLICK[0.3795,0.3429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.6064]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064]" + }, + { + "from": "gpt", + "value": "CLICK[0.3256,0.6580]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.9317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.5381]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381]" + }, + { + "from": "gpt", + "value": "CLICK[0.5971,0.1365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381],CLICK[0.5971,0.1365]" + }, + { + "from": "gpt", + "value": "CLICK[0.3077,0.4417]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381],CLICK[0.5971,0.1365],CLICK[0.3077,0.4417]" + }, + { + "from": "gpt", + "value": "CLICK[0.4321,0.3788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381],CLICK[0.5971,0.1365],CLICK[0.3077,0.4417],CLICK[0.4321,0.3788]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381],CLICK[0.5971,0.1365],CLICK[0.3077,0.4417],CLICK[0.4321,0.3788],CLICK[0.4522,0.9205]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381],CLICK[0.5971,0.1365],CLICK[0.3077,0.4417],CLICK[0.4321,0.3788],CLICK[0.4522,0.9205],CLICK[0.4119,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.5006,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in San Francisco. Pickup on July 15 and drop off on July 21. Choose an SUV. Sort by customer rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6667,0.2577],CLICK[0.6218,0.1285],CLICK[0.3939,0.2779],CLICK[0.6946,0.0635],TYPE[San Francisco],CLICK[0.3728,0.4135],CLICK[0.3795,0.3429],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3917,0.6064],CLICK[0.3256,0.6580],CLICK[0.4051,0.9317],CLICK[0.4567,0.5381],CLICK[0.5971,0.1365],CLICK[0.3077,0.4417],CLICK[0.4321,0.3788],CLICK[0.4522,0.9205],CLICK[0.4119,0.1263],CLICK[0.5006,0.9205]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_45_87b4421eb48c4abab4511282f08f8e07-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6429,0.2564]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564]" + }, + { + "from": "gpt", + "value": "CLICK[0.4096,0.2093]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093]" + }, + { + "from": "gpt", + "value": "TYPE[Manhattan]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.1365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365]" + }, + { + "from": "gpt", + "value": "CLICK[0.3907,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.3894,0.5471]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471]" + }, + { + "from": "gpt", + "value": "CLICK[0.6744,0.5436]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.4244,0.3282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.6907]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907]" + }, + { + "from": "gpt", + "value": "CLICK[0.4692,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4276,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022]" + }, + { + "from": "gpt", + "value": "CLICK[0.4849,0.1487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487]" + }, + { + "from": "gpt", + "value": "CLICK[0.3077,0.6378]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.4686]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6907]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.6481]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.7939]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.3272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.3272]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.3272],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.3080]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.3272],SWIPE[UP],CLICK[0.6913,0.3080]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.3272],SWIPE[UP],CLICK[0.6913,0.3080],CLICK[0.4423,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Manhattan, New York, from August 5 to August 10. 3 guests in one room. Price range $600 to $2500. Filters: very good rating, excellent location, inclusive breakfast, non-smoking rooms and pets allowed.\nHistory Actions: SWIPE[LEFT],CLICK[0.6429,0.2564],CLICK[0.4096,0.2093],TYPE[Manhattan],CLICK[0.3929,0.1365],CLICK[0.3907,0.2734],CLICK[0.3894,0.5471],CLICK[0.6744,0.5436],CLICK[0.4625,0.9160],CLICK[0.4244,0.3282],CLICK[0.6869,0.6907],CLICK[0.4692,0.9340],CLICK[0.4276,0.4022],CLICK[0.4849,0.1487],CLICK[0.3077,0.6378],SWIPE[LEFT],SWIPE[UP],CLICK[0.7003,0.4686],SWIPE[UP],CLICK[0.6968,0.6907],SWIPE[UP],CLICK[0.7003,0.6481],CLICK[0.6936,0.7939],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.3272],SWIPE[UP],CLICK[0.6913,0.3080],CLICK[0.4423,0.9228],CLICK[0.4423,0.9228]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_47_29e0088098604b68a46f760c2ce8ded0-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6497,0.2510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.2151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151]" + }, + { + "from": "gpt", + "value": "TYPE[Lake Tahoe]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.2856]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856]" + }, + { + "from": "gpt", + "value": "CLICK[0.6093,0.5481]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481]" + }, + { + "from": "gpt", + "value": "CLICK[0.5567,0.5997]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997]" + }, + { + "from": "gpt", + "value": "CLICK[0.4881,0.9151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151]" + }, + { + "from": "gpt", + "value": "CLICK[0.4715,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362]" + }, + { + "from": "gpt", + "value": "CLICK[0.5689,0.6849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849]" + }, + { + "from": "gpt", + "value": "CLICK[0.5801,0.6862]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862]" + }, + { + "from": "gpt", + "value": "CLICK[0.5119,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205]" + }, + { + "from": "gpt", + "value": "CLICK[0.5276,0.4112]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112]" + }, + { + "from": "gpt", + "value": "CLICK[0.5051,0.1500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7080,0.8276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.3744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.5112]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744],SWIPE[DOWN],CLICK[0.3064,0.5112]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.7131]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744],SWIPE[DOWN],CLICK[0.3064,0.5112],CLICK[0.7026,0.7131]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744],SWIPE[DOWN],CLICK[0.3064,0.5112],CLICK[0.7026,0.7131],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.4462]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744],SWIPE[DOWN],CLICK[0.3064,0.5112],CLICK[0.7026,0.7131],SWIPE[DOWN],CLICK[0.7035,0.4462]" + }, + { + "from": "gpt", + "value": "CLICK[0.4782,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a cabin for rent in Lake Tahoe from 9th August to 15th August. 1 adult. Apply filters: view, pet-friendly and free parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6497,0.2510],CLICK[0.4410,0.2151],TYPE[Lake Tahoe],CLICK[0.4029,0.1263],CLICK[0.4154,0.2856],CLICK[0.6093,0.5481],CLICK[0.5567,0.5997],CLICK[0.4881,0.9151],CLICK[0.4715,0.3362],CLICK[0.5689,0.6849],CLICK[0.5801,0.6862],CLICK[0.5119,0.9205],CLICK[0.5276,0.4112],CLICK[0.5051,0.1500],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.8276],SWIPE[LEFT],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6981,0.3744],SWIPE[DOWN],CLICK[0.3064,0.5112],CLICK[0.7026,0.7131],SWIPE[DOWN],CLICK[0.7035,0.4462],CLICK[0.4782,0.9138]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_51_e9334f278ce641d48b27071e17847183-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.2519]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519]" + }, + { + "from": "gpt", + "value": "CLICK[0.4590,0.1208]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208]" + }, + { + "from": "gpt", + "value": "CLICK[0.4446,0.2196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.2846]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.1679]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679]" + }, + { + "from": "gpt", + "value": "TYPE[Tokyo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.3272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272]" + }, + { + "from": "gpt", + "value": "TYPE[Berlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.4163,0.2833]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833]" + }, + { + "from": "gpt", + "value": "CLICK[0.4019,0.4035]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035]" + }, + { + "from": "gpt", + "value": "CLICK[0.4333,0.8038]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038]" + }, + { + "from": "gpt", + "value": "CLICK[0.4917,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.4574]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240],CLICK[0.3885,0.4574]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.2218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240],CLICK[0.3885,0.4574],CLICK[0.6913,0.2218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3032,0.5032]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240],CLICK[0.3885,0.4574],CLICK[0.6913,0.2218],CLICK[0.3032,0.5032]" + }, + { + "from": "gpt", + "value": "CLICK[0.5859,0.9128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240],CLICK[0.3885,0.4574],CLICK[0.6913,0.2218],CLICK[0.3032,0.5032],CLICK[0.5859,0.9128]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.5817]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240],CLICK[0.3885,0.4574],CLICK[0.6913,0.2218],CLICK[0.3032,0.5032],CLICK[0.5859,0.9128],CLICK[0.6869,0.5817]" + }, + { + "from": "gpt", + "value": "CLICK[0.4545,0.5099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Tokyo to Berlin. Departure date 25 June. 2 adults in business class. Filter for direct flights.\nHistory Actions: SWIPE[LEFT],CLICK[0.6487,0.2519],CLICK[0.4590,0.1208],CLICK[0.4446,0.2196],CLICK[0.4288,0.2846],CLICK[0.4378,0.1679],TYPE[Tokyo],CLICK[0.4064,0.2712],CLICK[0.3817,0.3272],TYPE[Berlin],CLICK[0.4163,0.2833],CLICK[0.4019,0.4035],CLICK[0.4333,0.8038],CLICK[0.4917,0.9240],CLICK[0.3885,0.4574],CLICK[0.6913,0.2218],CLICK[0.3032,0.5032],CLICK[0.5859,0.9128],CLICK[0.6869,0.5817],CLICK[0.4545,0.5099]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_55_15b06c0282f4469789f0c8b371c7218e-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6564,0.2397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397]" + }, + { + "from": "gpt", + "value": "CLICK[0.4535,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.1724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724]" + }, + { + "from": "gpt", + "value": "TYPE[Chicago]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.4131,0.3295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.4208,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.4513,0.2138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138]" + }, + { + "from": "gpt", + "value": "CLICK[0.3391,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000]" + }, + { + "from": "gpt", + "value": "CLICK[0.4433,0.8096]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.4529]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529],CLICK[0.6824,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.5869,0.9285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529],CLICK[0.6824,0.2744],CLICK[0.5869,0.9285]" + }, + { + "from": "gpt", + "value": "CLICK[0.4769,0.5179]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529],CLICK[0.6824,0.2744],CLICK[0.5869,0.9285],CLICK[0.4769,0.5179]" + }, + { + "from": "gpt", + "value": "CLICK[0.4433,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529],CLICK[0.6824,0.2744],CLICK[0.5869,0.9285],CLICK[0.4769,0.5179],CLICK[0.4433,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529],CLICK[0.6824,0.2744],CLICK[0.5869,0.9285],CLICK[0.4769,0.5179],CLICK[0.4433,0.1276],CLICK[0.6968,0.6760]" + }, + { + "from": "gpt", + "value": "CLICK[0.6497,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Chicago to London departing 25th June. 1 adult and 1 children in economy class. One-stop maximum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2397],CLICK[0.4535,0.1276],CLICK[0.4410,0.2744],CLICK[0.4378,0.1724],TYPE[Chicago],CLICK[0.3885,0.2631],CLICK[0.4131,0.3295],TYPE[London],CLICK[0.4208,0.2721],CLICK[0.4513,0.2138],CLICK[0.3391,0.4000],CLICK[0.4433,0.8096],CLICK[0.4625,0.9160],CLICK[0.4288,0.4529],CLICK[0.6824,0.2744],CLICK[0.5869,0.9285],CLICK[0.4769,0.5179],CLICK[0.4433,0.1276],CLICK[0.6968,0.6760],CLICK[0.6497,0.9218]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_5_6585e99b068b4e02ac74aa3580e25e8c-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6676,0.2487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487]" + }, + { + "from": "gpt", + "value": "CLICK[0.3907,0.2083]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083]" + }, + { + "from": "gpt", + "value": "TYPE[Barcelona]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.1442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.5526]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.6660]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022]" + }, + { + "from": "gpt", + "value": "CLICK[0.4971,0.1478]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.4561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.6737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.6939]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737],SWIPE[UP],CLICK[0.6901,0.6939]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737],SWIPE[UP],CLICK[0.6901,0.6939],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.5718]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737],SWIPE[UP],CLICK[0.6901,0.6939],SWIPE[UP],CLICK[0.3064,0.5718]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.8298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737],SWIPE[UP],CLICK[0.6901,0.6939],SWIPE[UP],CLICK[0.3064,0.5718],CLICK[0.6990,0.8298]" + }, + { + "from": "gpt", + "value": "CLICK[0.5330,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for a vacation home for rent in Barcelona for New Year\"s Eve. Adjust filters for city view, pet-friendly and room service.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.3907,0.2083],TYPE[Barcelona],CLICK[0.3974,0.1442],CLICK[0.4154,0.2734],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4388,0.5526],CLICK[0.5029,0.6660],CLICK[0.4984,0.9218],CLICK[0.4625,0.4022],CLICK[0.4971,0.1478],SWIPE[UP],CLICK[0.7013,0.4561],SWIPE[UP],CLICK[0.7035,0.6737],SWIPE[UP],CLICK[0.6901,0.6939],SWIPE[UP],CLICK[0.3064,0.5718],CLICK[0.6990,0.8298],CLICK[0.5330,0.9205]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_14_7_00b131dc132946e186a38730c90f0d58-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6756,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.4814,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4311,0.2699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.1724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.3526,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.3340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340]" + }, + { + "from": "gpt", + "value": "TYPE[Bangkok]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.3833]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833]" + }, + { + "from": "gpt", + "value": "CLICK[0.3288,0.6423]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423]" + }, + { + "from": "gpt", + "value": "CLICK[0.5545,0.7490]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490]" + }, + { + "from": "gpt", + "value": "CLICK[0.4208,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3984,0.4596]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.2160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160]" + }, + { + "from": "gpt", + "value": "CLICK[0.6833,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160],CLICK[0.6833,0.2788]" + }, + { + "from": "gpt", + "value": "CLICK[0.6106,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160],CLICK[0.6833,0.2788],CLICK[0.6106,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5808]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160],CLICK[0.6833,0.2788],CLICK[0.6106,0.9240],CLICK[0.6946,0.5808]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.5337]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160],CLICK[0.6833,0.2788],CLICK[0.6106,0.9240],CLICK[0.6946,0.5808],CLICK[0.5016,0.5337]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160],CLICK[0.6833,0.2788],CLICK[0.6106,0.9240],CLICK[0.6946,0.5808],CLICK[0.5016,0.5337],CLICK[0.3715,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3503,0.9420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Sydney to Bangkok. Departure date 2 June, return date 20 June. 2 adults and 1 child. Economy class. Sort by price. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2587],CLICK[0.4814,0.1375],CLICK[0.4311,0.2699],CLICK[0.4478,0.1724],TYPE[Sydney],CLICK[0.3526,0.2654],CLICK[0.3849,0.3340],TYPE[Bangkok],CLICK[0.4221,0.2801],CLICK[0.4625,0.3833],CLICK[0.3288,0.6423],CLICK[0.5545,0.7490],CLICK[0.4208,0.9218],CLICK[0.3984,0.4596],CLICK[0.6869,0.2160],CLICK[0.6833,0.2788],CLICK[0.6106,0.9240],CLICK[0.6946,0.5808],CLICK[0.5016,0.5337],CLICK[0.3715,0.1276],CLICK[0.3503,0.9420]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_0_016cd29c6da8488296e7ef18f014aee1-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6519,0.2554]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554]" + }, + { + "from": "gpt", + "value": "CLICK[0.4006,0.2083]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083]" + }, + { + "from": "gpt", + "value": "TYPE[Ibiza]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.1433]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.2689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5635,0.5212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212]" + }, + { + "from": "gpt", + "value": "CLICK[0.6667,0.7378]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378]" + }, + { + "from": "gpt", + "value": "CLICK[0.4612,0.9375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4087,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362]" + }, + { + "from": "gpt", + "value": "CLICK[0.5792,0.2497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497]" + }, + { + "from": "gpt", + "value": "CLICK[0.5734,0.3282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282]" + }, + { + "from": "gpt", + "value": "CLICK[0.4837,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4163,0.4090]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.1522]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3042,0.7119]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.4708]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.7468]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.4224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.4224]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.4224],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.5224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.4224],SWIPE[UP],CLICK[0.6968,0.5224]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a cottage to rent in Ibiza for the month of August. 2 adults. Include the filters \"sea view\", \"breakfast included\", and \"swimming pool\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6519,0.2554],CLICK[0.4006,0.2083],TYPE[Ibiza],CLICK[0.3872,0.1433],CLICK[0.4154,0.2689],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.5212],CLICK[0.6667,0.7378],CLICK[0.4612,0.9375],CLICK[0.4087,0.3362],CLICK[0.5792,0.2497],CLICK[0.5734,0.3282],CLICK[0.4837,0.9340],CLICK[0.4163,0.4090],CLICK[0.5016,0.1522],SWIPE[UP],CLICK[0.3042,0.7119],CLICK[0.6946,0.4708],SWIPE[UP],CLICK[0.6981,0.7468],SWIPE[UP],SWIPE[UP],CLICK[0.6946,0.4224],SWIPE[UP],CLICK[0.6968,0.5224],CLICK[0.5016,0.9160]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_10_9c05abab2740442f90660f4ba1e355f9-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6676,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644]" + }, + { + "from": "gpt", + "value": "CLICK[0.4859,0.1196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.4468,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[Moscow]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.2811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.3215]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215]" + }, + { + "from": "gpt", + "value": "TYPE[Paris]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris]" + }, + { + "from": "gpt", + "value": "CLICK[0.3683,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.3978]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.4897]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978],SWIPE[UP],SWIPE[UP],CLICK[0.4401,0.4897]" + }, + { + "from": "gpt", + "value": "CLICK[0.6173,0.5436]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978],SWIPE[UP],SWIPE[UP],CLICK[0.4401,0.4897],CLICK[0.6173,0.5436]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.9038]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978],SWIPE[UP],SWIPE[UP],CLICK[0.4401,0.4897],CLICK[0.6173,0.5436],CLICK[0.4343,0.9038]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.5269]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Moscow to Paris. Departure date 10 September, return date 20 September. 1 adult. Economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2644],CLICK[0.4859,0.1196],CLICK[0.4500,0.2801],CLICK[0.4468,0.1689],TYPE[Moscow],CLICK[0.3917,0.2811],CLICK[0.4029,0.3215],TYPE[Paris],CLICK[0.3683,0.2712],CLICK[0.4029,0.3978],SWIPE[UP],SWIPE[UP],CLICK[0.4401,0.4897],CLICK[0.6173,0.5436],CLICK[0.4343,0.9038],CLICK[0.4141,0.5269]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_14_d019ef060c2b40259458a5a8d8589b8f-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6554,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.5936,0.1218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.2699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.0679]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679]" + }, + { + "from": "gpt", + "value": "TYPE[Los Angeles]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles]" + }, + { + "from": "gpt", + "value": "CLICK[0.3862,0.2330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.3372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6712,0.4426]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426]" + }, + { + "from": "gpt", + "value": "CLICK[0.3189,0.6064]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.4904,0.5202]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202]" + }, + { + "from": "gpt", + "value": "CLICK[0.6285,0.1420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202],CLICK[0.6285,0.1420]" + }, + { + "from": "gpt", + "value": "CLICK[0.3199,0.4865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202],CLICK[0.6285,0.1420],CLICK[0.3199,0.4865]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.4394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202],CLICK[0.6285,0.1420],CLICK[0.3199,0.4865],CLICK[0.4266,0.4394]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.9115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202],CLICK[0.6285,0.1420],CLICK[0.3199,0.4865],CLICK[0.4266,0.4394],CLICK[0.4288,0.9115]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.1353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202],CLICK[0.6285,0.1420],CLICK[0.3199,0.4865],CLICK[0.4266,0.4394],CLICK[0.4288,0.9115],CLICK[0.4029,0.1353]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.9317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Reserve a rental car in Los Angeles. Pickup on June 15 and drop off on June 30. Choose a SUV class. Sort by customer reviews.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2587],CLICK[0.5936,0.1218],CLICK[0.4356,0.2699],CLICK[0.6901,0.0679],TYPE[Los Angeles],CLICK[0.3862,0.2330],CLICK[0.4064,0.3372],SWIPE[DOWN],CLICK[0.6712,0.4426],CLICK[0.3189,0.6064],CLICK[0.4401,0.9240],CLICK[0.4904,0.5202],CLICK[0.6285,0.1420],CLICK[0.3199,0.4865],CLICK[0.4266,0.4394],CLICK[0.4288,0.9115],CLICK[0.4029,0.1353],CLICK[0.3962,0.9317]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_15_e4cf61ddd9404357bca515cbe22df044-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6721,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.4692,0.1330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.1702]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702]" + }, + { + "from": "gpt", + "value": "TYPE[Los Angeles]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.3228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.3968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.4381]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.5471]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471]" + }, + { + "from": "gpt", + "value": "CLICK[0.4535,0.9093]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093]" + }, + { + "from": "gpt", + "value": "CLICK[0.4019,0.4583]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.2228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.6083,0.9128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.5212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128],CLICK[0.4522,0.5212]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128],CLICK[0.4522,0.5212],CLICK[0.3715,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.3244,0.8913]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128],CLICK[0.4522,0.5212],CLICK[0.3715,0.1298],CLICK[0.3244,0.8913]" + }, + { + "from": "gpt", + "value": "CLICK[0.4490,0.1196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128],CLICK[0.4522,0.5212],CLICK[0.3715,0.1298],CLICK[0.3244,0.8913],CLICK[0.4490,0.1196]" + }, + { + "from": "gpt", + "value": "CLICK[0.3952,0.7365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128],CLICK[0.4522,0.5212],CLICK[0.3715,0.1298],CLICK[0.3244,0.8913],CLICK[0.4490,0.1196],CLICK[0.3952,0.7365]" + }, + { + "from": "gpt", + "value": "CLICK[0.6712,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Los Angeles to Sydney, depart 1 July, return 15 July with 2 adults and 1 child in economy class and sort by duration with max one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.4692,0.1330],CLICK[0.4221,0.2788],CLICK[0.4423,0.1702],TYPE[Los Angeles],CLICK[0.3997,0.2756],CLICK[0.4109,0.3228],TYPE[Sydney],CLICK[0.3917,0.2766],CLICK[0.4042,0.3968],SWIPE[UP],CLICK[0.3885,0.4381],CLICK[0.3782,0.5471],CLICK[0.4535,0.9093],CLICK[0.4019,0.4583],CLICK[0.6869,0.2228],CLICK[0.6891,0.2801],CLICK[0.6083,0.9128],CLICK[0.4522,0.5212],CLICK[0.3715,0.1298],CLICK[0.3244,0.8913],CLICK[0.4490,0.1196],CLICK[0.3952,0.7365],CLICK[0.6712,0.9205]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_17_1fed0a6b768a43f189c9f6963a18a02b-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6587,0.2452]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.2183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183]" + }, + { + "from": "gpt", + "value": "TYPE[Bali]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410]" + }, + { + "from": "gpt", + "value": "CLICK[0.4176,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.5481]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.6580]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.4013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013]" + }, + { + "from": "gpt", + "value": "CLICK[0.5038,0.1500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.7087]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.7827]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087],SWIPE[UP],CLICK[0.6946,0.7827]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087],SWIPE[UP],CLICK[0.6946,0.7827],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087],SWIPE[UP],CLICK[0.6946,0.7827],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.6244]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087],SWIPE[UP],CLICK[0.6946,0.7827],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.6244]" + }, + { + "from": "gpt", + "value": "CLICK[0.4199,0.9295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a luxury villa in Bali from 5 August to 20 August with filters on \"private pool\", \"sea view\" and \"breakfast included\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2452],CLICK[0.4074,0.2183],TYPE[Bali],CLICK[0.3917,0.1410],CLICK[0.4176,0.2744],CLICK[0.3827,0.5481],CLICK[0.4410,0.6580],CLICK[0.4074,0.9330],CLICK[0.4567,0.4013],CLICK[0.5038,0.1500],SWIPE[UP],CLICK[0.6946,0.5122],CLICK[0.6981,0.7087],SWIPE[UP],CLICK[0.6946,0.7827],SWIPE[UP],SWIPE[UP],CLICK[0.6923,0.6244],CLICK[0.4199,0.9295]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_19_a4972a3ad505413e815265a41822d675-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6587,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.5881,0.1196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.2667]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.0625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625]" + }, + { + "from": "gpt", + "value": "TYPE[Berlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.3490,0.2205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205]" + }, + { + "from": "gpt", + "value": "CLICK[0.3458,0.3327]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6128,0.6333]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333]" + }, + { + "from": "gpt", + "value": "CLICK[0.3772,0.7500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500]" + }, + { + "from": "gpt", + "value": "CLICK[0.4917,0.9375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.3340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340]" + }, + { + "from": "gpt", + "value": "CLICK[0.3737,0.3529]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529]" + }, + { + "from": "gpt", + "value": "CLICK[0.6362,0.3968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968]" + }, + { + "from": "gpt", + "value": "CLICK[0.3670,0.3295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295]" + }, + { + "from": "gpt", + "value": "CLICK[0.4558,0.5359]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359]" + }, + { + "from": "gpt", + "value": "CLICK[0.6128,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359],CLICK[0.6128,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.5913,0.5606]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359],CLICK[0.6128,0.1263],CLICK[0.5913,0.5606]" + }, + { + "from": "gpt", + "value": "CLICK[0.3221,0.7949]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359],CLICK[0.6128,0.1263],CLICK[0.5913,0.5606],CLICK[0.3221,0.7949]" + }, + { + "from": "gpt", + "value": "CLICK[0.3244,0.4394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359],CLICK[0.6128,0.1263],CLICK[0.5913,0.5606],CLICK[0.3221,0.7949],CLICK[0.3244,0.4394]" + }, + { + "from": "gpt", + "value": "CLICK[0.4096,0.4404]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359],CLICK[0.6128,0.1263],CLICK[0.5913,0.5606],CLICK[0.3221,0.7949],CLICK[0.3244,0.4394],CLICK[0.4096,0.4404]" + }, + { + "from": "gpt", + "value": "CLICK[0.4513,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Check car rent availability in Berlin from Nov 1 9 am to Nov 11 9 pm. Filter by SUV type, Europcar as a supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2465],CLICK[0.5881,0.1196],CLICK[0.4231,0.2667],CLICK[0.6913,0.0625],TYPE[Berlin],CLICK[0.3490,0.2205],CLICK[0.3458,0.3327],SWIPE[UP],SWIPE[UP],CLICK[0.6128,0.6333],CLICK[0.3772,0.7500],CLICK[0.4917,0.9375],CLICK[0.6487,0.3340],CLICK[0.3737,0.3529],CLICK[0.6362,0.3968],CLICK[0.3670,0.3295],CLICK[0.4558,0.5359],CLICK[0.6128,0.1263],CLICK[0.5913,0.5606],CLICK[0.3221,0.7949],CLICK[0.3244,0.4394],CLICK[0.4096,0.4404],CLICK[0.4513,0.9218]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_21_7d28ee4d5a03417fbcaad631de432dad-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6599,0.2452]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452]" + }, + { + "from": "gpt", + "value": "CLICK[0.3804,0.2083]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083]" + }, + { + "from": "gpt", + "value": "TYPE[Paris]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris]" + }, + { + "from": "gpt", + "value": "CLICK[0.3760,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3984,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6667,0.5157]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.6109]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.9115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115]" + }, + { + "from": "gpt", + "value": "CLICK[0.4253,0.3394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.6917]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.9151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.4013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013]" + }, + { + "from": "gpt", + "value": "CLICK[0.3548,0.1420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420]" + }, + { + "from": "gpt", + "value": "CLICK[0.3481,0.8904]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904]" + }, + { + "from": "gpt", + "value": "CLICK[0.4715,0.1353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5830]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.5728]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728]" + }, + { + "from": "gpt", + "value": "CLICK[0.3324,0.8343]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.7401]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP],CLICK[0.7013,0.7401]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP],CLICK[0.7013,0.7401],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5077]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP],CLICK[0.7013,0.7401],SWIPE[UP],CLICK[0.6946,0.5077]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP],CLICK[0.7013,0.7401],SWIPE[UP],CLICK[0.6946,0.5077],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.7199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP],CLICK[0.7013,0.7401],SWIPE[UP],CLICK[0.6946,0.5077],SWIPE[UP],CLICK[0.6946,0.7199]" + }, + { + "from": "gpt", + "value": "CLICK[0.4468,0.9128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Look for hotels in Paris, from June 1 to June 10. 3 guests in one room, sort by guest rating with filters on balcony, room service, double bed and breakfast included.\nHistory Actions: SWIPE[LEFT],CLICK[0.6599,0.2452],CLICK[0.3804,0.2083],TYPE[Paris],CLICK[0.3760,0.1276],CLICK[0.3984,0.2654],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6667,0.5157],CLICK[0.3817,0.6109],CLICK[0.4747,0.9115],CLICK[0.4253,0.3394],CLICK[0.6936,0.6917],CLICK[0.4635,0.9151],CLICK[0.4378,0.4013],CLICK[0.3548,0.1420],CLICK[0.3481,0.8904],CLICK[0.4715,0.1353],SWIPE[UP],CLICK[0.6946,0.5830],SWIPE[UP],CLICK[0.7071,0.5728],CLICK[0.3324,0.8343],SWIPE[UP],CLICK[0.7013,0.7401],SWIPE[UP],CLICK[0.6946,0.5077],SWIPE[UP],CLICK[0.6946,0.7199],CLICK[0.4468,0.9128]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_23_b095339b935846ab85f5cac13123149e-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6622,0.2487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.1218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766]" + }, + { + "from": "gpt", + "value": "TYPE[Heathrow Airport]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport]" + }, + { + "from": "gpt", + "value": "CLICK[0.3660,0.3304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304]" + }, + { + "from": "gpt", + "value": "TYPE[thBritish Museum]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum]" + }, + { + "from": "gpt", + "value": "CLICK[0.3804,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349]" + }, + { + "from": "gpt", + "value": "CLICK[0.4006,0.3978]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.9038]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.8106]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038],CLICK[0.6913,0.8106]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038],CLICK[0.6913,0.8106],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038],CLICK[0.6913,0.8106],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038],CLICK[0.6913,0.8106],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.5141,0.9083]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038],CLICK[0.6913,0.8106],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.5141,0.9083]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.4596]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in London for tomorrow at 5 pm. Pick-up location is Heathrow Airport and destination is the British Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2487],CLICK[0.7026,0.1218],CLICK[0.3715,0.2766],TYPE[Heathrow Airport],CLICK[0.3660,0.3304],TYPE[thBritish Museum],CLICK[0.3804,0.3349],CLICK[0.4006,0.3978],CLICK[0.4378,0.9038],CLICK[0.6913,0.8106],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.5141,0.9083],CLICK[0.4119,0.4596]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_2_ecdd4d6c6c904c48a8ae6d6f39cacbae-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.2474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.2205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.4657,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.5500,0.5458]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458]" + }, + { + "from": "gpt", + "value": "CLICK[0.3705,0.6548]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.3670,0.4157]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157]" + }, + { + "from": "gpt", + "value": "CLICK[0.4859,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6279]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7058,0.6705]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279],SWIPE[UP],CLICK[0.7058,0.6705]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279],SWIPE[UP],CLICK[0.7058,0.6705],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.6401]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279],SWIPE[UP],CLICK[0.7058,0.6705],SWIPE[UP],CLICK[0.7013,0.6401]" + }, + { + "from": "gpt", + "value": "CLICK[0.7058,0.6872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279],SWIPE[UP],CLICK[0.7058,0.6705],SWIPE[UP],CLICK[0.7013,0.6401],CLICK[0.7058,0.6872]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for apartments in New York, stay from July 11 to July 22. 1 adult. Filters: \"balcony\", \"Wifi\" and \"Free parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2474],CLICK[0.4119,0.2205],TYPE[New York],CLICK[0.4365,0.1276],CLICK[0.4657,0.2801],CLICK[0.5500,0.5458],CLICK[0.3705,0.6548],CLICK[0.4231,0.9228],CLICK[0.3670,0.4157],CLICK[0.4859,0.1465],SWIPE[UP],CLICK[0.6968,0.6279],SWIPE[UP],CLICK[0.7058,0.6705],SWIPE[UP],CLICK[0.7013,0.6401],CLICK[0.7058,0.6872],CLICK[0.4266,0.9263]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_34_db416be932b8413bba38cd85e760cb61-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.2183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183]" + }, + { + "from": "gpt", + "value": "TYPE[Maldives]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.1397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3244,0.5234]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.7365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365]" + }, + { + "from": "gpt", + "value": "CLICK[0.5196,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4513,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022]" + }, + { + "from": "gpt", + "value": "CLICK[0.4814,0.1397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3513,0.5381]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP],CLICK[0.3513,0.5381]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.7333]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP],CLICK[0.3513,0.5381],CLICK[0.6981,0.7333]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP],CLICK[0.3513,0.5381],CLICK[0.6981,0.7333],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP],CLICK[0.3513,0.5381],CLICK[0.6981,0.7333],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.3788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP],CLICK[0.3513,0.5381],CLICK[0.6981,0.7333],SWIPE[UP],SWIPE[UP],CLICK[0.6891,0.3788]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.9474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa for rent in Maldives for December. Apply the filters \"free cancellation\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2766],CLICK[0.3849,0.2183],TYPE[Maldives],CLICK[0.3929,0.1397],CLICK[0.4064,0.2766],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3244,0.5234],CLICK[0.4343,0.7365],CLICK[0.5196,0.9340],CLICK[0.4513,0.4022],CLICK[0.4814,0.1397],SWIPE[UP],SWIPE[UP],CLICK[0.3513,0.5381],CLICK[0.6981,0.7333],SWIPE[UP],SWIPE[UP],CLICK[0.6891,0.3788],CLICK[0.4522,0.9474]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_36_a7e2849142ea458ab325e1eb688bbed6-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6554,0.2429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.2609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.1747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747]" + }, + { + "from": "gpt", + "value": "TYPE[Berlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.3535,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349]" + }, + { + "from": "gpt", + "value": "TYPE[Venice]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.2699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.3910]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3760,0.6131]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.6638]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638]" + }, + { + "from": "gpt", + "value": "CLICK[0.4298,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3894,0.4506]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218],CLICK[0.3894,0.4506]" + }, + { + "from": "gpt", + "value": "CLICK[0.6833,0.2205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218],CLICK[0.3894,0.4506],CLICK[0.6833,0.2205]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.5077]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218],CLICK[0.3894,0.4506],CLICK[0.6833,0.2205],CLICK[0.3064,0.5077]" + }, + { + "from": "gpt", + "value": "CLICK[0.5913,0.9407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218],CLICK[0.3894,0.4506],CLICK[0.6833,0.2205],CLICK[0.3064,0.5077],CLICK[0.5913,0.9407]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.5907]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218],CLICK[0.3894,0.4506],CLICK[0.6833,0.2205],CLICK[0.3064,0.5077],CLICK[0.5913,0.9407],CLICK[0.6968,0.5907]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a return flight from Berlin to Venice. Departure date 15 July, return date 22 July. 2 adults. Business class. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2429],CLICK[0.4423,0.1253],CLICK[0.4109,0.2609],CLICK[0.4388,0.1747],TYPE[Berlin],CLICK[0.3535,0.2676],CLICK[0.4154,0.3349],TYPE[Venice],CLICK[0.3715,0.2699],CLICK[0.4042,0.3910],SWIPE[UP],CLICK[0.3760,0.6131],CLICK[0.3782,0.6638],CLICK[0.4298,0.9218],CLICK[0.3894,0.4506],CLICK[0.6833,0.2205],CLICK[0.3064,0.5077],CLICK[0.5913,0.9407],CLICK[0.6968,0.5907],CLICK[0.4635,0.5189]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_39_af41b14992294864ac8c6abd4ab5ae2d-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6699,0.2510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510]" + }, + { + "from": "gpt", + "value": "CLICK[0.4702,0.1353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353]" + }, + { + "from": "gpt", + "value": "CLICK[0.4176,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.4468,0.1712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.3907,0.2846]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846]" + }, + { + "from": "gpt", + "value": "CLICK[0.3580,0.3282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282]" + }, + { + "from": "gpt", + "value": "TYPE[Dubai]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai]" + }, + { + "from": "gpt", + "value": "CLICK[0.3490,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.3901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.6939]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939]" + }, + { + "from": "gpt", + "value": "CLICK[0.5522,0.7401]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401]" + }, + { + "from": "gpt", + "value": "CLICK[0.4926,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196]" + }, + { + "from": "gpt", + "value": "CLICK[0.3750,0.4484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196],CLICK[0.3750,0.4484]" + }, + { + "from": "gpt", + "value": "CLICK[0.6766,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196],CLICK[0.3750,0.4484],CLICK[0.6766,0.2788]" + }, + { + "from": "gpt", + "value": "CLICK[0.3109,0.6401]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196],CLICK[0.3750,0.4484],CLICK[0.6766,0.2788],CLICK[0.3109,0.6401]" + }, + { + "from": "gpt", + "value": "CLICK[0.5667,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196],CLICK[0.3750,0.4484],CLICK[0.6766,0.2788],CLICK[0.3109,0.6401],CLICK[0.5667,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.5817]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196],CLICK[0.3750,0.4484],CLICK[0.6766,0.2788],CLICK[0.3109,0.6401],CLICK[0.5667,0.9330],CLICK[0.6801,0.5817]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.5346]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a flight from London to Dubai and return. Leave on June 10, come back on June 20. One adult and one child in business class. No stops preferred.\nHistory Actions: SWIPE[LEFT],CLICK[0.6699,0.2510],CLICK[0.4702,0.1353],CLICK[0.4176,0.2734],CLICK[0.4468,0.1712],TYPE[London],CLICK[0.3907,0.2846],CLICK[0.3580,0.3282],TYPE[Dubai],CLICK[0.3490,0.2734],CLICK[0.4051,0.3901],CLICK[0.3782,0.6939],CLICK[0.5522,0.7401],CLICK[0.4926,0.9196],CLICK[0.3750,0.4484],CLICK[0.6766,0.2788],CLICK[0.3109,0.6401],CLICK[0.5667,0.9330],CLICK[0.6801,0.5817],CLICK[0.4747,0.5346]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_40_e5359422e5b24ffba896334c14cb2749-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6474,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.4333,0.1321]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.2106]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.2811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.1724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.4087,0.2599]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.3372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372]" + }, + { + "from": "gpt", + "value": "TYPE[Madrid]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4545,0.3990]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5635,0.7333]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.7333]" + }, + { + "from": "gpt", + "value": "CLICK[0.4814,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.7333],CLICK[0.4814,0.9138]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.5885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.7333],CLICK[0.4814,0.9138],CLICK[0.6891,0.5885]" + }, + { + "from": "gpt", + "value": "CLICK[0.3481,0.5179]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from London to Madrid. Departure date 10 October. 1 adult in economy class. Filter for direct flights\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2542],CLICK[0.4333,0.1321],CLICK[0.4567,0.2106],CLICK[0.4288,0.2811],CLICK[0.4455,0.1724],TYPE[London],CLICK[0.4087,0.2599],CLICK[0.4119,0.3372],TYPE[Madrid],CLICK[0.3885,0.2756],CLICK[0.4545,0.3990],SWIPE[UP],SWIPE[UP],CLICK[0.5635,0.7333],CLICK[0.4814,0.9138],CLICK[0.6891,0.5885],CLICK[0.3481,0.5179]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_42_2681d3bef8204066be238149e007b720-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6452,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.5837,0.1240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240]" + }, + { + "from": "gpt", + "value": "CLICK[0.3580,0.2689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612]" + }, + { + "from": "gpt", + "value": "TYPE[Paris]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris]" + }, + { + "from": "gpt", + "value": "CLICK[0.3513,0.4897]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897]" + }, + { + "from": "gpt", + "value": "CLICK[0.3907,0.3340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6240,0.5090]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090]" + }, + { + "from": "gpt", + "value": "CLICK[0.3772,0.6144]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144]" + }, + { + "from": "gpt", + "value": "CLICK[0.4163,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.6385,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3683,0.2968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3683,0.2968]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.5202]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3683,0.2968],CLICK[0.4141,0.5202]" + }, + { + "from": "gpt", + "value": "CLICK[0.6093,0.1365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3683,0.2968],CLICK[0.4141,0.5202],CLICK[0.6093,0.1365]" + }, + { + "from": "gpt", + "value": "CLICK[0.3212,0.5561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3683,0.2968],CLICK[0.4141,0.5202],CLICK[0.6093,0.1365],CLICK[0.3212,0.5561]" + }, + { + "from": "gpt", + "value": "CLICK[0.3288,0.7378]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3683,0.2968],CLICK[0.4141,0.5202],CLICK[0.6093,0.1365],CLICK[0.3212,0.5561],CLICK[0.3288,0.7378]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Paris from July 5 9 am to July 15 11 am. Filter by coupe type, Avis Car Rental supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6452,0.2631],CLICK[0.5837,0.1240],CLICK[0.3580,0.2689],CLICK[0.6901,0.0612],TYPE[Paris],CLICK[0.3513,0.4897],CLICK[0.3907,0.3340],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.6240,0.5090],CLICK[0.3772,0.6144],CLICK[0.4163,0.9263],CLICK[0.6385,0.4058],SWIPE[RIGHT],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3683,0.2968],CLICK[0.4141,0.5202],CLICK[0.6093,0.1365],CLICK[0.3212,0.5561],CLICK[0.3288,0.7378],CLICK[0.4872,0.9160]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_44_3f1176830c5146a0b8d54b05b59cbce7-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6654,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644]" + }, + { + "from": "gpt", + "value": "CLICK[0.4792,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.2824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.1756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756]" + }, + { + "from": "gpt", + "value": "TYPE[Berlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.3369,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.3317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317]" + }, + { + "from": "gpt", + "value": "TYPE[Toronto]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.2667]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667]" + }, + { + "from": "gpt", + "value": "CLICK[0.4199,0.4035]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.5705]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705]" + }, + { + "from": "gpt", + "value": "CLICK[0.6228,0.6660]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.3625,0.4686]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.2250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.2833]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.2824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824]" + }, + { + "from": "gpt", + "value": "CLICK[0.5734,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4208,0.5144]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340],CLICK[0.4208,0.5144]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.1353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340],CLICK[0.4208,0.5144],CLICK[0.3715,0.1353]" + }, + { + "from": "gpt", + "value": "CLICK[0.3481,0.9362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340],CLICK[0.4208,0.5144],CLICK[0.3715,0.1353],CLICK[0.3481,0.9362]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340],CLICK[0.4208,0.5144],CLICK[0.3715,0.1353],CLICK[0.3481,0.9362],CLICK[0.4724,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.3279,0.8096]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340],CLICK[0.4208,0.5144],CLICK[0.3715,0.1353],CLICK[0.3481,0.9362],CLICK[0.4724,0.1263],CLICK[0.3279,0.8096]" + }, + { + "from": "gpt", + "value": "CLICK[0.6667,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a flight from Berlin to Toronto, depart 2 August, return 16 August with 2 adults and 2 children in economy class and sort by price with no more than two stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6654,0.2644],CLICK[0.4792,0.1298],CLICK[0.3962,0.2824],CLICK[0.4410,0.1756],TYPE[Berlin],CLICK[0.3369,0.2654],CLICK[0.3917,0.3317],TYPE[Toronto],CLICK[0.4042,0.2667],CLICK[0.4199,0.4035],SWIPE[UP],CLICK[0.6151,0.5705],CLICK[0.6228,0.6660],CLICK[0.4872,0.9183],CLICK[0.3625,0.4686],CLICK[0.6811,0.2250],CLICK[0.6923,0.2833],CLICK[0.6801,0.2824],CLICK[0.5734,0.9340],CLICK[0.4208,0.5144],CLICK[0.3715,0.1353],CLICK[0.3481,0.9362],CLICK[0.4724,0.1263],CLICK[0.3279,0.8096],CLICK[0.6667,0.9196]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_49_f59f9b00587345399a8e6b2ee6c4c89b-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.2519]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519]" + }, + { + "from": "gpt", + "value": "CLICK[0.4679,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.1734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734]" + }, + { + "from": "gpt", + "value": "TYPE[Berlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.3625,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.3385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385]" + }, + { + "from": "gpt", + "value": "TYPE[New Delhi]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.2811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.3933]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.4583]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.4673]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673]" + }, + { + "from": "gpt", + "value": "CLICK[0.5128,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.4583]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583]" + }, + { + "from": "gpt", + "value": "CLICK[0.6766,0.2263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.2779]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779]" + }, + { + "from": "gpt", + "value": "CLICK[0.5926,0.9397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.5234]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397],CLICK[0.4500,0.5234]" + }, + { + "from": "gpt", + "value": "CLICK[0.3401,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397],CLICK[0.4500,0.5234],CLICK[0.3401,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3670,0.8824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397],CLICK[0.4500,0.5234],CLICK[0.3401,0.1276],CLICK[0.3670,0.8824]" + }, + { + "from": "gpt", + "value": "CLICK[0.4490,0.1196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397],CLICK[0.4500,0.5234],CLICK[0.3401,0.1276],CLICK[0.3670,0.8824],CLICK[0.4490,0.1196]" + }, + { + "from": "gpt", + "value": "CLICK[0.3458,0.6728]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397],CLICK[0.4500,0.5234],CLICK[0.3401,0.1276],CLICK[0.3670,0.8824],CLICK[0.4490,0.1196],CLICK[0.3458,0.6728]" + }, + { + "from": "gpt", + "value": "CLICK[0.6856,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search round-trip flights from Berlin to New Delhi. Departure date 8 July, return date 12 July. 2 adults, 1 child. Economy class. Sort by total travel time with maximum one stop.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2519],CLICK[0.4679,0.1298],CLICK[0.3929,0.2788],CLICK[0.4401,0.1734],TYPE[Berlin],CLICK[0.3625,0.2721],CLICK[0.3728,0.3385],TYPE[New Delhi],CLICK[0.4119,0.2811],CLICK[0.3997,0.3933],SWIPE[UP],CLICK[0.3817,0.4583],CLICK[0.6151,0.4673],CLICK[0.5128,0.9330],CLICK[0.4042,0.4583],CLICK[0.6766,0.2263],CLICK[0.6869,0.2779],CLICK[0.5926,0.9397],CLICK[0.4500,0.5234],CLICK[0.3401,0.1276],CLICK[0.3670,0.8824],CLICK[0.4490,0.1196],CLICK[0.3458,0.6728],CLICK[0.6856,0.9353]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_4_543101eb91c7451da8f27f92a2757543-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6721,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.5881,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.4163,0.2689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.0670]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670]" + }, + { + "from": "gpt", + "value": "TYPE[San Francisco]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.4067]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067]" + }, + { + "from": "gpt", + "value": "CLICK[0.3804,0.3474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6474,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.4292]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292]" + }, + { + "from": "gpt", + "value": "CLICK[0.6250,0.4103]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4186,0.4269]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269]" + }, + { + "from": "gpt", + "value": "CLICK[0.4580,0.5247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269],CLICK[0.4580,0.5247]" + }, + { + "from": "gpt", + "value": "CLICK[0.6263,0.1397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269],CLICK[0.4580,0.5247],CLICK[0.6263,0.1397]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.3811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269],CLICK[0.4580,0.5247],CLICK[0.6263,0.1397],CLICK[0.4266,0.3811]" + }, + { + "from": "gpt", + "value": "CLICK[0.3795,0.6099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269],CLICK[0.4580,0.5247],CLICK[0.6263,0.1397],CLICK[0.4266,0.3811],CLICK[0.3795,0.6099]" + }, + { + "from": "gpt", + "value": "CLICK[0.3401,0.7881]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269],CLICK[0.4580,0.5247],CLICK[0.6263,0.1397],CLICK[0.4266,0.3811],CLICK[0.3795,0.6099],CLICK[0.3401,0.7881]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from July 5 8 am to July 15 10 am. Filter by medium type, Dollar as supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6721,0.2542],CLICK[0.5881,0.1253],CLICK[0.4163,0.2689],CLICK[0.6901,0.0670],TYPE[San Francisco],CLICK[0.3827,0.4067],CLICK[0.3804,0.3474],SWIPE[UP],CLICK[0.6474,0.3362],SWIPE[DOWN],CLICK[0.3962,0.4292],CLICK[0.6250,0.4103],SWIPE[DOWN],CLICK[0.4186,0.4269],CLICK[0.4580,0.5247],CLICK[0.6263,0.1397],CLICK[0.4266,0.3811],CLICK[0.3795,0.6099],CLICK[0.3401,0.7881],CLICK[0.4949,0.9196]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_51_e74c982e569d4bdfb83aa5f1aefcb175-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6676,0.2487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.1240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[Madrid]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.3385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.4298,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.3968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6756,0.6221]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221]" + }, + { + "from": "gpt", + "value": "CLICK[0.6689,0.6795]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.9362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362]" + }, + { + "from": "gpt", + "value": "CLICK[0.4186,0.4628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.2205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205]" + }, + { + "from": "gpt", + "value": "CLICK[0.6766,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766]" + }, + { + "from": "gpt", + "value": "CLICK[0.3077,0.6413]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766],CLICK[0.3077,0.6413]" + }, + { + "from": "gpt", + "value": "CLICK[0.5522,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766],CLICK[0.3077,0.6413],CLICK[0.5522,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.5830]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766],CLICK[0.3077,0.6413],CLICK[0.5522,0.9160],CLICK[0.6923,0.5830]" + }, + { + "from": "gpt", + "value": "CLICK[0.4670,0.5337]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766],CLICK[0.3077,0.6413],CLICK[0.5522,0.9160],CLICK[0.6923,0.5830],CLICK[0.4670,0.5337]" + }, + { + "from": "gpt", + "value": "CLICK[0.3378,0.1308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766],CLICK[0.3077,0.6413],CLICK[0.5522,0.9160],CLICK[0.6923,0.5830],CLICK[0.4670,0.5337],CLICK[0.3378,0.1308]" + }, + { + "from": "gpt", + "value": "CLICK[0.3737,0.8792]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from Madrid to London. Departure date August 3, return date August 10. 2 adults and 1 child. Business cabin class. Sort by duration. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2487],CLICK[0.4356,0.1240],CLICK[0.3817,0.2744],CLICK[0.4356,0.1689],TYPE[Madrid],CLICK[0.3827,0.2744],CLICK[0.3715,0.3385],TYPE[London],CLICK[0.4298,0.2712],CLICK[0.3782,0.3968],SWIPE[UP],CLICK[0.6756,0.6221],CLICK[0.6689,0.6795],CLICK[0.4455,0.9362],CLICK[0.4186,0.4628],CLICK[0.6824,0.2205],CLICK[0.6766,0.2766],CLICK[0.3077,0.6413],CLICK[0.5522,0.9160],CLICK[0.6923,0.5830],CLICK[0.4670,0.5337],CLICK[0.3378,0.1308],CLICK[0.3737,0.8792]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_54_db6add737de1446587aa155da634bc5e-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6622,0.2577]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.2138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138]" + }, + { + "from": "gpt", + "value": "TYPE[Berlin]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.3670,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122]" + }, + { + "from": "gpt", + "value": "CLICK[0.6721,0.7199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.9151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151]" + }, + { + "from": "gpt", + "value": "CLICK[0.4276,0.4157]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157]" + }, + { + "from": "gpt", + "value": "CLICK[0.4962,0.1487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.6615]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6580]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.4202]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7048,0.6670]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7048,0.6670]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a rental flat in Berlin for the month of November. Apply the filters \"2 bedrooms\", \"near city center\" and \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2577],CLICK[0.3939,0.2138],TYPE[Berlin],CLICK[0.3715,0.1375],CLICK[0.3670,0.2712],SWIPE[DOWN],CLICK[0.6151,0.5122],CLICK[0.6721,0.7199],CLICK[0.4388,0.9151],CLICK[0.4276,0.4157],CLICK[0.4962,0.1487],SWIPE[UP],CLICK[0.6936,0.6615],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6580],SWIPE[UP],SWIPE[UP],CLICK[0.6936,0.4202],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7048,0.6670],CLICK[0.4478,0.9218]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_56_04c39e8396e64629bf21d9e0e7ad5089-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6465,0.2474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474]" + }, + { + "from": "gpt", + "value": "CLICK[0.6061,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3692,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.0657]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657]" + }, + { + "from": "gpt", + "value": "TYPE[Barcelona]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona]" + }, + { + "from": "gpt", + "value": "CLICK[0.3827,0.2295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295]" + }, + { + "from": "gpt", + "value": "CLICK[0.3625,0.3340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340]" + }, + { + "from": "gpt", + "value": "CLICK[0.6712,0.4494]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.5481]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481]" + }, + { + "from": "gpt", + "value": "CLICK[0.4019,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.5202]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353],CLICK[0.4221,0.5202]" + }, + { + "from": "gpt", + "value": "CLICK[0.5971,0.1321]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353],CLICK[0.4221,0.5202],CLICK[0.5971,0.1321]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.4404]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353],CLICK[0.4221,0.5202],CLICK[0.5971,0.1321],CLICK[0.4029,0.4404]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.9317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353],CLICK[0.4221,0.5202],CLICK[0.5971,0.1321],CLICK[0.4029,0.4404],CLICK[0.4401,0.9317]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353],CLICK[0.4221,0.5202],CLICK[0.5971,0.1321],CLICK[0.4029,0.4404],CLICK[0.4401,0.9317],CLICK[0.4064,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.3333,0.9375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Barcelona. Pick up on July 20, drop off on July 29. Midsize car size. Sort by customer ratings.\nHistory Actions: SWIPE[LEFT],CLICK[0.6465,0.2474],CLICK[0.6061,0.1276],CLICK[0.3692,0.2712],CLICK[0.6891,0.0657],TYPE[Barcelona],CLICK[0.3827,0.2295],CLICK[0.3625,0.3340],CLICK[0.6712,0.4494],CLICK[0.3885,0.5481],CLICK[0.4019,0.9353],CLICK[0.4221,0.5202],CLICK[0.5971,0.1321],CLICK[0.4029,0.4404],CLICK[0.4401,0.9317],CLICK[0.4064,0.1298],CLICK[0.3333,0.9375]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_59_3d4a39e943b24d24acf0eac2a7de6684-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.1330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.1644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.3705,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.3647,0.3282]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282]" + }, + { + "from": "gpt", + "value": "TYPE[Toronto]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto]" + }, + { + "from": "gpt", + "value": "CLICK[0.3446,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.3772,0.3933]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.7513]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513]" + }, + { + "from": "gpt", + "value": "CLICK[0.6173,0.7994]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513],CLICK[0.6173,0.7994]" + }, + { + "from": "gpt", + "value": "CLICK[0.4590,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513],CLICK[0.6173,0.7994],CLICK[0.4590,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.4606]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513],CLICK[0.6173,0.7994],CLICK[0.4590,0.9218],CLICK[0.4266,0.4606]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.2263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513],CLICK[0.6173,0.7994],CLICK[0.4590,0.9218],CLICK[0.4266,0.4606],CLICK[0.6824,0.2263]" + }, + { + "from": "gpt", + "value": "CLICK[0.6196,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513],CLICK[0.6173,0.7994],CLICK[0.4590,0.9218],CLICK[0.4266,0.4606],CLICK[0.6824,0.2263],CLICK[0.6196,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.5308,0.5212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for a return flight from Sydney to Toronto. Leave on June 18, come back on June 28. With 2 adults in economy class.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2420],CLICK[0.4747,0.1330],CLICK[0.4029,0.2744],CLICK[0.4455,0.1644],TYPE[Sydney],CLICK[0.3705,0.2721],CLICK[0.3647,0.3282],TYPE[Toronto],CLICK[0.3446,0.2801],CLICK[0.3772,0.3933],CLICK[0.4365,0.7513],CLICK[0.6173,0.7994],CLICK[0.4590,0.9218],CLICK[0.4266,0.4606],CLICK[0.6824,0.2263],CLICK[0.6196,0.9340],CLICK[0.5308,0.5212]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_6_213802710d024b368924915577356cc3-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.2160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4019,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.4096,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788]" + }, + { + "from": "gpt", + "value": "CLICK[0.3266,0.5494]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494]" + }, + { + "from": "gpt", + "value": "CLICK[0.6128,0.5471]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.9016]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.3250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.7625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.4468,0.3888]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888]" + }, + { + "from": "gpt", + "value": "CLICK[0.4962,0.1500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888],CLICK[0.4962,0.1500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888],CLICK[0.4962,0.1500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.4686]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888],CLICK[0.4962,0.1500],SWIPE[UP],CLICK[0.6901,0.4686]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.4641]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888],CLICK[0.4962,0.1500],SWIPE[UP],CLICK[0.6901,0.4686],CLICK[0.7026,0.4641]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.8330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888],CLICK[0.4962,0.1500],SWIPE[UP],CLICK[0.6901,0.4686],CLICK[0.7026,0.4641],CLICK[0.6936,0.8330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4769,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a hotel room in New York. From May 5 to May 10. Occupants are 3 adults, 1 child. Filter by \"breakfast included\" + \"parking\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.4119,0.2160],TYPE[New York],CLICK[0.4019,0.1298],CLICK[0.4096,0.2788],CLICK[0.3266,0.5494],CLICK[0.6128,0.5471],CLICK[0.4064,0.9016],CLICK[0.3593,0.3250],CLICK[0.6869,0.7625],CLICK[0.4984,0.9160],CLICK[0.4468,0.3888],CLICK[0.4962,0.1500],SWIPE[UP],CLICK[0.6901,0.4686],CLICK[0.7026,0.4641],CLICK[0.6936,0.8330],CLICK[0.4769,0.9263]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_15_8_538d42392cd74e6faeaa8e20a571470d-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.2173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.1455]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455]" + }, + { + "from": "gpt", + "value": "CLICK[0.3660,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.5426]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426]" + }, + { + "from": "gpt", + "value": "CLICK[0.4994,0.7590]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590]" + }, + { + "from": "gpt", + "value": "CLICK[0.4657,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000]" + }, + { + "from": "gpt", + "value": "CLICK[0.5051,0.1510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5567,0.6535]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5006,0.6728]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4433,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.5449]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP],CLICK[0.3593,0.5449]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP],CLICK[0.3593,0.5449],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP],CLICK[0.3593,0.5449],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP],CLICK[0.3593,0.5449],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3558,0.2878]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP],CLICK[0.3593,0.5449],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3558,0.2878]" + }, + { + "from": "gpt", + "value": "CLICK[0.4096,0.9317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in Rome for the month of July. Include the filters \"Terrace\", \"private bathroom\", \"breakfast included\", and \"located near Colosseum\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6801,0.2465],CLICK[0.4064,0.2173],TYPE[Rome],CLICK[0.3593,0.1455],CLICK[0.3660,0.2721],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3872,0.5426],CLICK[0.4994,0.7590],CLICK[0.4657,0.9340],CLICK[0.3715,0.4000],CLICK[0.5051,0.1510],SWIPE[UP],CLICK[0.5567,0.6535],SWIPE[UP],CLICK[0.5006,0.6728],SWIPE[UP],CLICK[0.4433,0.4058],SWIPE[UP],CLICK[0.3593,0.5449],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3558,0.2878],CLICK[0.4096,0.9317]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_2_350180107ba0434f920ba14a8a5ca2a6-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.2474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474]" + }, + { + "from": "gpt", + "value": "CLICK[0.4545,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.2115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115]" + }, + { + "from": "gpt", + "value": "CLICK[0.4096,0.2622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.1644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644]" + }, + { + "from": "gpt", + "value": "TYPE[Shanghai]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.3250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250]" + }, + { + "from": "gpt", + "value": "TYPE[SIngapore]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore]" + }, + { + "from": "gpt", + "value": "CLICK[0.3804,0.2811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.3990]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811],CLICK[0.4378,0.3990]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811],CLICK[0.4378,0.3990],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6106,0.5381]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811],CLICK[0.4378,0.3990],SWIPE[UP],CLICK[0.6106,0.5381]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811],CLICK[0.4378,0.3990],SWIPE[UP],CLICK[0.6106,0.5381],CLICK[0.4343,0.9353]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.5897]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811],CLICK[0.4378,0.3990],SWIPE[UP],CLICK[0.6106,0.5381],CLICK[0.4343,0.9353],CLICK[0.6913,0.5897]" + }, + { + "from": "gpt", + "value": "CLICK[0.3647,0.5247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Shanghai to Singapore. Departure date 12 July. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2474],CLICK[0.4545,0.1276],CLICK[0.4500,0.2115],CLICK[0.4096,0.2622],CLICK[0.4365,0.1644],TYPE[Shanghai],CLICK[0.4109,0.2756],CLICK[0.4029,0.3250],TYPE[SIngapore],CLICK[0.3804,0.2811],CLICK[0.4378,0.3990],SWIPE[UP],CLICK[0.6106,0.5381],CLICK[0.4343,0.9353],CLICK[0.6913,0.5897],CLICK[0.3647,0.5247]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_16_6_480de1c88f3f49df9537cbbdcdb680f1-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.2420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.1343]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343]" + }, + { + "from": "gpt", + "value": "CLICK[0.4490,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[Madrid]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid]" + }, + { + "from": "gpt", + "value": "CLICK[0.5522,0.2689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.3340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.4154,0.3901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.6872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.7423]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3683,0.4561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.6766,0.2824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824]" + }, + { + "from": "gpt", + "value": "CLICK[0.6891,0.2151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151]" + }, + { + "from": "gpt", + "value": "CLICK[0.3131,0.7657]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151],CLICK[0.3131,0.7657]" + }, + { + "from": "gpt", + "value": "CLICK[0.5410,0.9295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151],CLICK[0.3131,0.7657],CLICK[0.5410,0.9295]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.5920]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151],CLICK[0.3131,0.7657],CLICK[0.5410,0.9295],CLICK[0.6869,0.5920]" + }, + { + "from": "gpt", + "value": "CLICK[0.4715,0.5256]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151],CLICK[0.3131,0.7657],CLICK[0.5410,0.9295],CLICK[0.6869,0.5920],CLICK[0.4715,0.5256]" + }, + { + "from": "gpt", + "value": "CLICK[0.3436,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151],CLICK[0.3131,0.7657],CLICK[0.5410,0.9295],CLICK[0.6869,0.5920],CLICK[0.4715,0.5256],CLICK[0.3436,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.3010,0.8869]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from Madrid to Rome. Departure date 11 June, return date 19 June. 2 adults and 2 children. Business class. Sort by duration. Direct flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6510,0.2420],CLICK[0.4567,0.1343],CLICK[0.4490,0.2756],CLICK[0.4423,0.1689],TYPE[Madrid],CLICK[0.5522,0.2689],CLICK[0.4109,0.3340],TYPE[Rome],CLICK[0.4724,0.2801],CLICK[0.4154,0.3901],CLICK[0.4365,0.6872],CLICK[0.5016,0.7423],CLICK[0.5029,0.9218],CLICK[0.3683,0.4561],CLICK[0.6891,0.2721],CLICK[0.6766,0.2824],CLICK[0.6891,0.2151],CLICK[0.3131,0.7657],CLICK[0.5410,0.9295],CLICK[0.6869,0.5920],CLICK[0.4715,0.5256],CLICK[0.3436,0.1253],CLICK[0.3010,0.8869]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_12_bbc083b05b4a4c8cb218634e2f316f91-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.2048]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048]" + }, + { + "from": "gpt", + "value": "TYPE[Central, Hong Kong]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong]" + }, + { + "from": "gpt", + "value": "CLICK[0.3683,0.1913]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913]" + }, + { + "from": "gpt", + "value": "CLICK[0.5016,0.6571]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571]" + }, + { + "from": "gpt", + "value": "CLICK[0.3333,0.7176]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176]" + }, + { + "from": "gpt", + "value": "CLICK[0.4760,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.3340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340]" + }, + { + "from": "gpt", + "value": "CLICK[0.6869,0.8298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000]" + }, + { + "from": "gpt", + "value": "CLICK[0.5006,0.1455]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3077,0.3304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP],CLICK[0.6968,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.3244,0.6369]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP],CLICK[0.6968,0.2744],CLICK[0.3244,0.6369]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.6849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP],CLICK[0.6968,0.2744],CLICK[0.3244,0.6369],CLICK[0.6946,0.6849]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP],CLICK[0.6968,0.2744],CLICK[0.3244,0.6369],CLICK[0.6946,0.6849],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.6526]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP],CLICK[0.6968,0.2744],CLICK[0.3244,0.6369],CLICK[0.6946,0.6849],SWIPE[UP],CLICK[0.6990,0.6526]" + }, + { + "from": "gpt", + "value": "CLICK[0.4827,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Central, Hong Kong, from 22 May to 26 May. 3 guests in 1 triple room with pets. $2000 to $3000. Filter with coffee machine and breakfast, non-smoking rooms.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2487],CLICK[0.4567,0.2048],TYPE[Central, Hong Kong],CLICK[0.3683,0.1913],CLICK[0.5016,0.6571],CLICK[0.3333,0.7176],CLICK[0.4760,0.9340],CLICK[0.4051,0.3340],CLICK[0.6869,0.8298],CLICK[0.4747,0.9330],CLICK[0.4949,0.4000],CLICK[0.5006,0.1455],SWIPE[RIGHT],SWIPE[UP],CLICK[0.3077,0.3304],SWIPE[UP],CLICK[0.6968,0.2744],CLICK[0.3244,0.6369],CLICK[0.6946,0.6849],SWIPE[UP],CLICK[0.6990,0.6526],CLICK[0.4827,0.9330]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_18_815fdc5eaee64b8f91264b94a0bcd810-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6756,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.4737,0.2093]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093]" + }, + { + "from": "gpt", + "value": "TYPE[Shanghai]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.1420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420]" + }, + { + "from": "gpt", + "value": "CLICK[0.4612,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.5006,0.6615]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615]" + }, + { + "from": "gpt", + "value": "CLICK[0.4994,0.7051]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.3394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.6109]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109]" + }, + { + "from": "gpt", + "value": "CLICK[0.5756,0.6849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849]" + }, + { + "from": "gpt", + "value": "CLICK[0.6833,0.7603]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.3250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.5952]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6429,0.6324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324]" + }, + { + "from": "gpt", + "value": "CLICK[0.4827,0.7119]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6474,0.6311]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311]" + }, + { + "from": "gpt", + "value": "CLICK[0.4962,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058]" + }, + { + "from": "gpt", + "value": "CLICK[0.5119,0.1522]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3288,0.5875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875]" + }, + { + "from": "gpt", + "value": "CLICK[0.7103,0.4449]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.6391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.6356]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.6750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.6750]" + }, + { + "from": "gpt", + "value": "CLICK[0.4647,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-33.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.6750],CLICK[0.4647,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.3346,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-34.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.6750],CLICK[0.4647,0.9330],CLICK[0.3346,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.6189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-35.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Shanghai, from 22 May to 29 May. 2 guests and 2 7-year-old children in 2 rooms. Queen bed with 2 single beds room. Filter with parking, room service, and spa/wellness center. Sort by distance to city center.\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2631],CLICK[0.4737,0.2093],TYPE[Shanghai],CLICK[0.3997,0.1420],CLICK[0.4612,0.2801],CLICK[0.5006,0.6615],CLICK[0.4994,0.7051],CLICK[0.4231,0.9138],CLICK[0.4343,0.3394],CLICK[0.6901,0.6109],CLICK[0.5756,0.6849],CLICK[0.6833,0.7603],CLICK[0.6811,0.3250],CLICK[0.4231,0.5952],SWIPE[UP],CLICK[0.6429,0.6324],CLICK[0.4827,0.7119],SWIPE[UP],CLICK[0.6474,0.6311],CLICK[0.4962,0.9263],CLICK[0.4388,0.4058],CLICK[0.5119,0.1522],SWIPE[UP],CLICK[0.3288,0.5875],CLICK[0.7103,0.4449],CLICK[0.7013,0.6391],SWIPE[UP],CLICK[0.7026,0.6356],SWIPE[UP],SWIPE[UP],CLICK[0.7026,0.6750],CLICK[0.4647,0.9330],CLICK[0.3346,0.1375],CLICK[0.6913,0.6189]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_22_662091b12c0f4040862fa6aa4ac45154-36.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6554,0.2397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4522,0.2061]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.1622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622]" + }, + { + "from": "gpt", + "value": "TYPE[Los Angeles]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles]" + }, + { + "from": "gpt", + "value": "CLICK[0.4513,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.4006,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407]" + }, + { + "from": "gpt", + "value": "TYPE[Tokyo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo]" + }, + { + "from": "gpt", + "value": "CLICK[0.3907,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.3894,0.3910]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910]" + }, + { + "from": "gpt", + "value": "CLICK[0.4926,0.3663]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910],CLICK[0.4926,0.3663]" + }, + { + "from": "gpt", + "value": "CLICK[0.5151,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910],CLICK[0.4926,0.3663],CLICK[0.5151,0.9205]" + }, + { + "from": "gpt", + "value": "CLICK[0.5173,0.5234]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910],CLICK[0.4926,0.3663],CLICK[0.5151,0.9205],CLICK[0.5173,0.5234]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.1231]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910],CLICK[0.4926,0.3663],CLICK[0.5151,0.9205],CLICK[0.5173,0.5234],CLICK[0.4266,0.1231]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.6099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910],CLICK[0.4926,0.3663],CLICK[0.5151,0.9205],CLICK[0.5173,0.5234],CLICK[0.4266,0.1231],CLICK[0.6901,0.6099]" + }, + { + "from": "gpt", + "value": "CLICK[0.6631,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a one-way flight from Los Angeles to Tokyo. Departure date 15 May. 1 adult in economy class. No stops.\nHistory Actions: SWIPE[LEFT],CLICK[0.6554,0.2397],CLICK[0.4625,0.1263],CLICK[0.4522,0.2061],CLICK[0.4221,0.2756],CLICK[0.4343,0.1622],TYPE[Los Angeles],CLICK[0.4513,0.2734],CLICK[0.4006,0.3407],TYPE[Tokyo],CLICK[0.3907,0.2734],CLICK[0.3894,0.3910],CLICK[0.4926,0.3663],CLICK[0.5151,0.9205],CLICK[0.5173,0.5234],CLICK[0.4266,0.1231],CLICK[0.6901,0.6099],CLICK[0.6631,0.9218]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_26_0f7ea2be354047abb2b6633e3fcad2b6-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6542,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.4131,0.2048]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048]" + }, + { + "from": "gpt", + "value": "TYPE[Dubai]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai]" + }, + { + "from": "gpt", + "value": "CLICK[0.4199,0.1397]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397]" + }, + { + "from": "gpt", + "value": "CLICK[0.4321,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6689,0.5234]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234]" + }, + { + "from": "gpt", + "value": "CLICK[0.6173,0.5840]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840]" + }, + { + "from": "gpt", + "value": "CLICK[0.5321,0.9295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.3304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304]" + }, + { + "from": "gpt", + "value": "CLICK[0.5769,0.1824]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824]" + }, + { + "from": "gpt", + "value": "CLICK[0.5667,0.3272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272]" + }, + { + "from": "gpt", + "value": "CLICK[0.5769,0.3192]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192]" + }, + { + "from": "gpt", + "value": "CLICK[0.4567,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.4558,0.4080]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3311,0.4538]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3378,0.5907]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.4920]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7080,0.5449]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-33.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-34.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6779,0.2138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-35.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449],SWIPE[UP],SWIPE[UP],CLICK[0.6779,0.2138]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.9205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-36.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449],SWIPE[UP],SWIPE[UP],CLICK[0.6779,0.2138],CLICK[0.4747,0.9205]" + }, + { + "from": "gpt", + "value": "CLICK[0.3356,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-37.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449],SWIPE[UP],SWIPE[UP],CLICK[0.6779,0.2138],CLICK[0.4747,0.9205],CLICK[0.3356,0.1410]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.7356]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-38.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Dubai, from 1 June to 7 June. 2 guests in 1 room. US$400 to US$1000. Filter with very good, free WiFi and kitchen. Free cancellation. Sort by highest price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2442],CLICK[0.4131,0.2048],TYPE[Dubai],CLICK[0.4199,0.1397],CLICK[0.4321,0.2676],SWIPE[UP],CLICK[0.6689,0.5234],CLICK[0.6173,0.5840],CLICK[0.5321,0.9295],CLICK[0.4064,0.3304],CLICK[0.5769,0.1824],CLICK[0.5667,0.3272],CLICK[0.5769,0.3192],CLICK[0.4567,0.9183],CLICK[0.4558,0.4080],CLICK[0.4984,0.1465],SWIPE[RIGHT],SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.3311,0.4538],CLICK[0.6981,0.4058],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3378,0.5907],CLICK[0.7035,0.4920],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7080,0.5449],SWIPE[UP],SWIPE[UP],CLICK[0.6779,0.2138],CLICK[0.4747,0.9205],CLICK[0.3356,0.1410],CLICK[0.6968,0.7356]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_27_83642fa4a55646fca8d96e395be836a6-39.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6676,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.2016]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016]" + }, + { + "from": "gpt", + "value": "TYPE[Venice]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410]" + }, + { + "from": "gpt", + "value": "CLICK[0.4446,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.5458]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458]" + }, + { + "from": "gpt", + "value": "CLICK[0.3176,0.6010]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010]" + }, + { + "from": "gpt", + "value": "CLICK[0.4545,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022]" + }, + { + "from": "gpt", + "value": "CLICK[0.4894,0.1420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3234,0.6692]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.5359]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.6817]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3423,0.6615]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.6615]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.6647]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.6615],CLICK[0.6990,0.6647]" + }, + { + "from": "gpt", + "value": "CLICK[0.7026,0.7849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.6615],CLICK[0.6990,0.6647],CLICK[0.7026,0.7849]" + }, + { + "from": "gpt", + "value": "CLICK[0.4881,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in Venice, from 10 July to 14 July. 2 guests in 1 room. Double bed room. Filter with coffee machine, sea view and airport shuttle.\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2587],CLICK[0.4141,0.2016],TYPE[Venice],CLICK[0.3929,0.1410],CLICK[0.4446,0.2631],CLICK[0.4949,0.5458],CLICK[0.3176,0.6010],CLICK[0.4545,0.9263],CLICK[0.4747,0.4022],CLICK[0.4894,0.1420],SWIPE[UP],CLICK[0.3234,0.6692],CLICK[0.7035,0.5359],SWIPE[UP],CLICK[0.7035,0.6817],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.6615],CLICK[0.6990,0.6647],CLICK[0.7026,0.7849],CLICK[0.4881,0.9138]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_36_83edb50b6ff043269cb5bb239b6c8aaa-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6564,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.1231]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.2093]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093]" + }, + { + "from": "gpt", + "value": "CLICK[0.3984,0.2689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney]" + }, + { + "from": "gpt", + "value": "CLICK[0.4276,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644]" + }, + { + "from": "gpt", + "value": "CLICK[0.3670,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349]" + }, + { + "from": "gpt", + "value": "TYPE[Wellington]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington]" + }, + { + "from": "gpt", + "value": "CLICK[0.4231,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.3894,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.8016]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.4253,0.4641]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.2151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.2801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801]" + }, + { + "from": "gpt", + "value": "CLICK[0.5949,0.9295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295]" + }, + { + "from": "gpt", + "value": "CLICK[0.4804,0.5212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295],CLICK[0.4804,0.5212]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.1285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295],CLICK[0.4804,0.5212],CLICK[0.4266,0.1285]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.7478]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295],CLICK[0.4804,0.5212],CLICK[0.4266,0.1285],CLICK[0.6878,0.7478]" + }, + { + "from": "gpt", + "value": "CLICK[0.6519,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295],CLICK[0.4804,0.5212],CLICK[0.4266,0.1285],CLICK[0.6878,0.7478],CLICK[0.6519,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.3580,0.1330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295],CLICK[0.4804,0.5212],CLICK[0.4266,0.1285],CLICK[0.6878,0.7478],CLICK[0.6519,0.9240],CLICK[0.3580,0.1330]" + }, + { + "from": "gpt", + "value": "CLICK[0.3131,0.8859]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a one-way flight from Sydney to Wellington. Departure date 12 August. 2 adults and 2 children in economy class. 1 stop is okay. Sort by shortest flight time.\nHistory Actions: SWIPE[LEFT],CLICK[0.6564,0.2587],CLICK[0.4500,0.1231],CLICK[0.4478,0.2093],CLICK[0.3984,0.2689],CLICK[0.4455,0.1689],TYPE[Sydney],CLICK[0.4276,0.2644],CLICK[0.3670,0.3349],TYPE[Wellington],CLICK[0.4231,0.2712],CLICK[0.3894,0.4000],SWIPE[UP],CLICK[0.3782,0.8016],CLICK[0.4288,0.9240],CLICK[0.4253,0.4641],CLICK[0.6824,0.2151],CLICK[0.6811,0.2756],CLICK[0.6901,0.2801],CLICK[0.5949,0.9295],CLICK[0.4804,0.5212],CLICK[0.4266,0.1285],CLICK[0.6878,0.7478],CLICK[0.6519,0.9240],CLICK[0.3580,0.1330],CLICK[0.3131,0.8859]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_39_00241511ed5049419dcc2844b50a84d8-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6587,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.2061]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.1410]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410]" + }, + { + "from": "gpt", + "value": "CLICK[0.4109,0.2631]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.5673]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673]" + }, + { + "from": "gpt", + "value": "CLICK[0.3840,0.6817]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.4276,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349]" + }, + { + "from": "gpt", + "value": "CLICK[0.6824,0.6907]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.9330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330]" + }, + { + "from": "gpt", + "value": "CLICK[0.4827,0.4035]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.1465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3458,0.5144]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.1500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3266,0.7288]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.2577]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.6324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.5750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324],SWIPE[UP],CLICK[0.6990,0.5750]" + }, + { + "from": "gpt", + "value": "CLICK[0.6923,0.4718]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324],SWIPE[UP],CLICK[0.6990,0.5750],CLICK[0.6923,0.4718]" + }, + { + "from": "gpt", + "value": "CLICK[0.4702,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324],SWIPE[UP],CLICK[0.6990,0.5750],CLICK[0.6923,0.4718],CLICK[0.4702,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.3369,0.1500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324],SWIPE[UP],CLICK[0.6990,0.5750],CLICK[0.6923,0.4718],CLICK[0.4702,0.9218],CLICK[0.3369,0.1500]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.8904]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find hotels in London, from 20 September to 30 September. 3 guests in 1 family room. Filters include pet friendly, breakfast included, bathtub and spa. Sort by guest rating.\nHistory Actions: SWIPE[LEFT],CLICK[0.6587,0.2542],CLICK[0.3939,0.2061],TYPE[London],CLICK[0.4029,0.1410],CLICK[0.4109,0.2631],SWIPE[UP],SWIPE[UP],CLICK[0.6151,0.5673],CLICK[0.3840,0.6817],CLICK[0.4872,0.9228],CLICK[0.4276,0.3349],CLICK[0.6824,0.6907],CLICK[0.5029,0.9330],CLICK[0.4827,0.4035],CLICK[0.4872,0.1465],SWIPE[UP],SWIPE[UP],CLICK[0.3458,0.5144],SWIPE[UP],CLICK[0.6981,0.1500],SWIPE[UP],CLICK[0.3266,0.7288],CLICK[0.6981,0.2577],CLICK[0.6936,0.6324],SWIPE[UP],CLICK[0.6990,0.5750],CLICK[0.6923,0.4718],CLICK[0.4702,0.9218],CLICK[0.3369,0.1500],CLICK[0.6936,0.8904]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_41_e725fedae3e74bd9aa8ce09195b2ec5c-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6474,0.2497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497]" + }, + { + "from": "gpt", + "value": "CLICK[0.4737,0.1330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.2667]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.1724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724]" + }, + { + "from": "gpt", + "value": "TYPE[Delhi]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644]" + }, + { + "from": "gpt", + "value": "CLICK[0.4131,0.3372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372]" + }, + { + "from": "gpt", + "value": "TYPE[Munich]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.2766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.3901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6721,0.4103]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103]" + }, + { + "from": "gpt", + "value": "CLICK[0.6061,0.5167]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167]" + }, + { + "from": "gpt", + "value": "CLICK[0.4917,0.9272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167],CLICK[0.4917,0.9272]" + }, + { + "from": "gpt", + "value": "CLICK[0.4244,0.4551]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167],CLICK[0.4917,0.9272],CLICK[0.4244,0.4551]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.4538]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167],CLICK[0.4917,0.9272],CLICK[0.4244,0.4551],CLICK[0.3064,0.4538]" + }, + { + "from": "gpt", + "value": "CLICK[0.6205,0.9272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167],CLICK[0.4917,0.9272],CLICK[0.4244,0.4551],CLICK[0.3064,0.4538],CLICK[0.6205,0.9272]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.5952]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167],CLICK[0.4917,0.9272],CLICK[0.4244,0.4551],CLICK[0.3064,0.4538],CLICK[0.6205,0.9272],CLICK[0.6801,0.5952]" + }, + { + "from": "gpt", + "value": "CLICK[0.4468,0.5179]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a return flight from Delhi to Munich. Departure date 12 October, return date 25 October. 1 Adult in premium economy class. Sort by direct flight.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2497],CLICK[0.4737,0.1330],CLICK[0.3728,0.2667],CLICK[0.4401,0.1724],TYPE[Delhi],CLICK[0.4051,0.2644],CLICK[0.4131,0.3372],TYPE[Munich],CLICK[0.4051,0.2766],CLICK[0.4074,0.3901],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6721,0.4103],CLICK[0.6061,0.5167],CLICK[0.4917,0.9272],CLICK[0.4244,0.4551],CLICK[0.3064,0.4538],CLICK[0.6205,0.9272],CLICK[0.6801,0.5952],CLICK[0.4468,0.5179]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_46_0ec3cd6403a3421fa158fc75a5557ed2-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6631,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.4176,0.2196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.1433]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433]" + }, + { + "from": "gpt", + "value": "CLICK[0.4410,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.6167]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167]" + }, + { + "from": "gpt", + "value": "CLICK[0.6173,0.7311]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311]" + }, + { + "from": "gpt", + "value": "CLICK[0.5388,0.9272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.3394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.6212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212]" + }, + { + "from": "gpt", + "value": "CLICK[0.5702,0.6952]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952]" + }, + { + "from": "gpt", + "value": "CLICK[0.6779,0.7535]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.3228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228]" + }, + { + "from": "gpt", + "value": "CLICK[0.5006,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.4119,0.4058]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.1545]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3423,0.7814]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.5817]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7035,0.7119]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6167]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6167]" + }, + { + "from": "gpt", + "value": "CLICK[0.4881,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6167],CLICK[0.4881,0.9196]" + }, + { + "from": "gpt", + "value": "CLICK[0.3346,0.1455]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6167],CLICK[0.4881,0.9196],CLICK[0.3346,0.1455]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.6221]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search hotels in New York, from 1 November to 15 November. 2 guests and 2 children in 2 rooms with 2 single beds. Filters include balcony. Accept online payments. Sort by distance to downtown.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2465],CLICK[0.4176,0.2196],TYPE[New York],CLICK[0.4051,0.1433],CLICK[0.4410,0.2644],SWIPE[UP],CLICK[0.6151,0.6167],CLICK[0.6173,0.7311],CLICK[0.5388,0.9272],CLICK[0.4388,0.3394],CLICK[0.6801,0.6212],CLICK[0.5702,0.6952],CLICK[0.6779,0.7535],CLICK[0.6801,0.3228],CLICK[0.5006,0.9183],CLICK[0.4119,0.4058],CLICK[0.5029,0.1545],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3423,0.7814],SWIPE[UP],CLICK[0.6901,0.5817],SWIPE[UP],CLICK[0.7035,0.7119],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6167],CLICK[0.4881,0.9196],CLICK[0.3346,0.1455],CLICK[0.6901,0.6221]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_48_c7882b1a075f4cdfbbfef916cec41604-33.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2587]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587]" + }, + { + "from": "gpt", + "value": "CLICK[0.5981,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3660,0.2788]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.3647,0.3304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304]" + }, + { + "from": "gpt", + "value": "CLICK[0.4311,0.3429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429]" + }, + { + "from": "gpt", + "value": "CLICK[0.6128,0.3821]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821]" + }, + { + "from": "gpt", + "value": "CLICK[0.5218,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.4013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.4349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349]" + }, + { + "from": "gpt", + "value": "CLICK[0.4199,0.9272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272]" + }, + { + "from": "gpt", + "value": "CLICK[0.6465,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3660,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734]" + }, + { + "from": "gpt", + "value": "CLICK[0.6308,0.3955]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3580,0.5740]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.4538]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538]" + }, + { + "from": "gpt", + "value": "TYPE[25-5330--to_--------45]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45]" + }, + { + "from": "gpt", + "value": "CLICK[0.4647,0.5234]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234]" + }, + { + "from": "gpt", + "value": "CLICK[0.6353,0.1343]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343]" + }, + { + "from": "gpt", + "value": "CLICK[0.4827,0.4314]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343],CLICK[0.4827,0.4314]" + }, + { + "from": "gpt", + "value": "CLICK[0.3266,0.8500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343],CLICK[0.4827,0.4314],CLICK[0.3266,0.8500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343],CLICK[0.4827,0.4314],CLICK[0.3266,0.8500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3256,0.4673]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343],CLICK[0.4827,0.4314],CLICK[0.3266,0.8500],SWIPE[UP],CLICK[0.3256,0.4673]" + }, + { + "from": "gpt", + "value": "CLICK[0.5410,0.4067]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343],CLICK[0.4827,0.4314],CLICK[0.3266,0.8500],SWIPE[UP],CLICK[0.3256,0.4673],CLICK[0.5410,0.4067]" + }, + { + "from": "gpt", + "value": "CLICK[0.3952,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in New York from May 10, 8 am to May 15, 7 pm. Driver\"s age 45. Filter by large car type, Hertz rent supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2587],CLICK[0.5981,0.1276],CLICK[0.3660,0.2788],TYPE[New York],CLICK[0.3647,0.3304],CLICK[0.4311,0.3429],CLICK[0.6128,0.3821],CLICK[0.5218,0.9353],CLICK[0.3962,0.4013],CLICK[0.4984,0.4349],CLICK[0.4199,0.9272],CLICK[0.6465,0.3362],SWIPE[DOWN],CLICK[0.3660,0.2734],CLICK[0.6308,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3580,0.5740],CLICK[0.3593,0.4538],TYPE[25-5330--to_--------45],CLICK[0.4647,0.5234],CLICK[0.6353,0.1343],CLICK[0.4827,0.4314],CLICK[0.3266,0.8500],SWIPE[UP],CLICK[0.3256,0.4673],CLICK[0.5410,0.4067],CLICK[0.3952,0.9218]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_53_81f02c0029fc4e3a9328e457bdec0347-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6474,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.1240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.2699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699]" + }, + { + "from": "gpt", + "value": "TYPE[Charles de Gaulle Airport]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport]" + }, + { + "from": "gpt", + "value": "CLICK[0.4019,0.3295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295]" + }, + { + "from": "gpt", + "value": "TYPE[Louvre Museum]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum]" + }, + { + "from": "gpt", + "value": "CLICK[0.3660,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349]" + }, + { + "from": "gpt", + "value": "CLICK[0.4321,0.3901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901]" + }, + { + "from": "gpt", + "value": "CLICK[0.4657,0.9061]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901],CLICK[0.4657,0.9061]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901],CLICK[0.4657,0.9061],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901],CLICK[0.4657,0.9061],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901],CLICK[0.4657,0.9061],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4859,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901],CLICK[0.4657,0.9061],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4859,0.9160]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.4673]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Paris for today 6 pm. Pick-up location Charles de Gaulle Airport, and destination is The Louvre Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6474,0.2465],CLICK[0.6990,0.1240],CLICK[0.3929,0.2699],TYPE[Charles de Gaulle Airport],CLICK[0.4019,0.3295],TYPE[Louvre Museum],CLICK[0.3660,0.3349],CLICK[0.4321,0.3901],CLICK[0.4657,0.9061],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4859,0.9160],CLICK[0.3997,0.4673]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_8_59_527d7ab6328649fe83b92ac39a95c2a4-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.2385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385]" + }, + { + "from": "gpt", + "value": "CLICK[0.5801,0.1196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.0657]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.3760,0.4337]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337]" + }, + { + "from": "gpt", + "value": "CLICK[0.3647,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4971,0.6580]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580]" + }, + { + "from": "gpt", + "value": "CLICK[0.4894,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353]" + }, + { + "from": "gpt", + "value": "CLICK[0.4298,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000]" + }, + { + "from": "gpt", + "value": "CLICK[0.6699,0.4404]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404]" + }, + { + "from": "gpt", + "value": "CLICK[0.4670,0.9474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474]" + }, + { + "from": "gpt", + "value": "CLICK[0.6654,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.3497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497]" + }, + { + "from": "gpt", + "value": "CLICK[0.6397,0.4013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3750,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.4651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651]" + }, + { + "from": "gpt", + "value": "CLICK[0.4131,0.5346]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346]" + }, + { + "from": "gpt", + "value": "CLICK[0.6160,0.1308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.4875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875]" + }, + { + "from": "gpt", + "value": "CLICK[0.4064,0.4830]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875],CLICK[0.4064,0.4830]" + }, + { + "from": "gpt", + "value": "CLICK[0.3558,0.7176]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875],CLICK[0.4064,0.4830],CLICK[0.3558,0.7176]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875],CLICK[0.4064,0.4830],CLICK[0.3558,0.7176],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4311,0.6356]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875],CLICK[0.4064,0.4830],CLICK[0.3558,0.7176],SWIPE[UP],CLICK[0.4311,0.6356]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.8702]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875],CLICK[0.4064,0.4830],CLICK[0.3558,0.7176],SWIPE[UP],CLICK[0.4311,0.6356],CLICK[0.4378,0.8702]" + }, + { + "from": "gpt", + "value": "CLICK[0.5208,0.9160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in Rome from July 10 9 am to July 20 11 am. Driver\"s age 45. Filter by SUV type, Sixt supplier, manual transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.5801,0.1196],CLICK[0.3974,0.2712],CLICK[0.6936,0.0657],TYPE[Rome],CLICK[0.3760,0.4337],CLICK[0.3647,0.3349],SWIPE[UP],CLICK[0.4971,0.6580],CLICK[0.4894,0.9353],CLICK[0.4298,0.4000],CLICK[0.6699,0.4404],CLICK[0.4670,0.9474],CLICK[0.6654,0.3407],CLICK[0.3593,0.3497],CLICK[0.6397,0.4013],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3750,0.2542],CLICK[0.3728,0.4651],CLICK[0.4131,0.5346],CLICK[0.6160,0.1308],CLICK[0.3064,0.4875],CLICK[0.4064,0.4830],CLICK[0.3558,0.7176],SWIPE[UP],CLICK[0.4311,0.6356],CLICK[0.4378,0.8702],CLICK[0.5208,0.9160]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_11_e28a9c7d2acf4de8931b088bf13b5223-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.2183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4163,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.4776]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776]" + }, + { + "from": "gpt", + "value": "CLICK[0.4904,0.6849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.4288,0.4022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7048,0.7657]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3256,0.6221]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221]" + }, + { + "from": "gpt", + "value": "CLICK[0.6958,0.5256]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5740]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256],CLICK[0.6946,0.5740]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256],CLICK[0.6946,0.5740],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256],CLICK[0.6946,0.5740],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256],CLICK[0.6946,0.5740],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.6176]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256],CLICK[0.6946,0.5740],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6176]" + }, + { + "from": "gpt", + "value": "CLICK[0.5196,0.9295]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find an apartment to rent in New York for July. Filter by \"2+ bedrooms\" + \"parking included\" + \"allows pets\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.3962,0.2183],TYPE[New York],CLICK[0.4163,0.1276],CLICK[0.3917,0.2734],SWIPE[DOWN],CLICK[0.3817,0.4776],CLICK[0.4904,0.6849],CLICK[0.4724,0.9240],CLICK[0.4288,0.4022],SWIPE[UP],CLICK[0.7048,0.7657],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3256,0.6221],CLICK[0.6958,0.5256],CLICK[0.6946,0.5740],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6968,0.6176],CLICK[0.5196,0.9295]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_18_69e2dab7abbf449db7901a6574dd640f-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6756,0.2554]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.2128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128]" + }, + { + "from": "gpt", + "value": "TYPE[Tokyo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.1263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3782,0.5202]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.5628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628]" + }, + { + "from": "gpt", + "value": "CLICK[0.4782,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.3683,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362]" + }, + { + "from": "gpt", + "value": "CLICK[0.4647,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.4513,0.4080]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080]" + }, + { + "from": "gpt", + "value": "CLICK[0.4939,0.1478]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6958,0.3304]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.5135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6990,0.5135]" + }, + { + "from": "gpt", + "value": "CLICK[0.5388,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find 2 hotel rooms in Tokyo. From June 10 to June 18. Occupants are 2 adults, 2 children. Filter by \"free wifi\" + \"5 stars\"\nHistory Actions: SWIPE[LEFT],CLICK[0.6756,0.2554],CLICK[0.3962,0.2128],TYPE[Tokyo],CLICK[0.3939,0.1263],CLICK[0.4051,0.2676],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.3782,0.5202],CLICK[0.4378,0.5628],CLICK[0.4782,0.9228],CLICK[0.3683,0.3362],CLICK[0.4647,0.9183],CLICK[0.4513,0.4080],CLICK[0.4939,0.1478],SWIPE[UP],SWIPE[UP],CLICK[0.6958,0.3304],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6990,0.5135],CLICK[0.5388,0.9263]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_1_1dff9496cc044864820b5bf340b85d13-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6542,0.2667]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667]" + }, + { + "from": "gpt", + "value": "CLICK[0.5734,0.1141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.0625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625]" + }, + { + "from": "gpt", + "value": "TYPE[San Francisco]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco]" + }, + { + "from": "gpt", + "value": "CLICK[0.3526,0.4090]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090]" + }, + { + "from": "gpt", + "value": "CLICK[0.3817,0.3417]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5635,0.3564]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564]" + }, + { + "from": "gpt", + "value": "CLICK[0.3369,0.4673]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673]" + }, + { + "from": "gpt", + "value": "CLICK[0.4670,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.6340,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.3705,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689]" + }, + { + "from": "gpt", + "value": "CLICK[0.6330,0.3955]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3715,0.7074]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.5144]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144]" + }, + { + "from": "gpt", + "value": "CLICK[0.6330,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144],CLICK[0.6330,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.3099,0.4359]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144],CLICK[0.6330,0.1375],CLICK[0.3099,0.4359]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.3910]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144],CLICK[0.6330,0.1375],CLICK[0.3099,0.4359],CLICK[0.4042,0.3910]" + }, + { + "from": "gpt", + "value": "CLICK[0.6330,0.5593]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144],CLICK[0.6330,0.1375],CLICK[0.3099,0.4359],CLICK[0.4042,0.3910],CLICK[0.6330,0.5593]" + }, + { + "from": "gpt", + "value": "CLICK[0.3378,0.7388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144],CLICK[0.6330,0.1375],CLICK[0.3099,0.4359],CLICK[0.4042,0.3910],CLICK[0.6330,0.5593],CLICK[0.3378,0.7388]" + }, + { + "from": "gpt", + "value": "CLICK[0.4311,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Rent a car in San Francisco from Sep 5 7 am to Sep 15 9 pm. Driver\"s age from 30 to 65. Filter by SUVs, Sixt supplier, automatic transmission.\nHistory Actions: SWIPE[LEFT],CLICK[0.6542,0.2667],CLICK[0.5734,0.1141],CLICK[0.3885,0.2756],CLICK[0.6901,0.0625],TYPE[San Francisco],CLICK[0.3526,0.4090],CLICK[0.3817,0.3417],SWIPE[UP],CLICK[0.5635,0.3564],CLICK[0.3369,0.4673],CLICK[0.4670,0.9218],CLICK[0.6340,0.3407],SWIPE[DOWN],CLICK[0.3705,0.1689],CLICK[0.6330,0.3955],SWIPE[UP],SWIPE[UP],CLICK[0.3715,0.7074],CLICK[0.4401,0.5144],CLICK[0.6330,0.1375],CLICK[0.3099,0.4359],CLICK[0.4042,0.3910],CLICK[0.6330,0.5593],CLICK[0.3378,0.7388],CLICK[0.4311,0.9263]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_21_a1eb6c5cff804540a50c65e3cd0fb161-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6689,0.2609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.2038]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038]" + }, + { + "from": "gpt", + "value": "TYPE[Bali]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4455,0.5494]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494]" + }, + { + "from": "gpt", + "value": "CLICK[0.4994,0.6593]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593]" + }, + { + "from": "gpt", + "value": "CLICK[0.5061,0.9240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240]" + }, + { + "from": "gpt", + "value": "CLICK[0.5353,0.4013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013]" + }, + { + "from": "gpt", + "value": "CLICK[0.4917,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6958,0.4471]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5548]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.2997,0.5247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.5808]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.5583]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP],CLICK[0.6968,0.5583]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP],CLICK[0.6968,0.5583],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP],CLICK[0.6968,0.5583],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP],CLICK[0.6968,0.5583],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.8016]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP],CLICK[0.6968,0.5583],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7003,0.8016]" + }, + { + "from": "gpt", + "value": "CLICK[0.4769,0.9173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Find a villa to rent in Bali for New Year\"s Eve. Filter by \"private pool\" + \"breakfast included\" + \"room service\" + \"Wonderful\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6689,0.2609],CLICK[0.4029,0.2038],TYPE[Bali],CLICK[0.3929,0.1375],CLICK[0.3939,0.2712],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.4455,0.5494],CLICK[0.4994,0.6593],CLICK[0.5061,0.9240],CLICK[0.5353,0.4013],CLICK[0.4917,0.1375],SWIPE[UP],CLICK[0.6958,0.4471],SWIPE[UP],CLICK[0.6946,0.5548],SWIPE[UP],SWIPE[UP],CLICK[0.2997,0.5247],SWIPE[UP],CLICK[0.6946,0.5808],SWIPE[UP],CLICK[0.6968,0.5583],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.7003,0.8016],CLICK[0.4769,0.9173]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_29_577969850f53457598bbf79bcd0f3d0e-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6779,0.2542]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542]" + }, + { + "from": "gpt", + "value": "CLICK[0.6958,0.1186]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.2734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734]" + }, + { + "from": "gpt", + "value": "TYPE[Madrid Airport]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.3250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250]" + }, + { + "from": "gpt", + "value": "TYPE[Prado Museum]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum]" + }, + { + "from": "gpt", + "value": "CLICK[0.3939,0.3394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.3955]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394],CLICK[0.3728,0.3955]" + }, + { + "from": "gpt", + "value": "CLICK[0.4545,0.9048]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394],CLICK[0.3728,0.3955],CLICK[0.4545,0.9048]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394],CLICK[0.3728,0.3955],CLICK[0.4545,0.9048],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394],CLICK[0.3728,0.3955],CLICK[0.4545,0.9048],SWIPE[RIGHT],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.5388,0.8994]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394],CLICK[0.3728,0.3955],CLICK[0.4545,0.9048],SWIPE[RIGHT],SWIPE[DOWN],CLICK[0.5388,0.8994]" + }, + { + "from": "gpt", + "value": "CLICK[0.5545,0.4696]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Madrid for today 5:40 pm. Pick-up location is Madrid Airport and destination is The Prado Museum.\nHistory Actions: SWIPE[LEFT],CLICK[0.6779,0.2542],CLICK[0.6958,0.1186],CLICK[0.3728,0.2734],TYPE[Madrid Airport],CLICK[0.4221,0.3250],TYPE[Prado Museum],CLICK[0.3939,0.3394],CLICK[0.3728,0.3955],CLICK[0.4545,0.9048],SWIPE[RIGHT],SWIPE[DOWN],CLICK[0.5388,0.8994],CLICK[0.5545,0.4696]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_34_f7473b3218b04073812e3137b9a443fa-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6609,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676]" + }, + { + "from": "gpt", + "value": "CLICK[0.4042,0.2128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4087,0.1330]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330]" + }, + { + "from": "gpt", + "value": "CLICK[0.3795,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.4446,0.6019]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.6535]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535]" + }, + { + "from": "gpt", + "value": "CLICK[0.5163,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.4266,0.3372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372]" + }, + { + "from": "gpt", + "value": "CLICK[0.6788,0.1846]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846]" + }, + { + "from": "gpt", + "value": "CLICK[0.6833,0.3160]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160]" + }, + { + "from": "gpt", + "value": "CLICK[0.3962,0.7208]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5500,0.9263]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263]" + }, + { + "from": "gpt", + "value": "CLICK[0.4221,0.3968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968]" + }, + { + "from": "gpt", + "value": "CLICK[0.3535,0.1455]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455]" + }, + { + "from": "gpt", + "value": "CLICK[0.4253,0.8958]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958]" + }, + { + "from": "gpt", + "value": "CLICK[0.4939,0.1500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6990,0.7849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500],SWIPE[UP],CLICK[0.6990,0.7849]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500],SWIPE[UP],CLICK[0.6990,0.7849],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6981,0.5279]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500],SWIPE[UP],CLICK[0.6990,0.7849],SWIPE[UP],CLICK[0.6981,0.5279]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.5785]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500],SWIPE[UP],CLICK[0.6990,0.7849],SWIPE[UP],CLICK[0.6981,0.5279],CLICK[0.7013,0.5785]" + }, + { + "from": "gpt", + "value": "CLICK[0.4837,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a hotel in New York from June 11 to June 17 for 2 Adults and 2 Child in 2 rooms. Sort by guests review score. Filter options should include free Wi-Fi, breakfast included, and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6609,0.2676],CLICK[0.4042,0.2128],TYPE[New York],CLICK[0.4087,0.1330],CLICK[0.3795,0.2712],CLICK[0.4446,0.6019],CLICK[0.3849,0.6535],CLICK[0.5163,0.9183],CLICK[0.4266,0.3372],CLICK[0.6788,0.1846],CLICK[0.6833,0.3160],CLICK[0.3962,0.7208],SWIPE[UP],CLICK[0.5500,0.9263],CLICK[0.4221,0.3968],CLICK[0.3535,0.1455],CLICK[0.4253,0.8958],CLICK[0.4939,0.1500],SWIPE[UP],CLICK[0.6990,0.7849],SWIPE[UP],CLICK[0.6981,0.5279],CLICK[0.7013,0.5785],CLICK[0.4837,0.9340]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_40_4391363161564fe0a20b1b892b18b4a6-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6631,0.2420]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.1208]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.1667]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.4208,0.2609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609]" + }, + { + "from": "gpt", + "value": "CLICK[0.3840,0.3362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362]" + }, + { + "from": "gpt", + "value": "TYPE[Miami]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami]" + }, + { + "from": "gpt", + "value": "CLICK[0.3795,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.3968]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5599,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122]" + }, + { + "from": "gpt", + "value": "CLICK[0.5567,0.5728]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728]" + }, + { + "from": "gpt", + "value": "CLICK[0.4433,0.9353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.4494]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.2173]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.2811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811]" + }, + { + "from": "gpt", + "value": "CLICK[0.3077,0.5830]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811],CLICK[0.3077,0.5830]" + }, + { + "from": "gpt", + "value": "CLICK[0.5792,0.9308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811],CLICK[0.3077,0.5830],CLICK[0.5792,0.9308]" + }, + { + "from": "gpt", + "value": "CLICK[0.7013,0.5952]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811],CLICK[0.3077,0.5830],CLICK[0.5792,0.9308],CLICK[0.7013,0.5952]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.5292]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811],CLICK[0.3077,0.5830],CLICK[0.5792,0.9308],CLICK[0.7013,0.5952],CLICK[0.4365,0.5292]" + }, + { + "from": "gpt", + "value": "CLICK[0.3603,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811],CLICK[0.3077,0.5830],CLICK[0.5792,0.9308],CLICK[0.7013,0.5952],CLICK[0.4365,0.5292],CLICK[0.3603,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3647,0.9375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a round-trip flight from New York to Miami. Departure date July 4, return date July 11. 2 Adults and 1 child. Premium Economy cabin class. Sort by price. Non-stop flights only.\nHistory Actions: SWIPE[LEFT],CLICK[0.6631,0.2420],CLICK[0.4635,0.1208],CLICK[0.3974,0.2676],CLICK[0.4478,0.1667],TYPE[New York],CLICK[0.4208,0.2609],CLICK[0.3840,0.3362],TYPE[Miami],CLICK[0.3795,0.2712],CLICK[0.4635,0.3968],SWIPE[UP],CLICK[0.5599,0.5122],CLICK[0.5567,0.5728],CLICK[0.4433,0.9353],CLICK[0.4423,0.4494],CLICK[0.6811,0.2173],CLICK[0.6801,0.2811],CLICK[0.3077,0.5830],CLICK[0.5792,0.9308],CLICK[0.7013,0.5952],CLICK[0.4365,0.5292],CLICK[0.3603,0.1276],CLICK[0.3647,0.9375]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_44_d6a8fa063656457c81d460db8ec22e79-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.4558,0.1285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285]" + }, + { + "from": "gpt", + "value": "CLICK[0.5667,0.2115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115]" + }, + { + "from": "gpt", + "value": "CLICK[0.4343,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676]" + }, + { + "from": "gpt", + "value": "TYPE[New York]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York]" + }, + { + "from": "gpt", + "value": "CLICK[0.3952,0.2465]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465]" + }, + { + "from": "gpt", + "value": "CLICK[0.3984,0.3407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407]" + }, + { + "from": "gpt", + "value": "TYPE[Paris]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris]" + }, + { + "from": "gpt", + "value": "CLICK[0.4535,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.3728,0.3933]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933]" + }, + { + "from": "gpt", + "value": "CLICK[0.5061,0.3744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744]" + }, + { + "from": "gpt", + "value": "CLICK[0.4186,0.9138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138]" + }, + { + "from": "gpt", + "value": "CLICK[0.3760,0.5112]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.2510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.5683]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683]" + }, + { + "from": "gpt", + "value": "CLICK[0.5612,0.4798]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.4446,0.6952]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228],CLICK[0.4446,0.6952]" + }, + { + "from": "gpt", + "value": "CLICK[0.3064,0.5628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228],CLICK[0.4446,0.6952],CLICK[0.3064,0.5628]" + }, + { + "from": "gpt", + "value": "CLICK[0.5679,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228],CLICK[0.4446,0.6952],CLICK[0.3064,0.5628],CLICK[0.5679,0.9196]" + }, + { + "from": "gpt", + "value": "CLICK[0.4792,0.7625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228],CLICK[0.4446,0.6952],CLICK[0.3064,0.5628],CLICK[0.5679,0.9196],CLICK[0.4792,0.7625]" + }, + { + "from": "gpt", + "value": "CLICK[0.3378,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228],CLICK[0.4446,0.6952],CLICK[0.3064,0.5628],CLICK[0.5679,0.9196],CLICK[0.4792,0.7625],CLICK[0.3378,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.3244,0.9385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search multi-city flights from New York to Paris on 15 May, from Paris to London on 30 May and from London to New York on 20 May. 1 Adult. First cabin class. Sort by price. No specific airlines.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2442],CLICK[0.4558,0.1285],CLICK[0.5667,0.2115],CLICK[0.4343,0.2676],TYPE[New York],CLICK[0.3952,0.2465],CLICK[0.3984,0.3407],TYPE[Paris],CLICK[0.4535,0.2442],CLICK[0.3728,0.3933],CLICK[0.5061,0.3744],CLICK[0.4186,0.9138],CLICK[0.3760,0.5112],TYPE[London],CLICK[0.3849,0.2510],CLICK[0.4141,0.5683],CLICK[0.5612,0.4798],CLICK[0.4949,0.9228],CLICK[0.4446,0.6952],CLICK[0.3064,0.5628],CLICK[0.5679,0.9196],CLICK[0.4792,0.7625],CLICK[0.3378,0.1253],CLICK[0.3244,0.9385]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_46_65656e7aae9640c582986348cdbc01d0-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6676,0.2474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474]" + }, + { + "from": "gpt", + "value": "CLICK[0.4692,0.1298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298]" + }, + { + "from": "gpt", + "value": "CLICK[0.4253,0.2779]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.1657]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2712]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.3205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.3885,0.2699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699]" + }, + { + "from": "gpt", + "value": "CLICK[0.4545,0.2106]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699],CLICK[0.4545,0.2106]" + }, + { + "from": "gpt", + "value": "CLICK[0.3929,0.3901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699],CLICK[0.4545,0.2106],CLICK[0.3929,0.3901]" + }, + { + "from": "gpt", + "value": "CLICK[0.6699,0.5862]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699],CLICK[0.4545,0.2106],CLICK[0.3929,0.3901],CLICK[0.6699,0.5862]" + }, + { + "from": "gpt", + "value": "CLICK[0.4333,0.9218]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699],CLICK[0.4545,0.2106],CLICK[0.3929,0.3901],CLICK[0.6699,0.5862],CLICK[0.4333,0.9218]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.5885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699],CLICK[0.4545,0.2106],CLICK[0.3929,0.3901],CLICK[0.6699,0.5862],CLICK[0.4333,0.9218],CLICK[0.6968,0.5885]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.5369]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a flight to Rome from London. Departure time is June 1. Passenger is one adult. Filter by \"non-stop\".\nHistory Actions: SWIPE[LEFT],CLICK[0.6676,0.2474],CLICK[0.4692,0.1298],CLICK[0.4253,0.2779],CLICK[0.4365,0.1657],TYPE[Rome],CLICK[0.3885,0.2712],CLICK[0.3849,0.3205],TYPE[London],CLICK[0.3885,0.2699],CLICK[0.4545,0.2106],CLICK[0.3929,0.3901],CLICK[0.6699,0.5862],CLICK[0.4333,0.9218],CLICK[0.6968,0.5885],CLICK[0.4984,0.5369]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_4_716333ffae0049d382af3a8d0ad00515-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.2385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385]" + }, + { + "from": "gpt", + "value": "CLICK[0.6026,0.1276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276]" + }, + { + "from": "gpt", + "value": "CLICK[0.3772,0.2676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676]" + }, + { + "from": "gpt", + "value": "CLICK[0.6946,0.0590]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590]" + }, + { + "from": "gpt", + "value": "TYPE[CHicago]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago]" + }, + { + "from": "gpt", + "value": "CLICK[0.3369,0.4785]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785]" + }, + { + "from": "gpt", + "value": "CLICK[0.3737,0.3385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385]" + }, + { + "from": "gpt", + "value": "CLICK[0.6228,0.3260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260]" + }, + { + "from": "gpt", + "value": "CLICK[0.3804,0.4394]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394]" + }, + { + "from": "gpt", + "value": "CLICK[0.4702,0.9285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285]" + }, + { + "from": "gpt", + "value": "CLICK[0.4490,0.5256]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285],CLICK[0.4490,0.5256]" + }, + { + "from": "gpt", + "value": "CLICK[0.6263,0.1388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285],CLICK[0.4490,0.5256],CLICK[0.6263,0.1388]" + }, + { + "from": "gpt", + "value": "CLICK[0.5814,0.3798]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285],CLICK[0.4490,0.5256],CLICK[0.6263,0.1388],CLICK[0.5814,0.3798]" + }, + { + "from": "gpt", + "value": "CLICK[0.4490,0.9228]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285],CLICK[0.4490,0.5256],CLICK[0.6263,0.1388],CLICK[0.5814,0.3798],CLICK[0.4490,0.9228]" + }, + { + "from": "gpt", + "value": "CLICK[0.3840,0.1353]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285],CLICK[0.4490,0.5256],CLICK[0.6263,0.1388],CLICK[0.5814,0.3798],CLICK[0.4490,0.9228],CLICK[0.3840,0.1353]" + }, + { + "from": "gpt", + "value": "CLICK[0.5263,0.8285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a rental car in Chicago. Pick up on July 5, drop off on July 10. Premium car size. Sort by price.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2385],CLICK[0.6026,0.1276],CLICK[0.3772,0.2676],CLICK[0.6946,0.0590],TYPE[CHicago],CLICK[0.3369,0.4785],CLICK[0.3737,0.3385],CLICK[0.6228,0.3260],CLICK[0.3804,0.4394],CLICK[0.4702,0.9285],CLICK[0.4490,0.5256],CLICK[0.6263,0.1388],CLICK[0.5814,0.3798],CLICK[0.4490,0.9228],CLICK[0.3840,0.1353],CLICK[0.5263,0.8285]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_53_b8e0d14a34234e40b0fa555d3493dfe9-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6622,0.2340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.2138]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138]" + }, + { + "from": "gpt", + "value": "TYPE[Rome]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome]" + }, + { + "from": "gpt", + "value": "CLICK[0.3862,0.1388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388]" + }, + { + "from": "gpt", + "value": "CLICK[0.3917,0.2644]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5545,0.5750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750]" + }, + { + "from": "gpt", + "value": "CLICK[0.5061,0.6244]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.9375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4333,0.3317]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317]" + }, + { + "from": "gpt", + "value": "CLICK[0.5814,0.1814]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814]" + }, + { + "from": "gpt", + "value": "CLICK[0.5824,0.3215]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215]" + }, + { + "from": "gpt", + "value": "CLICK[0.5747,0.3272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.6782]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782]" + }, + { + "from": "gpt", + "value": "CLICK[0.4926,0.9183]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183]" + }, + { + "from": "gpt", + "value": "CLICK[0.4715,0.4013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013]" + }, + { + "from": "gpt", + "value": "CLICK[0.4984,0.1433]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3167,0.5660]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6913,0.2429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6968,0.7760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-26.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-27.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-28.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-29.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-30.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-31.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.6099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-32.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6801,0.6099]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.9340]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-33.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search for hotels in Rome from August 1 to August 7 for 3 Adults. Sort by distance from city centre. Filter options should include free cancellation, air conditioning and parking.\nHistory Actions: SWIPE[LEFT],CLICK[0.6622,0.2340],CLICK[0.4378,0.2138],TYPE[Rome],CLICK[0.3862,0.1388],CLICK[0.3917,0.2644],SWIPE[UP],CLICK[0.5545,0.5750],CLICK[0.5061,0.6244],CLICK[0.4872,0.9375],CLICK[0.4333,0.3317],CLICK[0.5814,0.1814],CLICK[0.5824,0.3215],CLICK[0.5747,0.3272],CLICK[0.6811,0.6782],CLICK[0.4926,0.9183],CLICK[0.4715,0.4013],CLICK[0.4984,0.1433],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3167,0.5660],SWIPE[UP],CLICK[0.6913,0.2429],SWIPE[UP],CLICK[0.6968,0.7760],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6801,0.6099],CLICK[0.4724,0.9340]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_55_7eb750cd1eb84a7d8c6cc747ff47368c-34.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6644,0.2375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4558,0.1308]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308]" + }, + { + "from": "gpt", + "value": "CLICK[0.4814,0.2744]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744]" + }, + { + "from": "gpt", + "value": "CLICK[0.4478,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689]" + }, + { + "from": "gpt", + "value": "TYPE[London]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London]" + }, + { + "from": "gpt", + "value": "CLICK[0.4423,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721]" + }, + { + "from": "gpt", + "value": "CLICK[0.4096,0.3260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260]" + }, + { + "from": "gpt", + "value": "TYPE[Paris]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris]" + }, + { + "from": "gpt", + "value": "CLICK[0.4356,0.2654]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654]" + }, + { + "from": "gpt", + "value": "CLICK[0.4186,0.3946]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946]" + }, + { + "from": "gpt", + "value": "CLICK[0.5029,0.3731]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731]" + }, + { + "from": "gpt", + "value": "CLICK[0.6766,0.4215]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215]" + }, + { + "from": "gpt", + "value": "CLICK[0.4580,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196]" + }, + { + "from": "gpt", + "value": "CLICK[0.3952,0.4583]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.2196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.2756]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756]" + }, + { + "from": "gpt", + "value": "CLICK[0.5590,0.9285]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285]" + }, + { + "from": "gpt", + "value": "CLICK[0.5051,0.5212]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-19.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285],CLICK[0.5051,0.5212]" + }, + { + "from": "gpt", + "value": "CLICK[0.3593,0.1231]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-20.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285],CLICK[0.5051,0.5212],CLICK[0.3593,0.1231]" + }, + { + "from": "gpt", + "value": "CLICK[0.3234,0.9407]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-21.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285],CLICK[0.5051,0.5212],CLICK[0.3593,0.1231],CLICK[0.3234,0.9407]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.1240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-22.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285],CLICK[0.5051,0.5212],CLICK[0.3593,0.1231],CLICK[0.3234,0.9407],CLICK[0.4625,0.1240]" + }, + { + "from": "gpt", + "value": "CLICK[0.5231,0.8298]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-23.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285],CLICK[0.5051,0.5212],CLICK[0.3593,0.1231],CLICK[0.3234,0.9407],CLICK[0.4625,0.1240],CLICK[0.5231,0.8298]" + }, + { + "from": "gpt", + "value": "CLICK[0.6554,0.9196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-24.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Search a return flight from London to Paris. Departure date 15 May, return date 25 May. 2 adults and 1 child. Economy class. Sort by price. 1 stop is ok.\nHistory Actions: SWIPE[LEFT],CLICK[0.6644,0.2375],CLICK[0.4558,0.1308],CLICK[0.4814,0.2744],CLICK[0.4478,0.1689],TYPE[London],CLICK[0.4423,0.2721],CLICK[0.4096,0.3260],TYPE[Paris],CLICK[0.4356,0.2654],CLICK[0.4186,0.3946],CLICK[0.5029,0.3731],CLICK[0.6766,0.4215],CLICK[0.4580,0.9196],CLICK[0.3952,0.4583],CLICK[0.6901,0.2196],CLICK[0.6811,0.2756],CLICK[0.5590,0.9285],CLICK[0.5051,0.5212],CLICK[0.3593,0.1231],CLICK[0.3234,0.9407],CLICK[0.4625,0.1240],CLICK[0.5231,0.8298],CLICK[0.6554,0.9196]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_58_f0ad8c7bb2944f62802d9265007db615-25.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6577,0.2442]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442]" + }, + { + "from": "gpt", + "value": "CLICK[0.6936,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.4029,0.2721]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney Kingsford Smith Airport]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport]" + }, + { + "from": "gpt", + "value": "CLICK[0.4208,0.3272]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272]" + }, + { + "from": "gpt", + "value": "TYPE[Sydney Opera House]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House]" + }, + { + "from": "gpt", + "value": "CLICK[0.3862,0.3205]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205]" + }, + { + "from": "gpt", + "value": "CLICK[0.4087,0.4000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000]" + }, + { + "from": "gpt", + "value": "CLICK[0.6856,0.8141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4500,0.9151]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4500,0.9151]" + }, + { + "from": "gpt", + "value": "CLICK[0.4904,0.4619]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Booking.com. Book a taxi in Sydney for 2nd May at 3 pm. Pick-up location is Sydney Kingsord Smith Airport and destination is Sydney Opera House.\nHistory Actions: SWIPE[LEFT],CLICK[0.6577,0.2442],CLICK[0.6936,0.1253],CLICK[0.4029,0.2721],TYPE[Sydney Kingsford Smith Airport],CLICK[0.4208,0.3272],TYPE[Sydney Opera House],CLICK[0.3862,0.3205],CLICK[0.4087,0.4000],CLICK[0.6856,0.8141],SWIPE[UP],SWIPE[DOWN],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4500,0.9151],CLICK[0.4904,0.4619]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_1_9_6_cb8eb88633d549149aeaad91bdc399db-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5911,0.9220]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220]" + }, + { + "from": "gpt", + "value": "CLICK[0.6849,0.0737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220],CLICK[0.6849,0.0737]" + }, + { + "from": "gpt", + "value": "CLICK[0.6464,0.0599]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220],CLICK[0.6849,0.0737],CLICK[0.6464,0.0599]" + }, + { + "from": "gpt", + "value": "TYPE[Ed Sheeran]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220],CLICK[0.6849,0.0737],CLICK[0.6464,0.0599],TYPE[Ed Sheeran]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220],CLICK[0.6849,0.0737],CLICK[0.6464,0.0599],TYPE[Ed Sheeran],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3237,0.3026]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220],CLICK[0.6849,0.0737],CLICK[0.6464,0.0599],TYPE[Ed Sheeran],PRESS_ENTER,CLICK[0.3237,0.3026]" + }, + { + "from": "gpt", + "value": "CLICK[0.4118,0.7250]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Shape of You\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9220],CLICK[0.6849,0.0737],CLICK[0.6464,0.0599],TYPE[Ed Sheeran],PRESS_ENTER,CLICK[0.3237,0.3026],CLICK[0.4118,0.7250]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_32_69c1e659fb7142fe9da5a0e6032c1254-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6036,0.8730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6036,0.8730]" + }, + { + "from": "gpt", + "value": "CLICK[0.3773,0.3632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6036,0.8730],CLICK[0.3773,0.3632]" + }, + { + "from": "gpt", + "value": "CLICK[0.4056,0.5931]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Mixed for you\". Play the first song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6036,0.8730],CLICK[0.3773,0.3632],CLICK[0.4056,0.5931]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_07511944b83b40e59b2b9cc3a0342d03-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5727,0.8691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5727,0.8691]" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.0520]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5727,0.8691],CLICK[0.6434,0.0520]" + }, + { + "from": "gpt", + "value": "TYPE[Taylor Swift]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5727,0.8691],CLICK[0.6434,0.0520],TYPE[Taylor Swift]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5727,0.8691],CLICK[0.6434,0.0520],TYPE[Taylor Swift],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3865,0.3141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5727,0.8691],CLICK[0.6434,0.0520],TYPE[Taylor Swift],PRESS_ENTER,CLICK[0.3865,0.3141]" + }, + { + "from": "gpt", + "value": "CLICK[0.4470,0.7329]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Taylor Swift\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5727,0.8691],CLICK[0.6434,0.0520],TYPE[Taylor Swift],PRESS_ENTER,CLICK[0.3865,0.3141],CLICK[0.4470,0.7329]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_34_6cf7d6c87ce94e0891d7c1629ea062e2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.8770]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.8770]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.8770],CLICK[0.6503,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[Chill Pop]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.8770],CLICK[0.6503,0.0530],TYPE[Chill Pop]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Pop\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.8770],CLICK[0.6503,0.0530],TYPE[Chill Pop],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_36_405ff93abd854aeb83a333f05f90e3ea-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"Downloads\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_38_c6e07063162d48f6b9c6fc2ed0a06344-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"Downloads\".\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_38_c6e07063162d48f6b9c6fc2ed0a06344-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"Downloads\".\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_38_c6e07063162d48f6b9c6fc2ed0a06344-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"Downloads\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6003,0.8701]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_38_c6e07063162d48f6b9c6fc2ed0a06344-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"Downloads\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8701]" + }, + { + "from": "gpt", + "value": "CLICK[0.6089,0.9743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_38_c6e07063162d48f6b9c6fc2ed0a06344-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"Downloads\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8701],CLICK[0.6089,0.9743]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_38_c6e07063162d48f6b9c6fc2ed0a06344-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5967,0.9122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.9122]" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.9122],CLICK[0.6434,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[Live Love Party Zumba]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.9122],CLICK[0.6434,0.0530],TYPE[Live Love Party Zumba]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.9122],CLICK[0.6434,0.0530],TYPE[Live Love Party Zumba],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5852,0.1763]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.9122],CLICK[0.6434,0.0530],TYPE[Live Love Party Zumba],PRESS_ENTER,CLICK[0.5852,0.1763]" + }, + { + "from": "gpt", + "value": "CLICK[0.4947,0.1826]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Live Love Party Zumba\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.9122],CLICK[0.6434,0.0530],TYPE[Live Love Party Zumba],PRESS_ENTER,CLICK[0.5852,0.1763],CLICK[0.4947,0.1826]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_39_5f9ebb8f13904722a6c3266887d91c96-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6082,0.8638]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8638]" + }, + { + "from": "gpt", + "value": "CLICK[0.5039,0.9780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8638],CLICK[0.5039,0.9780]" + }, + { + "from": "gpt", + "value": "CLICK[0.3461,0.1724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8638],CLICK[0.5039,0.9780],CLICK[0.3461,0.1724]" + }, + { + "from": "gpt", + "value": "CLICK[0.3773,0.3049]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8638],CLICK[0.5039,0.9780],CLICK[0.3461,0.1724],CLICK[0.3773,0.3049]" + }, + { + "from": "gpt", + "value": "CLICK[0.4678,0.6714]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New Releases\". Play the second song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8638],CLICK[0.5039,0.9780],CLICK[0.3461,0.1724],CLICK[0.3773,0.3049],CLICK[0.4678,0.6714]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_40_1eafa08e538a4ab1ae2e0bd4aefac71a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5905,0.8645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8645]" + }, + { + "from": "gpt", + "value": "CLICK[0.6026,0.9727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8645],CLICK[0.6026,0.9727]" + }, + { + "from": "gpt", + "value": "CLICK[0.6095,0.0553]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8645],CLICK[0.6026,0.9727],CLICK[0.6095,0.0553]" + }, + { + "from": "gpt", + "value": "CLICK[0.3888,0.4010]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8645],CLICK[0.6026,0.9727],CLICK[0.6095,0.0553],CLICK[0.3888,0.4010]" + }, + { + "from": "gpt", + "value": "CLICK[0.3461,0.5612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the highest bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8645],CLICK[0.6026,0.9727],CLICK[0.6095,0.0553],CLICK[0.3888,0.4010],CLICK[0.3461,0.5612]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_42_bfa1005e3e6f44bc8bcb49315844a3ae-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5911,0.9007]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007]" + }, + { + "from": "gpt", + "value": "CLICK[0.6418,0.0543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007],CLICK[0.6418,0.0543]" + }, + { + "from": "gpt", + "value": "TYPE[Abbey Road]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007],CLICK[0.6418,0.0543],TYPE[Abbey Road]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007],CLICK[0.6418,0.0543],TYPE[Abbey Road],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3750,0.2934]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007],CLICK[0.6418,0.0543],TYPE[Abbey Road],PRESS_ENTER,CLICK[0.3750,0.2934]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007],CLICK[0.6418,0.0543],TYPE[Abbey Road],PRESS_ENTER,CLICK[0.3750,0.2934],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.6743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Abbey Road\" by \"The Beatles\" and play the song \"Here Comes The Sun\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5911,0.9007],CLICK[0.6418,0.0543],TYPE[Abbey Road],PRESS_ENTER,CLICK[0.3750,0.2934],SWIPE[UP],CLICK[0.4388,0.6743]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_44_7de548a0075d41168cacb2f8c6b64a22-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5783,0.9099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9099]" + }, + { + "from": "gpt", + "value": "CLICK[0.6464,0.0484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9099],CLICK[0.6464,0.0484]" + }, + { + "from": "gpt", + "value": "TYPE[Yesterday]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9099],CLICK[0.6464,0.0484],TYPE[Yesterday]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9099],CLICK[0.6464,0.0484],TYPE[Yesterday],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3405,0.3089]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the song \"Yesterday\" by \"The Beatles\" and play the song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9099],CLICK[0.6464,0.0484],TYPE[Yesterday],PRESS_ENTER,CLICK[0.3405,0.3089]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_45_40fcad01aa0d4e079d2e40ff9e6dc6df-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5980,0.8730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8730]" + }, + { + "from": "gpt", + "value": "CLICK[0.6365,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8730],CLICK[0.6365,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[Ed Sheeran]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8730],CLICK[0.6365,0.0530],TYPE[Ed Sheeran]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8730],CLICK[0.6365,0.0530],TYPE[Ed Sheeran],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3312,0.3118]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8730],CLICK[0.6365,0.0530],TYPE[Ed Sheeran],PRESS_ENTER,CLICK[0.3312,0.3118]" + }, + { + "from": "gpt", + "value": "CLICK[0.4171,0.8141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ed Sheeran\" and play the second song from his top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8730],CLICK[0.6365,0.0530],TYPE[Ed Sheeran],PRESS_ENTER,CLICK[0.3312,0.3118],CLICK[0.4171,0.8141]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_46_377d26f508194d1987912ceecef28975-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5990,0.9220]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.9220]" + }, + { + "from": "gpt", + "value": "CLICK[0.5796,0.9757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.9220],CLICK[0.5796,0.9757]" + }, + { + "from": "gpt", + "value": "CLICK[0.5921,0.0622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.9220],CLICK[0.5796,0.9757],CLICK[0.5921,0.0622]" + }, + { + "from": "gpt", + "value": "CLICK[0.4211,0.4852]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.9220],CLICK[0.5796,0.9757],CLICK[0.5921,0.0622],CLICK[0.4211,0.4852]" + }, + { + "from": "gpt", + "value": "CLICK[0.3826,0.5349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.9220],CLICK[0.5796,0.9757],CLICK[0.5921,0.0622],CLICK[0.4211,0.4852],CLICK[0.3826,0.5349]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_47_650d809c6a004b0886b8dcdb780e157c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5760,0.8707]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5760,0.8707]" + }, + { + "from": "gpt", + "value": "CLICK[0.5967,0.9727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5760,0.8707],CLICK[0.5967,0.9727]" + }, + { + "from": "gpt", + "value": "CLICK[0.5888,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5760,0.8707],CLICK[0.5967,0.9727],CLICK[0.5888,0.0582]" + }, + { + "from": "gpt", + "value": "CLICK[0.3760,0.3227]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Library\". Choose \"History\". Play the third song.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5760,0.8707],CLICK[0.5967,0.9727],CLICK[0.5888,0.0582],CLICK[0.3760,0.3227]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_48_bd8ccccc042947d08ff6812e5a6f931a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5905,0.8776]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8776]" + }, + { + "from": "gpt", + "value": "CLICK[0.6424,0.0566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8776],CLICK[0.6424,0.0566]" + }, + { + "from": "gpt", + "value": "TYPE[1989]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8776],CLICK[0.6424,0.0566],TYPE[1989]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.4789]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8776],CLICK[0.6424,0.0566],TYPE[1989],CLICK[0.3872,0.4789]" + }, + { + "from": "gpt", + "value": "CLICK[0.4615,0.6737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"1989\" by \"Taylor Swift\" and play the song \"Blank Space\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5905,0.8776],CLICK[0.6424,0.0566],TYPE[1989],CLICK[0.3872,0.4789],CLICK[0.4615,0.6737]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_49_51f4d863a34f4a188a3ac1fb00a05206-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6013,0.8793]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3812,0.4484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3812,0.4484]" + }, + { + "from": "gpt", + "value": "CLICK[0.4003,0.6507]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend playists\". Play the second song from the first list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8793],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.3812,0.4484],CLICK[0.4003,0.6507]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_51_c640438818e04550a70701cbb86b7a85-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5974,0.8586]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5974,0.8586]" + }, + { + "from": "gpt", + "value": "CLICK[0.6526,0.0536]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5974,0.8586],CLICK[0.6526,0.0536]" + }, + { + "from": "gpt", + "value": "TYPE[Billie Eilish]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5974,0.8586],CLICK[0.6526,0.0536],TYPE[Billie Eilish]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5974,0.8586],CLICK[0.6526,0.0536],TYPE[Billie Eilish],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3484,0.3013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5974,0.8586],CLICK[0.6526,0.0536],TYPE[Billie Eilish],PRESS_ENTER,CLICK[0.3484,0.3013]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.7411]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Billie Eilish\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5974,0.8586],CLICK[0.6526,0.0536],TYPE[Billie Eilish],PRESS_ENTER,CLICK[0.3484,0.3013],CLICK[0.4388,0.7411]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_52_a82e6180148848909e0de5691a5dcfd7-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5888,0.8724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.8724]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.0553]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.8724],CLICK[0.6487,0.0553]" + }, + { + "from": "gpt", + "value": "TYPE[Upbeat Pop]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.8724],CLICK[0.6487,0.0553],TYPE[Upbeat Pop]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.8724],CLICK[0.6487,0.0553],TYPE[Upbeat Pop],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4303,0.3066]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.8724],CLICK[0.6487,0.0553],TYPE[Upbeat Pop],PRESS_ENTER,CLICK[0.4303,0.3066]" + }, + { + "from": "gpt", + "value": "CLICK[0.4257,0.7987]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Upbeat Pop\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.8724],CLICK[0.6487,0.0553],TYPE[Upbeat Pop],PRESS_ENTER,CLICK[0.4303,0.3066],CLICK[0.4257,0.7987]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_54_08f443f0f0ec4ffbb923b31b549c93da-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5576,0.8622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622]" + }, + { + "from": "gpt", + "value": "CLICK[0.6401,0.0484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622],CLICK[0.6401,0.0484]" + }, + { + "from": "gpt", + "value": "TYPE[Ariana Grande]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622],CLICK[0.6401,0.0484],TYPE[Ariana Grande]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622],CLICK[0.6401,0.0484],TYPE[Ariana Grande],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3345,0.3026]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622],CLICK[0.6401,0.0484],TYPE[Ariana Grande],PRESS_ENTER,CLICK[0.3345,0.3026]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622],CLICK[0.6401,0.0484],TYPE[Ariana Grande],PRESS_ENTER,CLICK[0.3345,0.3026],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4539,0.6247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the fourth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5576,0.8622],CLICK[0.6401,0.0484],TYPE[Ariana Grande],PRESS_ENTER,CLICK[0.3345,0.3026],SWIPE[UP],CLICK[0.4539,0.6247]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_56_9210898da72748019e221745e692830c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5743,0.8632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5743,0.8632]" + }, + { + "from": "gpt", + "value": "CLICK[0.6493,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5743,0.8632],CLICK[0.6493,0.0582]" + }, + { + "from": "gpt", + "value": "TYPE[Coldplay]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5743,0.8632],CLICK[0.6493,0.0582],TYPE[Coldplay]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5743,0.8632],CLICK[0.6493,0.0582],TYPE[Coldplay],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3276,0.3197]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5743,0.8632],CLICK[0.6493,0.0582],TYPE[Coldplay],PRESS_ENTER,CLICK[0.3276,0.3197]" + }, + { + "from": "gpt", + "value": "CLICK[0.4740,0.7503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Coldplay\" and play the third song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5743,0.8632],CLICK[0.6493,0.0582],TYPE[Coldplay],PRESS_ENTER,CLICK[0.3276,0.3197],CLICK[0.4740,0.7503]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_57_180e9ee0804a4bab8e925a65366b999b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5980,0.8984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8984]" + }, + { + "from": "gpt", + "value": "CLICK[0.6493,0.0543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8984],CLICK[0.6493,0.0543]" + }, + { + "from": "gpt", + "value": "TYPE[Just Dance 2022]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8984],CLICK[0.6493,0.0543],TYPE[Just Dance 2022]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8984],CLICK[0.6493,0.0543],TYPE[Just Dance 2022],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3918,0.3717]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Just Dance 2022\" and watch the first video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5980,0.8984],CLICK[0.6493,0.0543],TYPE[Just Dance 2022],PRESS_ENTER,CLICK[0.3918,0.3717]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_16_58_afd4f809d2e241f0b9ada8869cb72ae2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5842,0.9013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.9013]" + }, + { + "from": "gpt", + "value": "CLICK[0.6480,0.0566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.9013],CLICK[0.6480,0.0566]" + }, + { + "from": "gpt", + "value": "TYPE[Bad Habits]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.9013],CLICK[0.6480,0.0566],TYPE[Bad Habits]" + }, + { + "from": "gpt", + "value": "CLICK[0.4908,0.4707]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search and play the song \"Bad Habits\" by Ed Sheeran.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.9013],CLICK[0.6480,0.0566],TYPE[Bad Habits],CLICK[0.4908,0.4707]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_3_17_0_8b94933e3a024baf80091a371ee2537f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3230,0.5809]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3230,0.5809]" + }, + { + "from": "gpt", + "value": "CLICK[0.3069,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3230,0.5809],CLICK[0.3069,0.0674]" + }, + { + "from": "gpt", + "value": "CLICK[0.3720,0.5342]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3230,0.5809],CLICK[0.3069,0.0674],CLICK[0.3720,0.5342]" + }, + { + "from": "gpt", + "value": "CLICK[0.4095,0.3694]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3230,0.5809],CLICK[0.3069,0.0674],CLICK[0.3720,0.5342],CLICK[0.4095,0.3694]" + }, + { + "from": "gpt", + "value": "CLICK[0.7039,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3230,0.5809],CLICK[0.3069,0.0674],CLICK[0.3720,0.5342],CLICK[0.4095,0.3694],CLICK[0.7039,0.0612]" + }, + { + "from": "gpt", + "value": "CLICK[0.6194,0.1872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Sent\" folder. Read the first email in the list, and then return it to the \"Inbox\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3230,0.5809],CLICK[0.3069,0.0674],CLICK[0.3720,0.5342],CLICK[0.4095,0.3694],CLICK[0.7039,0.0612],CLICK[0.6194,0.1872]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_10_2746adcb47fd4c9d883c4665d37483df-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3145,0.5786]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5786]" + }, + { + "from": "gpt", + "value": "CLICK[0.4003,0.0668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5786],CLICK[0.4003,0.0668]" + }, + { + "from": "gpt", + "value": "TYPE[meeting notes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5786],CLICK[0.4003,0.0668],TYPE[meeting notes]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5786],CLICK[0.4003,0.0668],TYPE[meeting notes],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4125,0.3342]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with the subject \"Meeting Notes\". Open the latest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5786],CLICK[0.4003,0.0668],TYPE[meeting notes],PRESS_ENTER,CLICK[0.4125,0.3342]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_12_14dba5e57ee2489da34b167f077235ba-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3059,0.6125]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.9000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000]" + }, + { + "from": "gpt", + "value": "TYPE[testautomation101@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TYPE[automation202@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER,PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.2845,0.3763]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.2845,0.3763]" + }, + { + "from": "gpt", + "value": "CLICK[0.4273,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.2845,0.3763],CLICK[0.4273,0.3349]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.2845,0.3763],CLICK[0.4273,0.3349],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TYPE[This is a test group email for task generation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.2845,0.3763],CLICK[0.4273,0.3349],PRESS_ENTER,TYPE[This is a test group email for task generation]" + }, + { + "from": "gpt", + "value": "CLICK[0.6526,0.0635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to multiple recipients testautomationtest101@gmail.com and testautomationtest202@gmail.com with the subject \"Group Email\", and the body saying \"This is a test group email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6125],CLICK[0.6632,0.9000],TYPE[testautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,TYPE[automation202@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.2845,0.3763],CLICK[0.4273,0.3349],PRESS_ENTER,TYPE[This is a test group email for task generation],CLICK[0.6526,0.0635]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_20_dbed5e30bf934edc94d6c19fd2d54009-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3099,0.5931]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5931]" + }, + { + "from": "gpt", + "value": "CLICK[0.3168,0.0727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5931],CLICK[0.3168,0.0727]" + }, + { + "from": "gpt", + "value": "CLICK[0.3980,0.3579]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5931],CLICK[0.3168,0.0727],CLICK[0.3980,0.3579]" + }, + { + "from": "gpt", + "value": "CLICK[0.4954,0.3586]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Starred\" folder. If there are any emails, open the first email, else perform a search for \"Amazon\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5931],CLICK[0.3168,0.0727],CLICK[0.3980,0.3579],CLICK[0.4954,0.3586]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_26_4a0b3885f9d7463da49184eb4fd2e8eb-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3322,0.5947]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947]" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.0658]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3128,0.0658]" + }, + { + "from": "gpt", + "value": "CLICK[0.3812,0.8109]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3128,0.0658],CLICK[0.3812,0.8109]" + }, + { + "from": "gpt", + "value": "CLICK[0.3789,0.4109]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3128,0.0658],CLICK[0.3812,0.8109],CLICK[0.3789,0.4109]" + }, + { + "from": "gpt", + "value": "CLICK[0.3789,0.4437]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the \"Spam\" folder. If there are any emails, mark the first email as \"Not Spam\", else perform a search for \"Advertisement\" and open the first result email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3128,0.0658],CLICK[0.3812,0.8109],CLICK[0.3789,0.4109],CLICK[0.3789,0.4437]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_27_1ced5e4388f940a3be77da61a0583d17-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3289,0.5918]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918]" + }, + { + "from": "gpt", + "value": "CLICK[0.6141,0.8984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984]" + }, + { + "from": "gpt", + "value": "TYPE[testautomation101@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5191,0.2714]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER,CLICK[0.5191,0.2714]" + }, + { + "from": "gpt", + "value": "CLICK[0.3289,0.3059]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER,CLICK[0.5191,0.2714],CLICK[0.3289,0.3059]" + }, + { + "from": "gpt", + "value": "TYPE[Inivtation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER,CLICK[0.5191,0.2714],CLICK[0.3289,0.3059],TYPE[Inivtation]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER,CLICK[0.5191,0.2714],CLICK[0.3289,0.3059],TYPE[Inivtation],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TYPE[You are invited for the upcoming automation testing event]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER,CLICK[0.5191,0.2714],CLICK[0.3289,0.3059],TYPE[Inivtation],PRESS_ENTER,TYPE[You are invited for the upcoming automation testing event]" + }, + { + "from": "gpt", + "value": "CLICK[0.6533,0.0622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with \"Invitation\" as subject and \"You are invited for the upcoming automation testing event\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5918],CLICK[0.6141,0.8984],TYPE[testautomation101@gmail.com],PRESS_ENTER,CLICK[0.5191,0.2714],CLICK[0.3289,0.3059],TYPE[Inivtation],PRESS_ENTER,TYPE[You are invited for the upcoming automation testing event],CLICK[0.6533,0.0622]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_32_3a81197757574520b1dd72735e2d60aa-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3345,0.5786]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786]" + }, + { + "from": "gpt", + "value": "CLICK[0.3760,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786],CLICK[0.3760,0.0674]" + }, + { + "from": "gpt", + "value": "TYPE[ebay]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786],CLICK[0.3760,0.0674],TYPE[ebay]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786],CLICK[0.3760,0.0674],TYPE[ebay],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3859,0.3349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786],CLICK[0.3760,0.0674],TYPE[ebay],PRESS_ENTER,CLICK[0.3859,0.3349]" + }, + { + "from": "gpt", + "value": "CLICK[0.7053,0.0645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786],CLICK[0.3760,0.0674],TYPE[ebay],PRESS_ENTER,CLICK[0.3859,0.3349],CLICK[0.7053,0.0645]" + }, + { + "from": "gpt", + "value": "CLICK[0.5928,0.1855]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"eBay\". Open the latest email and mark it as \"Important\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3345,0.5786],CLICK[0.3760,0.0674],TYPE[ebay],PRESS_ENTER,CLICK[0.3859,0.3349],CLICK[0.7053,0.0645],CLICK[0.5928,0.1855]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_34_24e9b58c02ee4d769d468817e08a9b4c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.2977,0.5809]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2977,0.5809]" + }, + { + "from": "gpt", + "value": "CLICK[0.3030,0.0658]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2977,0.5809],CLICK[0.3030,0.0658]" + }, + { + "from": "gpt", + "value": "CLICK[0.3842,0.8609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2977,0.5809],CLICK[0.3030,0.0658],CLICK[0.3842,0.8609]" + }, + { + "from": "gpt", + "value": "CLICK[0.3842,0.4437]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. open the first email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2977,0.5809],CLICK[0.3030,0.0658],CLICK[0.3842,0.8609],CLICK[0.3842,0.4437]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_36_93bc46b659a743d99915f45d0a3c1639-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3214,0.5931]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931]" + }, + { + "from": "gpt", + "value": "CLICK[0.6480,0.8891]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891]" + }, + { + "from": "gpt", + "value": "TYPE[pastestautomation101@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.3227]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3128,0.3227]" + }, + { + "from": "gpt", + "value": "TYPE[Demo Email]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3128,0.3227],TYPE[Demo Email]" + }, + { + "from": "gpt", + "value": "TYPE[il\n]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3128,0.3227],TYPE[Demo Email],TYPE[il\n]" + }, + { + "from": "gpt", + "value": "TYPE[paste]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3128,0.3227],TYPE[Demo Email],TYPE[il\n],TYPE[paste]" + }, + { + "from": "gpt", + "value": "CLICK[0.4135,0.3579]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3128,0.3227],TYPE[Demo Email],TYPE[il\n],TYPE[paste],CLICK[0.4135,0.3579]" + }, + { + "from": "gpt", + "value": "CLICK[0.6470,0.0651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to testautomation101@gmail.com with the subject \"Demo Email\", and the body saying \"This is a demonstration email for task generation\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5931],CLICK[0.6480,0.8891],TYPE[pastestautomation101@gmail.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3128,0.3227],TYPE[Demo Email],TYPE[il\n],TYPE[paste],CLICK[0.4135,0.3579],CLICK[0.6470,0.0651]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_5_19_5_a0d70c0aaa134996827ed3cfe274a2ad-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6747,0.8493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493]" + }, + { + "from": "gpt", + "value": "CLICK[0.6678,0.4753]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493],CLICK[0.6678,0.4753]" + }, + { + "from": "gpt", + "value": "CLICK[0.4832,0.2375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493],CLICK[0.6678,0.4753],CLICK[0.4832,0.2375]" + }, + { + "from": "gpt", + "value": "CLICK[0.6572,0.8470]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493],CLICK[0.6678,0.4753],CLICK[0.4832,0.2375],CLICK[0.6572,0.8470]" + }, + { + "from": "gpt", + "value": "CLICK[0.5086,0.9727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493],CLICK[0.6678,0.4753],CLICK[0.4832,0.2375],CLICK[0.6572,0.8470],CLICK[0.5086,0.9727]" + }, + { + "from": "gpt", + "value": "CLICK[0.5283,0.8530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493],CLICK[0.6678,0.4753],CLICK[0.4832,0.2375],CLICK[0.6572,0.8470],CLICK[0.5086,0.9727],CLICK[0.5283,0.8530]" + }, + { + "from": "gpt", + "value": "CLICK[0.4832,0.9566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Dresses\". Sort items by \"popular\". Select the fifth dress from the top. Look for the size guide. Add the dress to the cart in size \"M\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6747,0.8493],CLICK[0.6678,0.4753],CLICK[0.4832,0.2375],CLICK[0.6572,0.8470],CLICK[0.5086,0.9727],CLICK[0.5283,0.8530],CLICK[0.4832,0.9566]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_14_c01a6980e8b449bb95bd81c547ccf7b2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6687,0.5543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543]" + }, + { + "from": "gpt", + "value": "CLICK[0.4010,0.0559]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559]" + }, + { + "from": "gpt", + "value": "TYPE[leather jacket]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5191,0.1464]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket],PRESS_ENTER,CLICK[0.5191,0.1464]" + }, + { + "from": "gpt", + "value": "CLICK[0.3260,0.2875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket],PRESS_ENTER,CLICK[0.5191,0.1464],CLICK[0.3260,0.2875]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.3924]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket],PRESS_ENTER,CLICK[0.5191,0.1464],CLICK[0.3260,0.2875],CLICK[0.6901,0.3924]" + }, + { + "from": "gpt", + "value": "CLICK[0.3941,0.7957]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket],PRESS_ENTER,CLICK[0.5191,0.1464],CLICK[0.3260,0.2875],CLICK[0.6901,0.3924],CLICK[0.3941,0.7957]" + }, + { + "from": "gpt", + "value": "CLICK[0.3007,0.9674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"leather jacket\". Filter search results by color \"black\". Add the third jacket from the list to the wishlist. Check the customer reviews for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.5543],CLICK[0.4010,0.0559],TYPE[leather jacket],PRESS_ENTER,CLICK[0.5191,0.1464],CLICK[0.3260,0.2875],CLICK[0.6901,0.3924],CLICK[0.3941,0.7957],CLICK[0.3007,0.9674]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_15_82e494abf8334547bd3fb25b23365b1d-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"New In\" on the homepage. Sort items by \"price low to high\". Add the first three accessories to the wishlist. Then check the total wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_16_42852a576eee4f7199682464c6d4591b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"New In\" on the homepage. Sort items by \"price low to high\". Add the first three accessories to the wishlist. Then check the total wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_16_42852a576eee4f7199682464c6d4591b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"New In\" on the homepage. Sort items by \"price low to high\". Add the first three accessories to the wishlist. Then check the total wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6793,0.8484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_16_42852a576eee4f7199682464c6d4591b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"New In\" on the homepage. Sort items by \"price low to high\". Add the first three accessories to the wishlist. Then check the total wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6793,0.8484]" + }, + { + "from": "gpt", + "value": "CLICK[0.5086,0.9674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_16_42852a576eee4f7199682464c6d4591b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"New In\" on the homepage. Sort items by \"price low to high\". Add the first three accessories to the wishlist. Then check the total wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6793,0.8484],CLICK[0.5086,0.9674]" + }, + { + "from": "gpt", + "value": "CLICK[0.6286,0.7704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_16_42852a576eee4f7199682464c6d4591b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"New In\" on the homepage. Sort items by \"price low to high\". Add the first three accessories to the wishlist. Then check the total wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6793,0.8484],CLICK[0.5086,0.9674],CLICK[0.6286,0.7704]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_16_42852a576eee4f7199682464c6d4591b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6954,0.5319]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319]" + }, + { + "from": "gpt", + "value": "CLICK[0.4263,0.0543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543]" + }, + { + "from": "gpt", + "value": "TYPE[beauty products]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.7076,0.1066]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER,CLICK[0.7076,0.1066]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.4691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER,CLICK[0.7076,0.1066],CLICK[0.3997,0.4691]" + }, + { + "from": "gpt", + "value": "CLICK[0.3865,0.5082]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER,CLICK[0.7076,0.1066],CLICK[0.3997,0.4691],CLICK[0.3865,0.5082]" + }, + { + "from": "gpt", + "value": "CLICK[0.6924,0.9697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER,CLICK[0.7076,0.1066],CLICK[0.3997,0.4691],CLICK[0.3865,0.5082],CLICK[0.6924,0.9697]" + }, + { + "from": "gpt", + "value": "CLICK[0.3612,0.4375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER,CLICK[0.7076,0.1066],CLICK[0.3997,0.4691],CLICK[0.3865,0.5082],CLICK[0.6924,0.9697],CLICK[0.3612,0.4375]" + }, + { + "from": "gpt", + "value": "CLICK[0.4164,0.9688]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"beauty products\". Filter by \"Cotton\". Select any product and add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6954,0.5319],CLICK[0.4263,0.0543],TYPE[beauty products],PRESS_ENTER,CLICK[0.7076,0.1066],CLICK[0.3997,0.4691],CLICK[0.3865,0.5082],CLICK[0.6924,0.9697],CLICK[0.3612,0.4375],CLICK[0.4164,0.9688]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_17_891ff60b62fa430fb328570605442634-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6655,0.5257]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257]" + }, + { + "from": "gpt", + "value": "CLICK[0.6618,0.7490]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257],CLICK[0.6618,0.7490]" + }, + { + "from": "gpt", + "value": "CLICK[0.3605,0.2865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257],CLICK[0.6618,0.7490],CLICK[0.3605,0.2865]" + }, + { + "from": "gpt", + "value": "CLICK[0.3398,0.3849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257],CLICK[0.6618,0.7490],CLICK[0.3605,0.2865],CLICK[0.3398,0.3849]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.5786]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257],CLICK[0.6618,0.7490],CLICK[0.3605,0.2865],CLICK[0.3398,0.3849],CLICK[0.3872,0.5786]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257],CLICK[0.6618,0.7490],CLICK[0.3605,0.2865],CLICK[0.3398,0.3849],CLICK[0.3872,0.5786],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4211,0.9572]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Home Goods\". Filter by \"recently added\". Choose the first item and check estimated delivery time. Add it to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.5257],CLICK[0.6618,0.7490],CLICK[0.3605,0.2865],CLICK[0.3398,0.3849],CLICK[0.3872,0.5786],SWIPE[UP],CLICK[0.4211,0.9572]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_24_f3a7e8d4bbf849a1b2215232e79b09cd-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6013,0.8546]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546]" + }, + { + "from": "gpt", + "value": "CLICK[0.6832,0.6224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224]" + }, + { + "from": "gpt", + "value": "CLICK[0.6464,0.3342]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342]" + }, + { + "from": "gpt", + "value": "CLICK[0.5414,0.2329]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329]" + }, + { + "from": "gpt", + "value": "CLICK[0.6924,0.5141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141]" + }, + { + "from": "gpt", + "value": "CLICK[0.3635,0.4477]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4602,0.4730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477],SWIPE[UP],CLICK[0.4602,0.4730]" + }, + { + "from": "gpt", + "value": "CLICK[0.7161,0.2214]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477],SWIPE[UP],CLICK[0.4602,0.4730],CLICK[0.7161,0.2214]" + }, + { + "from": "gpt", + "value": "CLICK[0.5023,0.9734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477],SWIPE[UP],CLICK[0.4602,0.4730],CLICK[0.7161,0.2214],CLICK[0.5023,0.9734]" + }, + { + "from": "gpt", + "value": "CLICK[0.6273,0.8095]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477],SWIPE[UP],CLICK[0.4602,0.4730],CLICK[0.7161,0.2214],CLICK[0.5023,0.9734],CLICK[0.6273,0.8095]" + }, + { + "from": "gpt", + "value": "CLICK[0.5339,0.9688]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse the category \"Shoes\". Filter by \"size L\". Add the first pair to the cart. Check the return policy for the product.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8546],CLICK[0.6832,0.6224],CLICK[0.6464,0.3342],CLICK[0.5414,0.2329],CLICK[0.6924,0.5141],CLICK[0.3635,0.4477],SWIPE[UP],CLICK[0.4602,0.4730],CLICK[0.7161,0.2214],CLICK[0.5023,0.9734],CLICK[0.6273,0.8095],CLICK[0.5339,0.9688]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_36_c5b51f05a0204839b27a58d85612d101-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6013,0.8247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8247]" + }, + { + "from": "gpt", + "value": "CLICK[0.4257,0.0428]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8247],CLICK[0.4257,0.0428]" + }, + { + "from": "gpt", + "value": "TYPE[women swimsuits]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8247],CLICK[0.4257,0.0428],TYPE[women swimsuits]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8247],CLICK[0.4257,0.0428],TYPE[women swimsuits],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.7076,0.0974]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women swimsuits\". Filter results by \"best selling\". Select a swimsuit and check the size guide and user reviews. Add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6013,0.8247],CLICK[0.4257,0.0428],TYPE[women swimsuits],PRESS_ENTER,CLICK[0.7076,0.0974]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_37_966510de85fd4985b280329211251335-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5859,0.4444]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444]" + }, + { + "from": "gpt", + "value": "CLICK[0.6839,0.4845]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845]" + }, + { + "from": "gpt", + "value": "CLICK[0.7099,0.2905]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905]" + }, + { + "from": "gpt", + "value": "CLICK[0.3918,0.2329]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329]" + }, + { + "from": "gpt", + "value": "CLICK[0.7016,0.4730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730]" + }, + { + "from": "gpt", + "value": "CLICK[0.4707,0.1342]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342]" + }, + { + "from": "gpt", + "value": "CLICK[0.3359,0.8171]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4296,0.3941]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171],SWIPE[UP],CLICK[0.4296,0.3941]" + }, + { + "from": "gpt", + "value": "CLICK[0.7145,0.2730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171],SWIPE[UP],CLICK[0.4296,0.3941],CLICK[0.7145,0.2730]" + }, + { + "from": "gpt", + "value": "CLICK[0.4296,0.9641]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171],SWIPE[UP],CLICK[0.4296,0.3941],CLICK[0.7145,0.2730],CLICK[0.4296,0.9641]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.8668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171],SWIPE[UP],CLICK[0.4296,0.3941],CLICK[0.7145,0.2730],CLICK[0.4296,0.9641],CLICK[0.3076,0.8668]" + }, + { + "from": "gpt", + "value": "CLICK[0.5398,0.9674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"women dresses\". Filter by \"maxi length\" and sort by \"popular\". Select the third item. Access and read its description. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.4444],CLICK[0.6839,0.4845],CLICK[0.7099,0.2905],CLICK[0.3918,0.2329],CLICK[0.7016,0.4730],CLICK[0.4707,0.1342],CLICK[0.3359,0.8171],SWIPE[UP],CLICK[0.4296,0.3941],CLICK[0.7145,0.2730],CLICK[0.4296,0.9641],CLICK[0.3076,0.8668],CLICK[0.5398,0.9674]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_43_1524f0c08555426fb63bcfa1d755a17a-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.9059]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059]" + }, + { + "from": "gpt", + "value": "CLICK[0.6204,0.1020]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020]" + }, + { + "from": "gpt", + "value": "CLICK[0.4595,0.5757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757]" + }, + { + "from": "gpt", + "value": "CLICK[0.6036,0.5303]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303]" + }, + { + "from": "gpt", + "value": "CLICK[0.4303,0.4069]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Home & Pet\" category. Choose the \"Home Decor\" subcategory. Select a random item and view all its images.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5944,0.9059],CLICK[0.6204,0.1020],CLICK[0.4595,0.5757],CLICK[0.6036,0.5303],CLICK[0.4303,0.4069],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_46_ddfe71c0ae264b989d1a51ddda727d6c-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6135,0.9128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128]" + }, + { + "from": "gpt", + "value": "CLICK[0.5161,0.0944]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944]" + }, + { + "from": "gpt", + "value": "CLICK[0.5681,0.5849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849]" + }, + { + "from": "gpt", + "value": "CLICK[0.4319,0.2829]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849],CLICK[0.4319,0.2829]" + }, + { + "from": "gpt", + "value": "CLICK[0.3773,0.3632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849],CLICK[0.4319,0.2829],CLICK[0.3773,0.3632]" + }, + { + "from": "gpt", + "value": "CLICK[0.6885,0.5135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849],CLICK[0.4319,0.2829],CLICK[0.3773,0.3632],CLICK[0.6885,0.5135]" + }, + { + "from": "gpt", + "value": "CLICK[0.5875,0.2461]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849],CLICK[0.4319,0.2829],CLICK[0.3773,0.3632],CLICK[0.6885,0.5135],CLICK[0.5875,0.2461]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.5497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849],CLICK[0.4319,0.2829],CLICK[0.3773,0.3632],CLICK[0.6885,0.5135],CLICK[0.5875,0.2461],CLICK[0.3872,0.5497]" + }, + { + "from": "gpt", + "value": "CLICK[0.2970,0.9697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men sneakers\". Filter by \"EU size 44\". Add the cheapest one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6135,0.9128],CLICK[0.5161,0.0944],CLICK[0.5681,0.5849],CLICK[0.4319,0.2829],CLICK[0.3773,0.3632],CLICK[0.6885,0.5135],CLICK[0.5875,0.2461],CLICK[0.3872,0.5497],CLICK[0.2970,0.9697]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_47_faff583d45374bdbad0273bdc3a78bca-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.8586]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582]" + }, + { + "from": "gpt", + "value": "TYPE[lipstick]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4740,0.0980]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980]" + }, + { + "from": "gpt", + "value": "CLICK[0.5645,0.3563]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980],CLICK[0.5645,0.3563]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980],CLICK[0.5645,0.3563],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980],CLICK[0.5645,0.3563],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6941,0.3418]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980],CLICK[0.5645,0.3563],SWIPE[UP],SWIPE[UP],CLICK[0.6941,0.3418]" + }, + { + "from": "gpt", + "value": "CLICK[0.3836,0.2490]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980],CLICK[0.5645,0.3563],SWIPE[UP],SWIPE[UP],CLICK[0.6941,0.3418],CLICK[0.3836,0.2490]" + }, + { + "from": "gpt", + "value": "CLICK[0.3928,0.3474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"lipstick\". Sort by \"Most Popular\". Choose one and view its user reviews. Filter reviews by \"most recent\".\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8586],CLICK[0.3974,0.0582],TYPE[lipstick],PRESS_ENTER,CLICK[0.4740,0.0980],CLICK[0.5645,0.3563],SWIPE[UP],SWIPE[UP],CLICK[0.6941,0.3418],CLICK[0.3836,0.2490],CLICK[0.3928,0.3474]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_49_74dae7c256e64542aeb082277cced501-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6043,0.8132]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.4441,0.6125]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125]" + }, + { + "from": "gpt", + "value": "CLICK[0.3895,0.1763]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.7000,0.2859]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763],SWIPE[LEFT],CLICK[0.7000,0.2859]" + }, + { + "from": "gpt", + "value": "CLICK[0.3306,0.3243]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763],SWIPE[LEFT],CLICK[0.7000,0.2859],CLICK[0.3306,0.3243]" + }, + { + "from": "gpt", + "value": "CLICK[0.6908,0.4224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763],SWIPE[LEFT],CLICK[0.7000,0.2859],CLICK[0.3306,0.3243],CLICK[0.6908,0.4224]" + }, + { + "from": "gpt", + "value": "CLICK[0.4164,0.4605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763],SWIPE[LEFT],CLICK[0.7000,0.2859],CLICK[0.3306,0.3243],CLICK[0.6908,0.4224],CLICK[0.4164,0.4605]" + }, + { + "from": "gpt", + "value": "CLICK[0.4799,0.9674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to the \"Bag\" section. Select \"backpacks\". Choose nude color nail black and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.8132],SWIPE[LEFT],CLICK[0.4441,0.6125],CLICK[0.3895,0.1763],SWIPE[LEFT],CLICK[0.7000,0.2859],CLICK[0.3306,0.3243],CLICK[0.6908,0.4224],CLICK[0.4164,0.4605],CLICK[0.4799,0.9674]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_53_76e49107e22c4a3c98c4f4748b768033-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6082,0.8655]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8655]" + }, + { + "from": "gpt", + "value": "CLICK[0.5582,0.6092]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8655],CLICK[0.5582,0.6092]" + }, + { + "from": "gpt", + "value": "CLICK[0.5980,0.2882]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8655],CLICK[0.5582,0.6092],CLICK[0.5980,0.2882]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.5174]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8655],CLICK[0.5582,0.6092],CLICK[0.5980,0.2882],CLICK[0.5898,0.5174]" + }, + { + "from": "gpt", + "value": "CLICK[0.4878,0.9664]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Everyday loungewear\" section in women category. Sort by \"price low to high\". Select second product and add to cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6082,0.8655],CLICK[0.5582,0.6092],CLICK[0.5980,0.2882],CLICK[0.5898,0.5174],CLICK[0.4878,0.9664]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_56_3ed812a6469546b2811073070c791883-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5829,0.8977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977]" + }, + { + "from": "gpt", + "value": "CLICK[0.6609,0.6178]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.2928]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928]" + }, + { + "from": "gpt", + "value": "CLICK[0.4171,0.4546]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5720,0.8240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546],SWIPE[UP],CLICK[0.5720,0.8240]" + }, + { + "from": "gpt", + "value": "CLICK[0.6885,0.9635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546],SWIPE[UP],CLICK[0.5720,0.8240],CLICK[0.6885,0.9635]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546],SWIPE[UP],CLICK[0.5720,0.8240],CLICK[0.6885,0.9635],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6401,0.5349]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546],SWIPE[UP],CLICK[0.5720,0.8240],CLICK[0.6885,0.9635],SWIPE[UP],CLICK[0.6401,0.5349]" + }, + { + "from": "gpt", + "value": "CLICK[0.2931,0.9674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women shoes\". Filter size to EU 38 and sort by highest rating. Select the fifth one and save it to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8977],CLICK[0.6609,0.6178],CLICK[0.6872,0.2928],CLICK[0.4171,0.4546],SWIPE[UP],CLICK[0.5720,0.8240],CLICK[0.6885,0.9635],SWIPE[UP],CLICK[0.6401,0.5349],CLICK[0.2931,0.9674]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_58_c6af89437fba46549be4ec52d37f44ae-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5783,0.8378]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.5668,0.6125]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125]" + }, + { + "from": "gpt", + "value": "CLICK[0.7053,0.2905]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905]" + }, + { + "from": "gpt", + "value": "CLICK[0.4194,0.4997]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.6967]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997],SWIPE[UP],SWIPE[UP],CLICK[0.4378,0.6967]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997],SWIPE[UP],SWIPE[UP],CLICK[0.4378,0.6967],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4401,0.5319]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997],SWIPE[UP],SWIPE[UP],CLICK[0.4378,0.6967],SWIPE[UP],CLICK[0.4401,0.5319]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check the \"beauty products\" category. Filter by \"organic\". Select the second product and add it to the cart. Proceed to the ordering process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8378],SWIPE[LEFT],CLICK[0.5668,0.6125],CLICK[0.7053,0.2905],CLICK[0.4194,0.4997],SWIPE[UP],SWIPE[UP],CLICK[0.4378,0.6967],SWIPE[UP],CLICK[0.4401,0.5319],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_14_59_ad5eb65a6ec84eec985b75e3b07040aa-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5888,0.9046]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.9046]" + }, + { + "from": "gpt", + "value": "CLICK[0.5201,0.0990]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.9046],CLICK[0.5201,0.0990]" + }, + { + "from": "gpt", + "value": "CLICK[0.5612,0.4668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.9046],CLICK[0.5201,0.0990],CLICK[0.5612,0.4668]" + }, + { + "from": "gpt", + "value": "CLICK[0.3668,0.5053]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.9046],CLICK[0.5201,0.0990],CLICK[0.5612,0.4668],CLICK[0.3668,0.5053]" + }, + { + "from": "gpt", + "value": "CLICK[0.2931,0.9720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Men\"s Clothing\". Select \"T-Shirts\" subcategory. Choose an item with 4-star rating or more and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.9046],CLICK[0.5201,0.0990],CLICK[0.5612,0.4668],CLICK[0.3668,0.5053],CLICK[0.2931,0.9720]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_62d3e331ac24406abb1bc17ce4737b7d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5888,0.5418]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418]" + }, + { + "from": "gpt", + "value": "CLICK[0.4447,0.0437]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437]" + }, + { + "from": "gpt", + "value": "TYPE[kid's book]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437],TYPE[kid's book]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437],TYPE[kid's book],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5842,0.1026]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437],TYPE[kid's book],PRESS_ENTER,CLICK[0.5842,0.1026]" + }, + { + "from": "gpt", + "value": "CLICK[0.3536,0.3688]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437],TYPE[kid's book],PRESS_ENTER,CLICK[0.5842,0.1026],CLICK[0.3536,0.3688]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437],TYPE[kid's book],PRESS_ENTER,CLICK[0.5842,0.1026],CLICK[0.3536,0.3688],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5543,0.9618]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"kid\"s book\". Sort results by \"price low to high\". Add the first book to the cart and check its width and length.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5888,0.5418],CLICK[0.4447,0.0437],TYPE[kid's book],PRESS_ENTER,CLICK[0.5842,0.1026],CLICK[0.3536,0.3688],SWIPE[UP],CLICK[0.5543,0.9618]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_12_6d6ab3fc037e4300951376cea975f803-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5957,0.9181]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181]" + }, + { + "from": "gpt", + "value": "CLICK[0.4227,0.0536]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536]" + }, + { + "from": "gpt", + "value": "TYPE[hangbags]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5990,0.1533]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags],PRESS_ENTER,CLICK[0.5990,0.1533]" + }, + { + "from": "gpt", + "value": "CLICK[0.5076,0.2421]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags],PRESS_ENTER,CLICK[0.5990,0.1533],CLICK[0.5076,0.2421]" + }, + { + "from": "gpt", + "value": "CLICK[0.6977,0.3895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags],PRESS_ENTER,CLICK[0.5990,0.1533],CLICK[0.5076,0.2421],CLICK[0.6977,0.3895]" + }, + { + "from": "gpt", + "value": "CLICK[0.5842,0.3977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags],PRESS_ENTER,CLICK[0.5990,0.1533],CLICK[0.5076,0.2421],CLICK[0.6977,0.3895],CLICK[0.5842,0.3977]" + }, + { + "from": "gpt", + "value": "CLICK[0.2938,0.9688]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"handbags\" in the search bar. Filter by \"color blue\". Add one item to the wishlist.\nHistory Actions: SWIPE[UP],CLICK[0.5957,0.9181],CLICK[0.4227,0.0536],TYPE[hangbags],PRESS_ENTER,CLICK[0.5990,0.1533],CLICK[0.5076,0.2421],CLICK[0.6977,0.3895],CLICK[0.5842,0.3977],CLICK[0.2938,0.9688]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_14_c72251f48f324fa99f5ef52fcb6066cc-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5875,0.8438]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.5566,0.6102]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102]" + }, + { + "from": "gpt", + "value": "CLICK[0.6263,0.1655]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3727,0.5572]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655],SWIPE[UP],CLICK[0.3727,0.5572]" + }, + { + "from": "gpt", + "value": "CLICK[0.5115,0.9743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655],SWIPE[UP],CLICK[0.3727,0.5572],CLICK[0.5115,0.9743]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655],SWIPE[UP],CLICK[0.3727,0.5572],CLICK[0.5115,0.9743],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655],SWIPE[UP],CLICK[0.3727,0.5572],CLICK[0.5115,0.9743],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.3586]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Makeup\" subcategory. Add a red color lipstick to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8438],SWIPE[LEFT],CLICK[0.5566,0.6102],CLICK[0.6263,0.1655],SWIPE[UP],CLICK[0.3727,0.5572],CLICK[0.5115,0.9743],SWIPE[DOWN],SWIPE[DOWN],CLICK[0.4141,0.3586]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_16_92f2c68b0ee14de8bf4808c853b12454-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5836,0.7648]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648]" + }, + { + "from": "gpt", + "value": "CLICK[0.4602,0.0566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566]" + }, + { + "from": "gpt", + "value": "TYPE[Plus Size]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3589,0.0974]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974]" + }, + { + "from": "gpt", + "value": "CLICK[0.3520,0.1947]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947]" + }, + { + "from": "gpt", + "value": "CLICK[0.3836,0.3388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388]" + }, + { + "from": "gpt", + "value": "CLICK[0.4977,0.9711]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711]" + }, + { + "from": "gpt", + "value": "CLICK[0.6043,0.8661]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711],CLICK[0.6043,0.8661]" + }, + { + "from": "gpt", + "value": "CLICK[0.5053,0.9503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711],CLICK[0.6043,0.8661],CLICK[0.5053,0.9503]" + }, + { + "from": "gpt", + "value": "CLICK[0.6128,0.0461]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711],CLICK[0.6043,0.8661],CLICK[0.5053,0.9503],CLICK[0.6128,0.0461]" + }, + { + "from": "gpt", + "value": "CLICK[0.2947,0.0520]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711],CLICK[0.6043,0.8661],CLICK[0.5053,0.9503],CLICK[0.6128,0.0461],CLICK[0.2947,0.0520]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711],CLICK[0.6043,0.8661],CLICK[0.5053,0.9503],CLICK[0.6128,0.0461],CLICK[0.2947,0.0520],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Plus Size\" on the homepage. Filter items by \"new in\". Add the first item to the cart and check shipping details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5836,0.7648],CLICK[0.4602,0.0566],TYPE[Plus Size],PRESS_ENTER,CLICK[0.3589,0.0974],CLICK[0.3520,0.1947],CLICK[0.3836,0.3388],CLICK[0.4977,0.9711],CLICK[0.6043,0.8661],CLICK[0.5053,0.9503],CLICK[0.6128,0.0461],CLICK[0.2947,0.0520],SWIPE[DOWN],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_18_844e737902104856a7bdc36fdf5a6cf9-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6043,0.5066]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.5066]" + }, + { + "from": "gpt", + "value": "CLICK[0.6141,0.0990]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.5066],CLICK[0.6141,0.0990]" + }, + { + "from": "gpt", + "value": "CLICK[0.3513,0.5572]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.5066],CLICK[0.6141,0.0990],CLICK[0.3513,0.5572]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.5388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.5066],CLICK[0.6141,0.0990],CLICK[0.3513,0.5572],CLICK[0.3997,0.5388]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.5066],CLICK[0.6141,0.0990],CLICK[0.3513,0.5572],CLICK[0.3997,0.5388],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6747,0.7664]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open the \"Home & Pet\" category and then the \"Kitchen & Dining\" subcategory. Choose a product and check its customer reviews and ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6043,0.5066],CLICK[0.6141,0.0990],CLICK[0.3513,0.5572],CLICK[0.3997,0.5388],SWIPE[UP],CLICK[0.6747,0.7664]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_1_6fd3847ffd7e41d39ef627141aa311a2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5750,0.8668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668]" + }, + { + "from": "gpt", + "value": "CLICK[0.4342,0.0497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497]" + }, + { + "from": "gpt", + "value": "TYPE[running shoes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.1418]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER,CLICK[0.6434,0.1418]" + }, + { + "from": "gpt", + "value": "CLICK[0.6227,0.1964]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER,CLICK[0.6434,0.1418],CLICK[0.6227,0.1964]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.4737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER,CLICK[0.6434,0.1418],CLICK[0.6227,0.1964],CLICK[0.6895,0.4737]" + }, + { + "from": "gpt", + "value": "CLICK[0.4112,0.3648]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER,CLICK[0.6434,0.1418],CLICK[0.6227,0.1964],CLICK[0.6895,0.4737],CLICK[0.4112,0.3648]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER,CLICK[0.6434,0.1418],CLICK[0.6227,0.1964],CLICK[0.6895,0.4737],CLICK[0.4112,0.3648],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.2970,0.9628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"running shoes\". Filter results by \"size M\". Check the shipping details of the first item and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8668],CLICK[0.4342,0.0497],TYPE[running shoes],PRESS_ENTER,CLICK[0.6434,0.1418],CLICK[0.6227,0.1964],CLICK[0.6895,0.4737],CLICK[0.4112,0.3648],SWIPE[UP],CLICK[0.2970,0.9628]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_20_afc471fbe5c04177bb14ab023f4ed095-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.8240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240]" + }, + { + "from": "gpt", + "value": "CLICK[0.6118,0.0928]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928]" + }, + { + "from": "gpt", + "value": "CLICK[0.3543,0.5635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928],CLICK[0.3543,0.5635]" + }, + { + "from": "gpt", + "value": "CLICK[0.5651,0.8224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928],CLICK[0.3543,0.5635],CLICK[0.5651,0.8224]" + }, + { + "from": "gpt", + "value": "CLICK[0.4947,0.9704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928],CLICK[0.3543,0.5635],CLICK[0.5651,0.8224],CLICK[0.4947,0.9704]" + }, + { + "from": "gpt", + "value": "CLICK[0.4701,0.7641]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928],CLICK[0.3543,0.5635],CLICK[0.5651,0.8224],CLICK[0.4947,0.9704],CLICK[0.4701,0.7641]" + }, + { + "from": "gpt", + "value": "CLICK[0.5428,0.9566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928],CLICK[0.3543,0.5635],CLICK[0.5651,0.8224],CLICK[0.4947,0.9704],CLICK[0.4701,0.7641],CLICK[0.5428,0.9566]" + }, + { + "from": "gpt", + "value": "CLICK[0.4822,0.1240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to the \"Home & Pet\" category. Choose \"Kitchen & Dining\" subcategory. Add one product to the cart and check customer reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.8240],CLICK[0.6118,0.0928],CLICK[0.3543,0.5635],CLICK[0.5651,0.8224],CLICK[0.4947,0.9704],CLICK[0.4701,0.7641],CLICK[0.5428,0.9566],CLICK[0.4822,0.1240]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_22_77d4577324774abb8fba89b6c533f08b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5882,0.7405]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405]" + }, + { + "from": "gpt", + "value": "CLICK[0.6763,0.6000]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000]" + }, + { + "from": "gpt", + "value": "CLICK[0.6424,0.3365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365]" + }, + { + "from": "gpt", + "value": "CLICK[0.4204,0.2322]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322]" + }, + { + "from": "gpt", + "value": "CLICK[0.6849,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122]" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.7336]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336]" + }, + { + "from": "gpt", + "value": "CLICK[0.5497,0.9727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336],CLICK[0.5497,0.9727]" + }, + { + "from": "gpt", + "value": "CLICK[0.6816,0.7727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336],CLICK[0.5497,0.9727],CLICK[0.6816,0.7727]" + }, + { + "from": "gpt", + "value": "CLICK[0.5086,0.9697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336],CLICK[0.5497,0.9727],CLICK[0.6816,0.7727],CLICK[0.5086,0.9697]" + }, + { + "from": "gpt", + "value": "CLICK[0.3352,0.1059]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336],CLICK[0.5497,0.9727],CLICK[0.6816,0.7727],CLICK[0.5086,0.9697],CLICK[0.3352,0.1059]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336],CLICK[0.5497,0.9727],CLICK[0.6816,0.7727],CLICK[0.5086,0.9697],CLICK[0.3352,0.1059],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3704,0.5664]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Shoes\" category and filter by \"size S\". Select the fifth shoe and add it to cart. What is its return policy?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5882,0.7405],CLICK[0.6763,0.6000],CLICK[0.6424,0.3365],CLICK[0.4204,0.2322],CLICK[0.6849,0.5122],CLICK[0.6434,0.7336],CLICK[0.5497,0.9727],CLICK[0.6816,0.7727],CLICK[0.5086,0.9697],CLICK[0.3352,0.1059],SWIPE[UP],CLICK[0.3704,0.5664]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_23_fbf9db3d64174b8face078ef6a71aa65-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6020,0.8691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.5474,0.6102]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6931,0.1632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT],CLICK[0.6931,0.1632]" + }, + { + "from": "gpt", + "value": "CLICK[0.4072,0.5941]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT],CLICK[0.6931,0.1632],CLICK[0.4072,0.5941]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT],CLICK[0.6931,0.1632],CLICK[0.4072,0.5941],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6533,0.7987]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT],CLICK[0.6931,0.1632],CLICK[0.4072,0.5941],SWIPE[UP],CLICK[0.6533,0.7987]" + }, + { + "from": "gpt", + "value": "CLICK[0.5191,0.6023]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT],CLICK[0.6931,0.1632],CLICK[0.4072,0.5941],SWIPE[UP],CLICK[0.6533,0.7987],CLICK[0.5191,0.6023]" + }, + { + "from": "gpt", + "value": "CLICK[0.4855,0.9688]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Beauty & Health\" section. Select \"Health Care\" subcategory. Add a eyeshare to the cart and read its ingredients.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6020,0.8691],SWIPE[LEFT],CLICK[0.5474,0.6102],SWIPE[LEFT],CLICK[0.6931,0.1632],CLICK[0.4072,0.5941],SWIPE[UP],CLICK[0.6533,0.7987],CLICK[0.5191,0.6023],CLICK[0.4855,0.9688]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_25_094ce4393c644d1b97e357d69d5cdaef-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5789,0.8500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500]" + }, + { + "from": "gpt", + "value": "CLICK[0.3957,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[earrings]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530],TYPE[earrings]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530],TYPE[earrings],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5888,0.1036]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530],TYPE[earrings],PRESS_ENTER,CLICK[0.5888,0.1036]" + }, + { + "from": "gpt", + "value": "CLICK[0.5819,0.1026]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530],TYPE[earrings],PRESS_ENTER,CLICK[0.5888,0.1036],CLICK[0.5819,0.1026]" + }, + { + "from": "gpt", + "value": "CLICK[0.3605,0.6678]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530],TYPE[earrings],PRESS_ENTER,CLICK[0.5888,0.1036],CLICK[0.5819,0.1026],CLICK[0.3605,0.6678]" + }, + { + "from": "gpt", + "value": "CLICK[0.2931,0.9605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search and select \"earrings\". Filter results by \"price high to low\". Select the third earrings from the list and add them to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5789,0.8500],CLICK[0.3957,0.0530],TYPE[earrings],PRESS_ENTER,CLICK[0.5888,0.1036],CLICK[0.5819,0.1026],CLICK[0.3605,0.6678],CLICK[0.2931,0.9605]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_27_670c9a8e101c4429a02976e57b8c90d3-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5819,0.8355]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355]" + }, + { + "from": "gpt", + "value": "CLICK[0.5451,0.7497]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.5247,0.1671]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497],SWIPE[LEFT],CLICK[0.5247,0.1671]" + }, + { + "from": "gpt", + "value": "CLICK[0.3474,0.2928]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497],SWIPE[LEFT],CLICK[0.5247,0.1671],CLICK[0.3474,0.2928]" + }, + { + "from": "gpt", + "value": "CLICK[0.3237,0.3747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497],SWIPE[LEFT],CLICK[0.5247,0.1671],CLICK[0.3474,0.2928],CLICK[0.3237,0.3747]" + }, + { + "from": "gpt", + "value": "CLICK[0.3559,0.4829]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497],SWIPE[LEFT],CLICK[0.5247,0.1671],CLICK[0.3474,0.2928],CLICK[0.3237,0.3747],CLICK[0.3559,0.4829]" + }, + { + "from": "gpt", + "value": "CLICK[0.2891,0.9559]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Go to \"Accessories\". Find \"women hat\" and sort by \"New Arrivals\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5819,0.8355],CLICK[0.5451,0.7497],SWIPE[LEFT],CLICK[0.5247,0.1671],CLICK[0.3474,0.2928],CLICK[0.3237,0.3747],CLICK[0.3559,0.4829],CLICK[0.2891,0.9559]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_30_82b09065a5eb420f924deb003889078a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5859,0.8747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747]" + }, + { + "from": "gpt", + "value": "CLICK[0.4273,0.0559]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559]" + }, + { + "from": "gpt", + "value": "TYPE[men formal shoes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4349,0.1447]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3941,0.3924]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924]" + }, + { + "from": "gpt", + "value": "CLICK[0.6908,0.4882]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924],CLICK[0.6908,0.4882]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.3987]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924],CLICK[0.6908,0.4882],CLICK[0.6151,0.3987]" + }, + { + "from": "gpt", + "value": "CLICK[0.2961,0.9720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924],CLICK[0.6908,0.4882],CLICK[0.6151,0.3987],CLICK[0.2961,0.9720]" + }, + { + "from": "gpt", + "value": "CLICK[0.3016,0.0391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924],CLICK[0.6908,0.4882],CLICK[0.6151,0.3987],CLICK[0.2961,0.9720],CLICK[0.3016,0.0391]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924],CLICK[0.6908,0.4882],CLICK[0.6151,0.3987],CLICK[0.2961,0.9720],CLICK[0.3016,0.0391],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3895,0.3579]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"men formal shoes\" in the search bar. Filter by \"EU size 46\". Add a pair to the wishlist, and check the return policy.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5859,0.8747],CLICK[0.4273,0.0559],TYPE[men formal shoes],PRESS_ENTER,CLICK[0.4349,0.1447],SWIPE[UP],CLICK[0.3941,0.3924],CLICK[0.6908,0.4882],CLICK[0.6151,0.3987],CLICK[0.2961,0.9720],CLICK[0.3016,0.0391],SWIPE[UP],CLICK[0.3895,0.3579]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_31_772e0d86feda442998fbbbf95ebf04f0-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5829,0.8724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8724]" + }, + { + "from": "gpt", + "value": "CLICK[0.5997,0.9651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8724],CLICK[0.5997,0.9651]" + }, + { + "from": "gpt", + "value": "CLICK[0.6556,0.3211]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8724],CLICK[0.5997,0.9651],CLICK[0.6556,0.3211]" + }, + { + "from": "gpt", + "value": "CLICK[0.5622,0.5579]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the last item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5829,0.8724],CLICK[0.5997,0.9651],CLICK[0.6556,0.3211],CLICK[0.5622,0.5579]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_32_3abcb5dccfcc4a39b0ab5881c1e7938a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5928,0.8701]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701]" + }, + { + "from": "gpt", + "value": "CLICK[0.4181,0.0507]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507]" + }, + { + "from": "gpt", + "value": "TYPE[women sunglasses]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3589,0.1020]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020]" + }, + { + "from": "gpt", + "value": "CLICK[0.4033,0.2414]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020],CLICK[0.4033,0.2414]" + }, + { + "from": "gpt", + "value": "CLICK[0.3345,0.3595]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020],CLICK[0.4033,0.2414],CLICK[0.3345,0.3595]" + }, + { + "from": "gpt", + "value": "CLICK[0.5201,0.9664]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020],CLICK[0.4033,0.2414],CLICK[0.3345,0.3595],CLICK[0.5201,0.9664]" + }, + { + "from": "gpt", + "value": "CLICK[0.3414,0.1118]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020],CLICK[0.4033,0.2414],CLICK[0.3345,0.3595],CLICK[0.5201,0.9664],CLICK[0.3414,0.1118]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020],CLICK[0.4033,0.2414],CLICK[0.3345,0.3595],CLICK[0.5201,0.9664],CLICK[0.3414,0.1118],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4763,0.3372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"women sunglasses\". Filter by \"best rated\". Add the first item to the cart and check its delivery details.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8701],CLICK[0.4181,0.0507],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.3589,0.1020],CLICK[0.4033,0.2414],CLICK[0.3345,0.3595],CLICK[0.5201,0.9664],CLICK[0.3414,0.1118],SWIPE[UP],CLICK[0.4763,0.3372]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_33_92145b32be4345f3816731f51cb6a256-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5928,0.8484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484]" + }, + { + "from": "gpt", + "value": "CLICK[0.4227,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612]" + }, + { + "from": "gpt", + "value": "TYPE[Home Decor]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612],TYPE[Home Decor]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612],TYPE[Home Decor],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5934,0.1059]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612],TYPE[Home Decor],PRESS_ENTER,CLICK[0.5934,0.1059]" + }, + { + "from": "gpt", + "value": "CLICK[0.5875,0.1082]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612],TYPE[Home Decor],PRESS_ENTER,CLICK[0.5934,0.1059],CLICK[0.5875,0.1082]" + }, + { + "from": "gpt", + "value": "CLICK[0.6227,0.2783]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612],TYPE[Home Decor],PRESS_ENTER,CLICK[0.5934,0.1059],CLICK[0.5875,0.1082],CLICK[0.6227,0.2783]" + }, + { + "from": "gpt", + "value": "CLICK[0.2931,0.9773]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse \"Home Decor\" on the homepage. Filter items by \"price high to low\". Add the second item to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.8484],CLICK[0.4227,0.0612],TYPE[Home Decor],PRESS_ENTER,CLICK[0.5934,0.1059],CLICK[0.5875,0.1082],CLICK[0.6227,0.2783],CLICK[0.2931,0.9773]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_35_1db5da63ab434c1d9512a4c2870e24f2-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.7993]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993]" + }, + { + "from": "gpt", + "value": "CLICK[0.4263,0.7336]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.1711]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336],SWIPE[LEFT],CLICK[0.6632,0.1711]" + }, + { + "from": "gpt", + "value": "CLICK[0.5859,0.2477]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336],SWIPE[LEFT],CLICK[0.6632,0.1711],CLICK[0.5859,0.2477]" + }, + { + "from": "gpt", + "value": "CLICK[0.5921,0.2414]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336],SWIPE[LEFT],CLICK[0.6632,0.1711],CLICK[0.5859,0.2477],CLICK[0.5921,0.2414]" + }, + { + "from": "gpt", + "value": "CLICK[0.5737,0.7872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336],SWIPE[LEFT],CLICK[0.6632,0.1711],CLICK[0.5859,0.2477],CLICK[0.5921,0.2414],CLICK[0.5737,0.7872]" + }, + { + "from": "gpt", + "value": "CLICK[0.5306,0.9826]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"men\"s watch\". Sort by \"price high to low\". Check the details of the fifth item. Add it to the cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5898,0.7993],CLICK[0.4263,0.7336],SWIPE[LEFT],CLICK[0.6632,0.1711],CLICK[0.5859,0.2477],CLICK[0.5921,0.2414],CLICK[0.5737,0.7872],CLICK[0.5306,0.9826]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_37_093cfa5cd75a4c088408a65533bfe2a6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5783,0.9082]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082]" + }, + { + "from": "gpt", + "value": "CLICK[0.5553,0.7589]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082],CLICK[0.5553,0.7589]" + }, + { + "from": "gpt", + "value": "CLICK[0.3467,0.2944]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082],CLICK[0.5553,0.7589],CLICK[0.3467,0.2944]" + }, + { + "from": "gpt", + "value": "CLICK[0.3276,0.3803]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082],CLICK[0.5553,0.7589],CLICK[0.3467,0.2944],CLICK[0.3276,0.3803]" + }, + { + "from": "gpt", + "value": "CLICK[0.3230,0.4974]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082],CLICK[0.5553,0.7589],CLICK[0.3467,0.2944],CLICK[0.3276,0.3803],CLICK[0.3230,0.4974]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082],CLICK[0.5553,0.7589],CLICK[0.3467,0.2944],CLICK[0.3276,0.3803],CLICK[0.3230,0.4974],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4125,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open \"Accessories\" category from the homepage. Filter items by \"recently added\". Check the details of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.9082],CLICK[0.5553,0.7589],CLICK[0.3467,0.2944],CLICK[0.3276,0.3803],CLICK[0.3230,0.4974],SWIPE[UP],CLICK[0.4125,0.5122]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_39_26ade1bca57a429a8aa0e5ea695c7776-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5783,0.8984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984]" + }, + { + "from": "gpt", + "value": "CLICK[0.4204,0.0599]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599]" + }, + { + "from": "gpt", + "value": "TYPE[men joggers]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.6862,0.1036]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036]" + }, + { + "from": "gpt", + "value": "CLICK[0.3865,0.5441]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.5878]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878]" + }, + { + "from": "gpt", + "value": "CLICK[0.6924,0.9727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878],CLICK[0.6924,0.9727]" + }, + { + "from": "gpt", + "value": "CLICK[0.5905,0.1043]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878],CLICK[0.6924,0.9727],CLICK[0.5905,0.1043]" + }, + { + "from": "gpt", + "value": "CLICK[0.6273,0.3780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878],CLICK[0.6924,0.9727],CLICK[0.5905,0.1043],CLICK[0.6273,0.3780]" + }, + { + "from": "gpt", + "value": "CLICK[0.4632,0.9704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878],CLICK[0.6924,0.9727],CLICK[0.5905,0.1043],CLICK[0.6273,0.3780],CLICK[0.4632,0.9704]" + }, + { + "from": "gpt", + "value": "CLICK[0.4609,0.8592]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878],CLICK[0.6924,0.9727],CLICK[0.5905,0.1043],CLICK[0.6273,0.3780],CLICK[0.4632,0.9704],CLICK[0.4609,0.8592]" + }, + { + "from": "gpt", + "value": "CLICK[0.4938,0.9589]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men joggers\". Filter by \"Size L\" and sort by \"price low to high\". Add the second item to your cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.8984],CLICK[0.4204,0.0599],TYPE[men joggers],PRESS_ENTER,CLICK[0.6862,0.1036],CLICK[0.3865,0.5441],CLICK[0.6993,0.5878],CLICK[0.6924,0.9727],CLICK[0.5905,0.1043],CLICK[0.6273,0.3780],CLICK[0.4632,0.9704],CLICK[0.4609,0.8592],CLICK[0.4938,0.9589]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_3_fadeed0cc2d94329a3bee572b37361b8-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5967,0.4914]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.4914]" + }, + { + "from": "gpt", + "value": "CLICK[0.4510,0.0513]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.4914],CLICK[0.4510,0.0513]" + }, + { + "from": "gpt", + "value": "TYPE[running shoes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.4914],CLICK[0.4510,0.0513],TYPE[running shoes]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.4914],CLICK[0.4510,0.0513],TYPE[running shoes],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.6401,0.1503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.4914],CLICK[0.4510,0.0513],TYPE[running shoes],PRESS_ENTER,CLICK[0.6401,0.1503]" + }, + { + "from": "gpt", + "value": "CLICK[0.5398,0.4293]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"running shoes\". Filter search results by size \"9\". Add the first pair to the cart. Check its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5967,0.4914],CLICK[0.4510,0.0513],TYPE[running shoes],PRESS_ENTER,CLICK[0.6401,0.1503],CLICK[0.5398,0.4293]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_40_362e5ac8fe3e41a7b67b9f79b89740c3-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6112,0.8431]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6112,0.8431]" + }, + { + "from": "gpt", + "value": "CLICK[0.6757,0.4822]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6112,0.8431],CLICK[0.6757,0.4822]" + }, + { + "from": "gpt", + "value": "CLICK[0.3783,0.5388]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6112,0.8431],CLICK[0.6757,0.4822],CLICK[0.3783,0.5388]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6112,0.8431],CLICK[0.6757,0.4822],CLICK[0.3783,0.5388],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6066,0.4605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Select \"Dresses\" subcategory. Choose an item with 3-star rating or more and check its description.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6112,0.8431],CLICK[0.6757,0.4822],CLICK[0.3783,0.5388],SWIPE[UP],CLICK[0.6066,0.4605]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_42_e215473c2e28491f8113ab9d4f08c323-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5852,0.8431]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.8431]" + }, + { + "from": "gpt", + "value": "CLICK[0.5806,0.9605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.8431],CLICK[0.5806,0.9605]" + }, + { + "from": "gpt", + "value": "CLICK[0.6539,0.4645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.8431],CLICK[0.5806,0.9605],CLICK[0.6539,0.4645]" + }, + { + "from": "gpt", + "value": "CLICK[0.6257,0.5543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Check your cart. Remove the second item. What is the total price of the items in wishlist now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.8431],CLICK[0.5806,0.9605],CLICK[0.6539,0.4645],CLICK[0.6257,0.5543]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_43_25b6aed44c204fbe895c93b6e0a8ec5b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6003,0.8862]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862]" + }, + { + "from": "gpt", + "value": "CLICK[0.6717,0.4766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.4020,0.2829]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829]" + }, + { + "from": "gpt", + "value": "CLICK[0.3168,0.3257]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829],CLICK[0.3168,0.3257]" + }, + { + "from": "gpt", + "value": "CLICK[0.7109,0.4276]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829],CLICK[0.3168,0.3257],CLICK[0.7109,0.4276]" + }, + { + "from": "gpt", + "value": "CLICK[0.4786,0.1332]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829],CLICK[0.3168,0.3257],CLICK[0.7109,0.4276],CLICK[0.4786,0.1332]" + }, + { + "from": "gpt", + "value": "CLICK[0.5829,0.8332]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829],CLICK[0.3168,0.3257],CLICK[0.7109,0.4276],CLICK[0.4786,0.1332],CLICK[0.5829,0.8332]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829],CLICK[0.3168,0.3257],CLICK[0.7109,0.4276],CLICK[0.4786,0.1332],CLICK[0.5829,0.8332],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6780,0.6822]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find and select a \"black dress\". Sort by \"popularity\". Choose the fifth item and view its user reviews.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6003,0.8862],CLICK[0.6717,0.4766],SWIPE[LEFT],CLICK[0.4020,0.2829],CLICK[0.3168,0.3257],CLICK[0.7109,0.4276],CLICK[0.4786,0.1332],CLICK[0.5829,0.8332],SWIPE[UP],CLICK[0.6780,0.6822]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_44_e22c36a185ee4c2d871cefc686be4601-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5990,0.8280]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.8280]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.9543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.8280],CLICK[0.5898,0.9543]" + }, + { + "from": "gpt", + "value": "CLICK[0.6493,0.4651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.8280],CLICK[0.5898,0.9543],CLICK[0.6493,0.4651]" + }, + { + "from": "gpt", + "value": "CLICK[0.6072,0.5503]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the second item. What is the total price of the items in cart now?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5990,0.8280],CLICK[0.5898,0.9543],CLICK[0.6493,0.4651],CLICK[0.6072,0.5503]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_47_6e8005f769034899be11a849055ad1ae-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5852,0.9036]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582]" + }, + { + "from": "gpt", + "value": "TYPE[men sport shoes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4332,0.1464]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5875,0.4062]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062]" + }, + { + "from": "gpt", + "value": "CLICK[0.6908,0.4836]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836]" + }, + { + "from": "gpt", + "value": "CLICK[0.4135,0.3602]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836],CLICK[0.4135,0.3602]" + }, + { + "from": "gpt", + "value": "CLICK[0.4556,0.9572]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836],CLICK[0.4135,0.3602],CLICK[0.4556,0.9572]" + }, + { + "from": "gpt", + "value": "CLICK[0.4194,0.7826]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836],CLICK[0.4135,0.3602],CLICK[0.4556,0.9572],CLICK[0.4194,0.7826]" + }, + { + "from": "gpt", + "value": "CLICK[0.4648,0.9628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836],CLICK[0.4135,0.3602],CLICK[0.4556,0.9572],CLICK[0.4194,0.7826],CLICK[0.4648,0.9628]" + }, + { + "from": "gpt", + "value": "CLICK[0.6151,0.0599]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836],CLICK[0.4135,0.3602],CLICK[0.4556,0.9572],CLICK[0.4194,0.7826],CLICK[0.4648,0.9628],CLICK[0.6151,0.0599]" + }, + { + "from": "gpt", + "value": "CLICK[0.6388,0.9780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"men sport shoes\" in the search bar. Filter by \"EU size 40\". Add a pair to the cart, and proceed to the checkout process.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5852,0.9036],CLICK[0.3974,0.0582],TYPE[men sport shoes],PRESS_ENTER,CLICK[0.4332,0.1464],SWIPE[UP],CLICK[0.5875,0.4062],CLICK[0.6908,0.4836],CLICK[0.4135,0.3602],CLICK[0.4556,0.9572],CLICK[0.4194,0.7826],CLICK[0.4648,0.9628],CLICK[0.6151,0.0599],CLICK[0.6388,0.9780]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_48_33fa04a029b54d4ba082b63e08a31ea3-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5875,0.8954]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954]" + }, + { + "from": "gpt", + "value": "CLICK[0.6724,0.6178]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.2934]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934]" + }, + { + "from": "gpt", + "value": "CLICK[0.5819,0.2944]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944]" + }, + { + "from": "gpt", + "value": "CLICK[0.6240,0.5122]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.3694]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122],SWIPE[UP],CLICK[0.4141,0.3694]" + }, + { + "from": "gpt", + "value": "CLICK[0.7155,0.2214]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122],SWIPE[UP],CLICK[0.4141,0.3694],CLICK[0.7155,0.2214]" + }, + { + "from": "gpt", + "value": "CLICK[0.5474,0.9704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122],SWIPE[UP],CLICK[0.4141,0.3694],CLICK[0.7155,0.2214],CLICK[0.5474,0.9704]" + }, + { + "from": "gpt", + "value": "CLICK[0.4480,0.8102]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122],SWIPE[UP],CLICK[0.4141,0.3694],CLICK[0.7155,0.2214],CLICK[0.5474,0.9704],CLICK[0.4480,0.8102]" + }, + { + "from": "gpt", + "value": "CLICK[0.5589,0.9549]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Browse category \"Shoes\" in the home page. Sort price from high to low. Select the second item. Then check the return policy. Add the item to cart with size 37.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5875,0.8954],CLICK[0.6724,0.6178],CLICK[0.5944,0.2934],CLICK[0.5819,0.2944],CLICK[0.6240,0.5122],SWIPE[UP],CLICK[0.4141,0.3694],CLICK[0.7155,0.2214],CLICK[0.5474,0.9704],CLICK[0.4480,0.8102],CLICK[0.5589,0.9549]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_50_8984e5f6acfd4a57a4d7ad6604397645-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5750,0.8776]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776]" + }, + { + "from": "gpt", + "value": "CLICK[0.4234,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[men shirt]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.7178,0.1480]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480]" + }, + { + "from": "gpt", + "value": "CLICK[0.6401,0.1977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977]" + }, + { + "from": "gpt", + "value": "CLICK[0.6901,0.4391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391]" + }, + { + "from": "gpt", + "value": "CLICK[0.3207,0.1118]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391],CLICK[0.3207,0.1118]" + }, + { + "from": "gpt", + "value": "CLICK[0.3438,0.1993]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391],CLICK[0.3207,0.1118],CLICK[0.3438,0.1993]" + }, + { + "from": "gpt", + "value": "CLICK[0.3599,0.3211]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391],CLICK[0.3207,0.1118],CLICK[0.3438,0.1993],CLICK[0.3599,0.3211]" + }, + { + "from": "gpt", + "value": "CLICK[0.2993,0.9766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391],CLICK[0.3207,0.1118],CLICK[0.3438,0.1993],CLICK[0.3599,0.3211],CLICK[0.2993,0.9766]" + }, + { + "from": "gpt", + "value": "CLICK[0.3046,0.0421]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391],CLICK[0.3207,0.1118],CLICK[0.3438,0.1993],CLICK[0.3599,0.3211],CLICK[0.2993,0.9766],CLICK[0.3046,0.0421]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search \"men shirt\". Filter size to M and sort by recent. Select the first one. Set the size to M and add it to wishlist. Check the buyer ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5750,0.8776],CLICK[0.4234,0.0530],TYPE[men shirt],PRESS_ENTER,SWIPE[LEFT],CLICK[0.7178,0.1480],CLICK[0.6401,0.1977],CLICK[0.6901,0.4391],CLICK[0.3207,0.1118],CLICK[0.3438,0.1993],CLICK[0.3599,0.3211],CLICK[0.2993,0.9766],CLICK[0.3046,0.0421],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_51_3b3bf07015e4498ebcd64922c51c91f6-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5921,0.5533]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5921,0.5533]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.9681]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5921,0.5533],CLICK[0.5944,0.9681]" + }, + { + "from": "gpt", + "value": "CLICK[0.6549,0.3072]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5921,0.5533],CLICK[0.5944,0.9681],CLICK[0.6549,0.3072]" + }, + { + "from": "gpt", + "value": "CLICK[0.6280,0.5572]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Open cart and remove the first item. What is the total price of the items in cart?\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5921,0.5533],CLICK[0.5944,0.9681],CLICK[0.6549,0.3072],CLICK[0.6280,0.5572]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_53_ac1afd2704034b9fb1bf71d1525feefe-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5842,0.5036]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036]" + }, + { + "from": "gpt", + "value": "CLICK[0.6701,0.4737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737]" + }, + { + "from": "gpt", + "value": "CLICK[0.6941,0.2375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.6714]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714]" + }, + { + "from": "gpt", + "value": "CLICK[0.5681,0.7941]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941]" + }, + { + "from": "gpt", + "value": "CLICK[0.6809,0.9789]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789]" + }, + { + "from": "gpt", + "value": "CLICK[0.3398,0.2454]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454]" + }, + { + "from": "gpt", + "value": "CLICK[0.2984,0.3336]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336]" + }, + { + "from": "gpt", + "value": "CLICK[0.6447,0.4836]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336],CLICK[0.6447,0.4836]" + }, + { + "from": "gpt", + "value": "CLICK[0.5283,0.9711]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336],CLICK[0.6447,0.4836],CLICK[0.5283,0.9711]" + }, + { + "from": "gpt", + "value": "CLICK[0.5092,0.8530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336],CLICK[0.6447,0.4836],CLICK[0.5283,0.9711],CLICK[0.5092,0.8530]" + }, + { + "from": "gpt", + "value": "CLICK[0.4822,0.9566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336],CLICK[0.6447,0.4836],CLICK[0.5283,0.9711],CLICK[0.5092,0.8530],CLICK[0.4822,0.9566]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336],CLICK[0.6447,0.4836],CLICK[0.5283,0.9711],CLICK[0.5092,0.8530],CLICK[0.4822,0.9566],CLICK[0.5944,0.0628]" + }, + { + "from": "gpt", + "value": "CLICK[0.6263,0.9720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"dresses\". Filter by \"color red\", then sort by \"new in\". Add the second dress from the list to the cart and proceed to checkout.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5842,0.5036],CLICK[0.6701,0.4737],CLICK[0.6941,0.2375],CLICK[0.3997,0.6714],CLICK[0.5681,0.7941],CLICK[0.6809,0.9789],CLICK[0.3398,0.2454],CLICK[0.2984,0.3336],CLICK[0.6447,0.4836],CLICK[0.5283,0.9711],CLICK[0.5092,0.8530],CLICK[0.4822,0.9566],CLICK[0.5944,0.0628],CLICK[0.6263,0.9720]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_54_c2f79fe100024902b4902b2dc5aac311-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5928,0.9342]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.9342]" + }, + { + "from": "gpt", + "value": "CLICK[0.5622,0.7533]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.9342],CLICK[0.5622,0.7533]" + }, + { + "from": "gpt", + "value": "CLICK[0.5852,0.2467]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.9342],CLICK[0.5622,0.7533],CLICK[0.5852,0.2467]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.2437]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.9342],CLICK[0.5622,0.7533],CLICK[0.5852,0.2467],CLICK[0.5898,0.2437]" + }, + { + "from": "gpt", + "value": "CLICK[0.3197,0.5099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.9342],CLICK[0.5622,0.7533],CLICK[0.5852,0.2467],CLICK[0.5898,0.2437],CLICK[0.3197,0.5099]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Navigate to \"Accessories\" section. Sort items by \"highest price\". Check the shipping information of the first item.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5928,0.9342],CLICK[0.5622,0.7533],CLICK[0.5852,0.2467],CLICK[0.5898,0.2437],CLICK[0.3197,0.5099],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_56_a876774129b345b38af843bf8f92df13-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5681,0.9158]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158]" + }, + { + "from": "gpt", + "value": "CLICK[0.5990,0.0957]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957]" + }, + { + "from": "gpt", + "value": "CLICK[0.6763,0.6891]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.5345,0.1717]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5345,0.1717]" + }, + { + "from": "gpt", + "value": "CLICK[0.3928,0.3520]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5345,0.1717],CLICK[0.3928,0.3520]" + }, + { + "from": "gpt", + "value": "CLICK[0.3023,0.9628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Look for \"Birdd Supplies\". Select any product and add it to the wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5681,0.9158],CLICK[0.5990,0.0957],CLICK[0.6763,0.6891],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],CLICK[0.5345,0.1717],CLICK[0.3928,0.3520],CLICK[0.3023,0.9628]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_5_2f7ba0f9112f428ba432580961c1354c-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5783,0.4605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.0576]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605],CLICK[0.3872,0.0576]" + }, + { + "from": "gpt", + "value": "TYPE[women sunglasses]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605],CLICK[0.3872,0.0576],TYPE[women sunglasses]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605],CLICK[0.3872,0.0576],TYPE[women sunglasses],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5132,0.1013]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605],CLICK[0.3872,0.0576],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.5132,0.1013]" + }, + { + "from": "gpt", + "value": "CLICK[0.4273,0.3411]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605],CLICK[0.3872,0.0576],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.5132,0.1013],CLICK[0.4273,0.3411]" + }, + { + "from": "gpt", + "value": "CLICK[0.3007,0.9697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Find \"women sunglasses\" in the search bar. Sort by \"most popular\". Add the top one to wishlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.5783,0.4605],CLICK[0.3872,0.0576],TYPE[women sunglasses],PRESS_ENTER,CLICK[0.5132,0.1013],CLICK[0.4273,0.3411],CLICK[0.3007,0.9697]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_7_8bf5e43d9fa848c49684a2a7d79bb7f1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5796,0.8178]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178]" + }, + { + "from": "gpt", + "value": "CLICK[0.4033,0.0576]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576]" + }, + { + "from": "gpt", + "value": "TYPE[denim jeans]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4776,0.0905]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans],PRESS_ENTER,CLICK[0.4776,0.0905]" + }, + { + "from": "gpt", + "value": "CLICK[0.3987,0.3563]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans],PRESS_ENTER,CLICK[0.4776,0.0905],CLICK[0.3987,0.3563]" + }, + { + "from": "gpt", + "value": "CLICK[0.4539,0.9641]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans],PRESS_ENTER,CLICK[0.4776,0.0905],CLICK[0.3987,0.3563],CLICK[0.4539,0.9641]" + }, + { + "from": "gpt", + "value": "CLICK[0.6724,0.8898]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans],PRESS_ENTER,CLICK[0.4776,0.0905],CLICK[0.3987,0.3563],CLICK[0.4539,0.9641],CLICK[0.6724,0.8898]" + }, + { + "from": "gpt", + "value": "CLICK[0.5092,0.9720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open SHEIN. Search for \"denim jeans\". Sort the results by \"Popularity\". Choose the first item and add it to the Cart.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.5796,0.8178],CLICK[0.4033,0.0576],TYPE[denim jeans],PRESS_ENTER,CLICK[0.4776,0.0905],CLICK[0.3987,0.3563],CLICK[0.4539,0.9641],CLICK[0.6724,0.8898],CLICK[0.5092,0.9720]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_15_9_7902dd92de374f21ab9ed8eaa29ed869-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3283,0.5918]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4487,0.5618]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618]" + }, + { + "from": "gpt", + "value": "CLICK[0.5980,0.0651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.4227,0.7089]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089]" + }, + { + "from": "gpt", + "value": "CLICK[0.7132,0.2697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697]" + }, + { + "from": "gpt", + "value": "CLICK[0.5645,0.4461]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461]" + }, + { + "from": "gpt", + "value": "TYPE[john_doe@example.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461],TYPE[john_doe@example.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461],TYPE[john_doe@example.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461],TYPE[john_doe@example.com],PRESS_ENTER,PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3082,0.3464]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461],TYPE[john_doe@example.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3082,0.3464]" + }, + { + "from": "gpt", + "value": "TYPE[ing]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461],TYPE[john_doe@example.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3082,0.3464],TYPE[ing]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.0668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Trash the fifth email in the inbox. Then, forward the fourth email to \"john_doe@example.com\" with a message \"I thought you might find this interesting!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5918],SWIPE[UP],CLICK[0.4487,0.5618],CLICK[0.5980,0.0651],SWIPE[DOWN],CLICK[0.4227,0.7089],CLICK[0.7132,0.2697],CLICK[0.5645,0.4461],TYPE[john_doe@example.com],PRESS_ENTER,PRESS_ENTER,CLICK[0.3082,0.3464],TYPE[ing],CLICK[0.6510,0.0668]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_50_81046c2feba340c1b49c0c5f64597dcc-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the first unread email in the inbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_56_3057aee26a234c528a290588b0d57a1d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the first unread email in the inbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_56_3057aee26a234c528a290588b0d57a1d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the first unread email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.5819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_56_3057aee26a234c528a290588b0d57a1d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the first unread email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5819]" + }, + { + "from": "gpt", + "value": "CLICK[0.4885,0.6836]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_56_3057aee26a234c528a290588b0d57a1d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the first unread email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5819],CLICK[0.4885,0.6836]" + }, + { + "from": "gpt", + "value": "CLICK[0.5437,0.0635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_56_3057aee26a234c528a290588b0d57a1d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the first unread email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5819],CLICK[0.4885,0.6836],CLICK[0.5437,0.0635]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_56_3057aee26a234c528a290588b0d57a1d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3243,0.5901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901]" + }, + { + "from": "gpt", + "value": "CLICK[0.3161,0.0720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720]" + }, + { + "from": "gpt", + "value": "CLICK[0.3497,0.2431]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720],CLICK[0.3497,0.2431]" + }, + { + "from": "gpt", + "value": "CLICK[0.4470,0.6322]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720],CLICK[0.3497,0.2431],CLICK[0.4470,0.6322]" + }, + { + "from": "gpt", + "value": "CLICK[0.7145,0.3095]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720],CLICK[0.3497,0.2431],CLICK[0.4470,0.6322],CLICK[0.7145,0.3095]" + }, + { + "from": "gpt", + "value": "CLICK[0.5507,0.3671]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720],CLICK[0.3497,0.2431],CLICK[0.4470,0.6322],CLICK[0.7145,0.3095],CLICK[0.5507,0.3671]" + }, + { + "from": "gpt", + "value": "TYPE[sounds good to me]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720],CLICK[0.3497,0.2431],CLICK[0.4470,0.6322],CLICK[0.7145,0.3095],CLICK[0.5507,0.3671],TYPE[sounds good to me]" + }, + { + "from": "gpt", + "value": "CLICK[0.6526,0.0681]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the latest mail in the social mail with the message \"Sounds good to me!\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3243,0.5901],CLICK[0.3161,0.0720],CLICK[0.3497,0.2431],CLICK[0.4470,0.6322],CLICK[0.7145,0.3095],CLICK[0.5507,0.3671],TYPE[sounds good to me],CLICK[0.6526,0.0681]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_16_59_3e84ffadfe774830ab7fd1ce5bc59c66-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3099,0.5977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5977]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5977],CLICK[0.3122,0.0582]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5977],CLICK[0.3122,0.0582],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3789,0.7053]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5977],CLICK[0.3122,0.0582],SWIPE[UP],CLICK[0.3789,0.7053]" + }, + { + "from": "gpt", + "value": "CLICK[0.4378,0.3701]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5977],CLICK[0.3122,0.0582],SWIPE[UP],CLICK[0.3789,0.7053],CLICK[0.4378,0.3701]" + }, + { + "from": "gpt", + "value": "CLICK[0.3691,0.4454]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to Spam mailbox. Mark the first email as not spam and then move it to inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3099,0.5977],CLICK[0.3122,0.0582],SWIPE[UP],CLICK[0.3789,0.7053],CLICK[0.4378,0.3701],CLICK[0.3691,0.4454]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_10_746b5abc358740e797f55bdacd017724-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.2970,0.5819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819]" + }, + { + "from": "gpt", + "value": "CLICK[0.6372,0.8908]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908]" + }, + { + "from": "gpt", + "value": "TYPE[john.doe@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5299,0.2661]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661]" + }, + { + "from": "gpt", + "value": "CLICK[0.3336,0.2760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661],CLICK[0.3336,0.2760]" + }, + { + "from": "gpt", + "value": "TYPE[meeting schedule]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661],CLICK[0.3336,0.2760],TYPE[meeting schedule]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661],CLICK[0.3336,0.2760],TYPE[meeting schedule],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TYPE[Let's discuss the meeting schedule for next week]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661],CLICK[0.3336,0.2760],TYPE[meeting schedule],PRESS_ENTER,TYPE[Let's discuss the meeting schedule for next week]" + }, + { + "from": "gpt", + "value": "CLICK[0.7053,0.0658]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661],CLICK[0.3336,0.2760],TYPE[meeting schedule],PRESS_ENTER,TYPE[Let's discuss the meeting schedule for next week],CLICK[0.7053,0.0658]" + }, + { + "from": "gpt", + "value": "CLICK[0.5898,0.2437]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a new email to \"john.doe@gmail.com\" with the subject \"Meeting Schedule\" and content \"Let\"s discuss the meeting schedule for next week.\" and then save it as draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2970,0.5819],CLICK[0.6372,0.8908],TYPE[john.doe@gmail.com],PRESS_ENTER,CLICK[0.5299,0.2661],CLICK[0.3336,0.2760],TYPE[meeting schedule],PRESS_ENTER,TYPE[Let's discuss the meeting schedule for next week],CLICK[0.7053,0.0658],CLICK[0.5898,0.2437]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_12_9c60f4a992694442acd86b6e266d41d0-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3283,0.5977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5977]" + }, + { + "from": "gpt", + "value": "CLICK[0.3737,0.0444]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5977],CLICK[0.3737,0.0444]" + }, + { + "from": "gpt", + "value": "TYPE[invoice]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5977],CLICK[0.3737,0.0444],TYPE[invoice]" + }, + { + "from": "gpt", + "value": "CLICK[0.4020,0.3809]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails containing \"Invoice\". Open the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5977],CLICK[0.3737,0.0444],TYPE[invoice],CLICK[0.4020,0.3809]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_15_8574fe98c3744215ae5d331c71274ffa-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.2947,0.5872]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5872]" + }, + { + "from": "gpt", + "value": "CLICK[0.4724,0.6737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5872],CLICK[0.4724,0.6737]" + }, + { + "from": "gpt", + "value": "CLICK[0.6516,0.6789]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5872],CLICK[0.4724,0.6737],CLICK[0.6516,0.6789]" + }, + { + "from": "gpt", + "value": "CLICK[0.6441,0.8086]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a new label called \"Work\" to the first email in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5872],CLICK[0.4724,0.6737],CLICK[0.6516,0.6789],CLICK[0.6441,0.8086]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_19_aa3ca0920e794d06bea443941187fe8d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3174,0.5842]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842]" + }, + { + "from": "gpt", + "value": "CLICK[0.5421,0.5411]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842],CLICK[0.5421,0.5411]" + }, + { + "from": "gpt", + "value": "CLICK[0.6424,0.0605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842],CLICK[0.5421,0.5411],CLICK[0.6424,0.0605]" + }, + { + "from": "gpt", + "value": "CLICK[0.4303,0.6493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842],CLICK[0.5421,0.5411],CLICK[0.6424,0.0605],CLICK[0.4303,0.6493]" + }, + { + "from": "gpt", + "value": "CLICK[0.6464,0.0668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842],CLICK[0.5421,0.5411],CLICK[0.6424,0.0605],CLICK[0.4303,0.6493],CLICK[0.6464,0.0668]" + }, + { + "from": "gpt", + "value": "CLICK[0.4533,0.7296]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842],CLICK[0.5421,0.5411],CLICK[0.6424,0.0605],CLICK[0.4303,0.6493],CLICK[0.6464,0.0668],CLICK[0.4533,0.7296]" + }, + { + "from": "gpt", + "value": "CLICK[0.6625,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Mark the first three emails in the inbox as unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5842],CLICK[0.5421,0.5411],CLICK[0.6424,0.0605],CLICK[0.4303,0.6493],CLICK[0.6464,0.0668],CLICK[0.4533,0.7296],CLICK[0.6625,0.0674]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_4_b4ec5b38d497493f8fdf29295660f22b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3138,0.5819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5819]" + }, + { + "from": "gpt", + "value": "CLICK[0.3651,0.6720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5819],CLICK[0.3651,0.6720]" + }, + { + "from": "gpt", + "value": "CLICK[0.7076,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5819],CLICK[0.3651,0.6720],CLICK[0.7076,0.0582]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.2398]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5819],CLICK[0.3651,0.6720],CLICK[0.7076,0.0582],CLICK[0.6503,0.2398]" + }, + { + "from": "gpt", + "value": "CLICK[0.5398,0.0605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the 2th email in the inbox, flag it as not important and then archive it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5819],CLICK[0.3651,0.6720],CLICK[0.7076,0.0582],CLICK[0.6503,0.2398],CLICK[0.5398,0.0605]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_7_adc060b75fa94d8a9eb379933e6d3039-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3253,0.5878]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5878]" + }, + { + "from": "gpt", + "value": "CLICK[0.3059,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5878],CLICK[0.3059,0.0628]" + }, + { + "from": "gpt", + "value": "CLICK[0.3905,0.6730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5878],CLICK[0.3059,0.0628],CLICK[0.3905,0.6730]" + }, + { + "from": "gpt", + "value": "CLICK[0.4135,0.2760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5878],CLICK[0.3059,0.0628],CLICK[0.3905,0.6730],CLICK[0.4135,0.2760]" + }, + { + "from": "gpt", + "value": "CLICK[0.7046,0.1378]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Starred mailbox. Unstar the first email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5878],CLICK[0.3059,0.0628],CLICK[0.3905,0.6730],CLICK[0.4135,0.2760],CLICK[0.7046,0.1378]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_17_9_b27819a25c1f4fb3aee3739fc59644bb-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.2931,0.6207]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2931,0.6207]" + }, + { + "from": "gpt", + "value": "CLICK[0.3691,0.0691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2931,0.6207],CLICK[0.3691,0.0691]" + }, + { + "from": "gpt", + "value": "TYPE[conference]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2931,0.6207],CLICK[0.3691,0.0691],TYPE[conference]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2931,0.6207],CLICK[0.3691,0.0691],TYPE[conference],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4549,0.2819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2931,0.6207],CLICK[0.3691,0.0691],TYPE[conference],PRESS_ENTER,CLICK[0.4549,0.2819]" + }, + { + "from": "gpt", + "value": "CLICK[0.7023,0.1447]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Conference\". Star all the results.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2931,0.6207],CLICK[0.3691,0.0691],TYPE[conference],PRESS_ENTER,CLICK[0.4549,0.2819],CLICK[0.7023,0.1447]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_12_85c41537816a4efab11fbbd10fe3687c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3138,0.6016]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016]" + }, + { + "from": "gpt", + "value": "CLICK[0.6342,0.8862]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862]" + }, + { + "from": "gpt", + "value": "TYPE[tasktest1234@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4470,0.2796]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER,CLICK[0.4470,0.2796]" + }, + { + "from": "gpt", + "value": "CLICK[0.3405,0.2592]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER,CLICK[0.4470,0.2796],CLICK[0.3405,0.2592]" + }, + { + "from": "gpt", + "value": "TYPE[Email Task Test]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER,CLICK[0.4470,0.2796],CLICK[0.3405,0.2592],TYPE[Email Task Test]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER,CLICK[0.4470,0.2796],CLICK[0.3405,0.2592],TYPE[Email Task Test],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TYPE[This is an email sent for task testing and generation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER,CLICK[0.4470,0.2796],CLICK[0.3405,0.2592],TYPE[Email Task Test],PRESS_ENTER,TYPE[This is an email sent for task testing and generation]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"tasktest1234@gmail.com\". Use \"Email Task Test\" as the subject, include in the content: \"This is an email sent for task testing and generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.6016],CLICK[0.6342,0.8862],TYPE[tasktest1234@gmail.com],PRESS_ENTER,CLICK[0.4470,0.2796],CLICK[0.3405,0.2592],TYPE[Email Task Test],PRESS_ENTER,TYPE[This is an email sent for task testing and generation],CLICK[0.6487,0.0674]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_18_badcc8f1444542ad87fd30c2da1e03cf-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3253,0.5786]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5786]" + }, + { + "from": "gpt", + "value": "CLICK[0.3635,0.0622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5786],CLICK[0.3635,0.0622]" + }, + { + "from": "gpt", + "value": "TYPE[receipt]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5786],CLICK[0.3635,0.0622],TYPE[receipt]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5786],CLICK[0.3635,0.0622],TYPE[receipt],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4954,0.2500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Browse for emails that contain the word \"Receipt\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5786],CLICK[0.3635,0.0622],TYPE[receipt],PRESS_ENTER,CLICK[0.4954,0.2500]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_22_06078fd5f93746599297b61a4f70c2a0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.5865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5865]" + }, + { + "from": "gpt", + "value": "CLICK[0.4332,0.6609]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5865],CLICK[0.4332,0.6609]" + }, + { + "from": "gpt", + "value": "CLICK[0.7046,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5865],CLICK[0.4332,0.6609],CLICK[0.7046,0.0612]" + }, + { + "from": "gpt", + "value": "CLICK[0.5813,0.1901]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5865],CLICK[0.4332,0.6609],CLICK[0.7046,0.0612],CLICK[0.5813,0.1901]" + }, + { + "from": "gpt", + "value": "CLICK[0.6418,0.6806]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5865],CLICK[0.4332,0.6609],CLICK[0.7046,0.0612],CLICK[0.5813,0.1901],CLICK[0.6418,0.6806]" + }, + { + "from": "gpt", + "value": "CLICK[0.6365,0.8026]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Apply a label named \"work\" to the five recent emails in the inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5865],CLICK[0.4332,0.6609],CLICK[0.7046,0.0612],CLICK[0.5813,0.1901],CLICK[0.6418,0.6806],CLICK[0.6365,0.8026]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_32_3e7dd2d463ec4590a79e107ec96d5cfc-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3092,0.5970]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5970]" + }, + { + "from": "gpt", + "value": "CLICK[0.2931,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5970],CLICK[0.2931,0.0628]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5970],CLICK[0.2931,0.0628],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.6984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5970],CLICK[0.2931,0.0628],SWIPE[UP],CLICK[0.3849,0.6984]" + }, + { + "from": "gpt", + "value": "CLICK[0.4095,0.3632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5970],CLICK[0.2931,0.0628],SWIPE[UP],CLICK[0.3849,0.6984],CLICK[0.4095,0.3632]" + }, + { + "from": "gpt", + "value": "CLICK[0.3911,0.4391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the second email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5970],CLICK[0.2931,0.0628],SWIPE[UP],CLICK[0.3849,0.6984],CLICK[0.4095,0.3632],CLICK[0.3911,0.4391]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_34_a31d98096b524a798ca0de33ee237383-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.2947,0.5855]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855]" + }, + { + "from": "gpt", + "value": "CLICK[0.2977,0.0651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4118,0.5266]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651],SWIPE[UP],CLICK[0.4118,0.5266]" + }, + { + "from": "gpt", + "value": "CLICK[0.4947,0.2714]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651],SWIPE[UP],CLICK[0.4118,0.5266],CLICK[0.4947,0.2714]" + }, + { + "from": "gpt", + "value": "CLICK[0.7109,0.2145]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651],SWIPE[UP],CLICK[0.4118,0.5266],CLICK[0.4947,0.2714],CLICK[0.7109,0.2145]" + }, + { + "from": "gpt", + "value": "TYPE[amy.jackson@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651],SWIPE[UP],CLICK[0.4118,0.5266],CLICK[0.4947,0.2714],CLICK[0.7109,0.2145],TYPE[amy.jackson@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4303,0.4437]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the Drafts mailbox. Forward the first email to \"amy.jackson@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.2947,0.5855],CLICK[0.2977,0.0651],SWIPE[UP],CLICK[0.4118,0.5266],CLICK[0.4947,0.2714],CLICK[0.7109,0.2145],TYPE[amy.jackson@gmail.com],CLICK[0.4303,0.4437]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_36_9ef8327f8d1d4bde9f434b385a33a2bd-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3197,0.5911]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911]" + }, + { + "from": "gpt", + "value": "CLICK[0.6424,0.9023]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023]" + }, + { + "from": "gpt", + "value": "TYPE[automatointest202@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4878,0.3158]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com],CLICK[0.4878,0.3158]" + }, + { + "from": "gpt", + "value": "CLICK[0.3184,0.2967]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com],CLICK[0.4878,0.3158],CLICK[0.3184,0.2967]" + }, + { + "from": "gpt", + "value": "TYPE[Notice]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com],CLICK[0.4878,0.3158],CLICK[0.3184,0.2967],TYPE[Notice]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com],CLICK[0.4878,0.3158],CLICK[0.3184,0.2967],TYPE[Notice],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "TYPE[This is an announcement for the upcoming software testing symposium]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com],CLICK[0.4878,0.3158],CLICK[0.3184,0.2967],TYPE[Notice],PRESS_ENTER,TYPE[This is an announcement for the upcoming software testing symposium]" + }, + { + "from": "gpt", + "value": "CLICK[0.6480,0.0645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create an email addressed to automationtest202@gmail.com with \"Notice\" as the subject and \"This is an announcement for the upcoming software testing symposium\" as content. Remember to send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3197,0.5911],CLICK[0.6424,0.9023],TYPE[automatointest202@gmail.com],CLICK[0.4878,0.3158],CLICK[0.3184,0.2967],TYPE[Notice],PRESS_ENTER,TYPE[This is an announcement for the upcoming software testing symposium],CLICK[0.6480,0.0645]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_38_d9fe0cf2d5004c809ad1704c2cc8d880-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3174,0.5740]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740]" + }, + { + "from": "gpt", + "value": "CLICK[0.6549,0.9168]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168]" + }, + { + "from": "gpt", + "value": "TYPE[mary.jane@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4240,0.2592]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com],CLICK[0.4240,0.2592]" + }, + { + "from": "gpt", + "value": "CLICK[0.2822,0.2553]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com],CLICK[0.4240,0.2592],CLICK[0.2822,0.2553]" + }, + { + "from": "gpt", + "value": "TYPE[Agenda Discussion]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com],CLICK[0.4240,0.2592],CLICK[0.2822,0.2553],TYPE[Agenda Discussion]" + }, + { + "from": "gpt", + "value": "TYPE[leftinsertrighh]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com],CLICK[0.4240,0.2592],CLICK[0.2822,0.2553],TYPE[Agenda Discussion],TYPE[leftinsertrighh]" + }, + { + "from": "gpt", + "value": "CLICK[0.7063,0.0622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com],CLICK[0.4240,0.2592],CLICK[0.2822,0.2553],TYPE[Agenda Discussion],TYPE[leftinsertrighh],CLICK[0.7063,0.0622]" + }, + { + "from": "gpt", + "value": "CLICK[0.5451,0.2408]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Form a new email addressed to \"mary.jane@gmail.com\" possessing the subject \"Agenda Discussion\" and containing the content \"Let's set up a dialogue to discuss the agenda for next month.\" Now, save it as a draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5740],CLICK[0.6549,0.9168],TYPE[mary.jane@gmail.com],CLICK[0.4240,0.2592],CLICK[0.2822,0.2553],TYPE[Agenda Discussion],TYPE[leftinsertrighh],CLICK[0.7063,0.0622],CLICK[0.5451,0.2408]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_41_6eca65a6cae1410abab3a206fba5bbbb-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3191,0.5773]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[RIGHT],CLICK[0.3191,0.5773]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[RIGHT],CLICK[0.3191,0.5773],CLICK[0.3105,0.0674]" + }, + { + "from": "gpt", + "value": "CLICK[0.3743,0.9526]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[RIGHT],CLICK[0.3191,0.5773],CLICK[0.3105,0.0674],CLICK[0.3743,0.9526]" + }, + { + "from": "gpt", + "value": "CLICK[0.4257,0.4928]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[RIGHT],CLICK[0.3191,0.5773],CLICK[0.3105,0.0674],CLICK[0.3743,0.9526],CLICK[0.4257,0.4928]" + }, + { + "from": "gpt", + "value": "CLICK[0.7076,0.1457]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Check out the \"Sent\" mailbox and move the third email into the Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[LEFT],SWIPE[RIGHT],CLICK[0.3191,0.5773],CLICK[0.3105,0.0674],CLICK[0.3743,0.9526],CLICK[0.4257,0.4928],CLICK[0.7076,0.1457]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_46_d1e2b020bc574a1b9421fbdc93906a08-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3023,0.5849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849]" + }, + { + "from": "gpt", + "value": "CLICK[0.6395,0.8931]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931]" + }, + { + "from": "gpt", + "value": "TYPE[nametest101@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4678,0.3227]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227]" + }, + { + "from": "gpt", + "value": "TYPE[testname202@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4355,0.3770]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com],CLICK[0.4355,0.3770]" + }, + { + "from": "gpt", + "value": "CLICK[0.2924,0.3257]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com],CLICK[0.4355,0.3770],CLICK[0.2924,0.3257]" + }, + { + "from": "gpt", + "value": "TYPE[collective email]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com],CLICK[0.4355,0.3770],CLICK[0.2924,0.3257],TYPE[collective email]" + }, + { + "from": "gpt", + "value": "CLICK[0.3322,0.3947]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com],CLICK[0.4355,0.3770],CLICK[0.2924,0.3257],TYPE[collective email],CLICK[0.3322,0.3947]" + }, + { + "from": "gpt", + "value": "TYPE[This is a mass email for task production]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com],CLICK[0.4355,0.3770],CLICK[0.2924,0.3257],TYPE[collective email],CLICK[0.3322,0.3947],TYPE[This is a mass email for task production]" + }, + { + "from": "gpt", + "value": "CLICK[0.6602,0.0576]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to several recipients nametest101@gmail.com and testname202@gmail.com. The subject should read \"Collective Email\", and the body needs to mention \"This is a mass email for task production\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3023,0.5849],CLICK[0.6395,0.8931],TYPE[nametest101@gmail.com],CLICK[0.4678,0.3227],TYPE[testname202@gmail.com],CLICK[0.4355,0.3770],CLICK[0.2924,0.3257],TYPE[collective email],CLICK[0.3322,0.3947],TYPE[This is a mass email for task production],CLICK[0.6602,0.0576]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_48_114cff9c017148b586a6c3a7f360bf97-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the first email in inbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_51_39d5c73936244189965b75392d521584-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the first email in inbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_51_39d5c73936244189965b75392d521584-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the first email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3184,0.5750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_51_39d5c73936244189965b75392d521584-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the first email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3184,0.5750]" + }, + { + "from": "gpt", + "value": "CLICK[0.4273,0.6813]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_51_39d5c73936244189965b75392d521584-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the first email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3184,0.5750],CLICK[0.4273,0.6813]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.0691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_51_39d5c73936244189965b75392d521584-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the first email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3184,0.5750],CLICK[0.4273,0.6813],CLICK[0.5944,0.0691]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_51_39d5c73936244189965b75392d521584-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3161,0.5842]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.0865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842],CLICK[0.3122,0.0865]" + }, + { + "from": "gpt", + "value": "CLICK[0.4095,0.1895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842],CLICK[0.3122,0.0865],CLICK[0.4095,0.1895]" + }, + { + "from": "gpt", + "value": "CLICK[0.5086,0.4398]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842],CLICK[0.3122,0.0865],CLICK[0.4095,0.1895],CLICK[0.5086,0.4398]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.0691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842],CLICK[0.3122,0.0865],CLICK[0.4095,0.1895],CLICK[0.5086,0.4398],CLICK[0.6993,0.0691]" + }, + { + "from": "gpt", + "value": "CLICK[0.5842,0.3993]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842],CLICK[0.3122,0.0865],CLICK[0.4095,0.1895],CLICK[0.5086,0.4398],CLICK[0.6993,0.0691],CLICK[0.5842,0.3993]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.6592]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Move the emails in the \"Promotions\" mailbox to \"Spam\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3161,0.5842],CLICK[0.3122,0.0865],CLICK[0.4095,0.1895],CLICK[0.5086,0.4398],CLICK[0.6993,0.0691],CLICK[0.5842,0.3993],CLICK[0.4747,0.6592]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_52_d6c0403e323346d488e782106ebf469a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3260,0.5780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3260,0.5780]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3260,0.5780],CLICK[0.3076,0.0674]" + }, + { + "from": "gpt", + "value": "CLICK[0.4095,0.4829]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3260,0.5780],CLICK[0.3076,0.0674],CLICK[0.4095,0.4829]" + }, + { + "from": "gpt", + "value": "CLICK[0.5651,0.2951]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3260,0.5780],CLICK[0.3076,0.0674],CLICK[0.4095,0.4829],CLICK[0.5651,0.2951]" + }, + { + "from": "gpt", + "value": "CLICK[0.7132,0.1826]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the Sent mailbox and move the first email to Starred mailbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3260,0.5780],CLICK[0.3076,0.0674],CLICK[0.4095,0.4829],CLICK[0.5651,0.2951],CLICK[0.7132,0.1826]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_8_e0519163c5ff43fc88443fdcda99d4a4-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3115,0.5757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5184,0.5763]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763]" + }, + { + "from": "gpt", + "value": "CLICK[0.7063,0.2299]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763],CLICK[0.7063,0.2299]" + }, + { + "from": "gpt", + "value": "CLICK[0.5714,0.3533]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763],CLICK[0.7063,0.2299],CLICK[0.5714,0.3533]" + }, + { + "from": "gpt", + "value": "TYPE[mark.tylor@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763],CLICK[0.7063,0.2299],CLICK[0.5714,0.3533],TYPE[mark.tylor@gmail.com]" + }, + { + "from": "gpt", + "value": "TYPE[\n]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763],CLICK[0.7063,0.2299],CLICK[0.5714,0.3533],TYPE[mark.tylor@gmail.com],TYPE[\n]" + }, + { + "from": "gpt", + "value": "CLICK[0.4158,0.4056]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763],CLICK[0.7063,0.2299],CLICK[0.5714,0.3533],TYPE[mark.tylor@gmail.com],TYPE[\n],CLICK[0.4158,0.4056]" + }, + { + "from": "gpt", + "value": "CLICK[0.6441,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Sent mailbox. Forward the third email to \"mark.taylor@gmail.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3115,0.5757],SWIPE[UP],CLICK[0.5184,0.5763],CLICK[0.7063,0.2299],CLICK[0.5714,0.3533],TYPE[mark.tylor@gmail.com],TYPE[\n],CLICK[0.4158,0.4056],CLICK[0.6441,0.0628]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_18_9_1cb9048b34114235b0c8031feb3612f0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3266,0.5704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.0750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750]" + }, + { + "from": "gpt", + "value": "CLICK[0.3697,0.3003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750],CLICK[0.3697,0.3003]" + }, + { + "from": "gpt", + "value": "CLICK[0.3559,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750],CLICK[0.3697,0.3003],CLICK[0.3559,0.0612]" + }, + { + "from": "gpt", + "value": "TYPE[order confirmation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750],CLICK[0.3697,0.3003],CLICK[0.3559,0.0612],TYPE[order confirmation]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750],CLICK[0.3697,0.3003],CLICK[0.3559,0.0612],TYPE[order confirmation],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4707,0.2500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750],CLICK[0.3697,0.3003],CLICK[0.3559,0.0612],TYPE[order confirmation],PRESS_ENTER,CLICK[0.4707,0.2500]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Find email with \"Order Confirmation\" from the \"Updates\" box, open it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3266,0.5704],CLICK[0.3122,0.0750],CLICK[0.3697,0.3003],CLICK[0.3559,0.0612],TYPE[order confirmation],PRESS_ENTER,CLICK[0.4707,0.2500],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_22_ad0dbdc70ac94519842abdc845d5e0e1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3138,0.5878]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878]" + }, + { + "from": "gpt", + "value": "CLICK[0.6664,0.8990]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990]" + }, + { + "from": "gpt", + "value": "TYPE[softwaretesting@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com]" + }, + { + "from": "gpt", + "value": "TYPE[softwaretesting@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.5000,0.3128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com],CLICK[0.5000,0.3128]" + }, + { + "from": "gpt", + "value": "CLICK[0.3069,0.3266]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com],CLICK[0.5000,0.3128],CLICK[0.3069,0.3266]" + }, + { + "from": "gpt", + "value": "TYPE[Sofrware testing workshop]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com],CLICK[0.5000,0.3128],CLICK[0.3069,0.3266],TYPE[Sofrware testing workshop]" + }, + { + "from": "gpt", + "value": "CLICK[0.3207,0.3187]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com],CLICK[0.5000,0.3128],CLICK[0.3069,0.3266],TYPE[Sofrware testing workshop],CLICK[0.3207,0.3187]" + }, + { + "from": "gpt", + "value": "TYPE[Please join our software testing workshop next weekend]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com],CLICK[0.5000,0.3128],CLICK[0.3069,0.3266],TYPE[Sofrware testing workshop],CLICK[0.3207,0.3187],TYPE[Please join our software testing workshop next weekend]" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.0576]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to softwaretesting@gmail.com with \"Software Testing Workshop\" as subject and \"Please join our software testing workshop next weekend.\" as body. Send the email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3138,0.5878],CLICK[0.6664,0.8990],TYPE[softwaretesting@gmail.com],TYPE[softwaretesting@gmail.com],CLICK[0.5000,0.3128],CLICK[0.3069,0.3266],TYPE[Sofrware testing workshop],CLICK[0.3207,0.3187],TYPE[Please join our software testing workshop next weekend],CLICK[0.6434,0.0576]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_25_e70f297ceef2487fbc713c642d6f4782-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3082,0.5533]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3082,0.5533]" + }, + { + "from": "gpt", + "value": "CLICK[0.3697,0.0704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3082,0.5533],CLICK[0.3697,0.0704]" + }, + { + "from": "gpt", + "value": "TYPE[webinar]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3082,0.5533],CLICK[0.3697,0.0704],TYPE[webinar]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3082,0.5533],CLICK[0.3697,0.0704],TYPE[webinar],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.7109,0.2773]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails with word \"Webinar\". Move all the results to \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3082,0.5533],CLICK[0.3697,0.0704],TYPE[webinar],PRESS_ENTER,CLICK[0.7109,0.2773]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_27_89cb80e8c6eb4f1895d02d81cde134e2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3214,0.5941]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941]" + }, + { + "from": "gpt", + "value": "CLICK[0.3036,0.0697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941],CLICK[0.3036,0.0697]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941],CLICK[0.3036,0.0697],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3766,0.5931]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941],CLICK[0.3036,0.0697],SWIPE[UP],CLICK[0.3766,0.5931]" + }, + { + "from": "gpt", + "value": "CLICK[0.5306,0.2789]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941],CLICK[0.3036,0.0697],SWIPE[UP],CLICK[0.3766,0.5931],CLICK[0.5306,0.2789]" + }, + { + "from": "gpt", + "value": "CLICK[0.5944,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941],CLICK[0.3036,0.0697],SWIPE[UP],CLICK[0.3766,0.5931],CLICK[0.5306,0.2789],CLICK[0.5944,0.0628]" + }, + { + "from": "gpt", + "value": "CLICK[0.6342,0.5510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Access the \"Drafts\" mailbox. Delete the first draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3214,0.5941],CLICK[0.3036,0.0697],SWIPE[UP],CLICK[0.3766,0.5931],CLICK[0.5306,0.2789],CLICK[0.5944,0.0628],CLICK[0.6342,0.5510]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_32_38a5db61e4884ef48c32be2c8e31844c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3359,0.6039]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3359,0.6039]" + }, + { + "from": "gpt", + "value": "CLICK[0.4809,0.7099]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3359,0.6039],CLICK[0.4809,0.7099]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3359,0.6039],CLICK[0.4809,0.7099],CLICK[0.6510,0.0628]" + }, + { + "from": "gpt", + "value": "CLICK[0.5046,0.6898]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3359,0.6039],CLICK[0.4809,0.7099],CLICK[0.6510,0.0628],CLICK[0.5046,0.6898]" + }, + { + "from": "gpt", + "value": "CLICK[0.5405,0.0651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email in the inbox, mark it as unread and then move it to \"Archive\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3359,0.6039],CLICK[0.4809,0.7099],CLICK[0.6510,0.0628],CLICK[0.5046,0.6898],CLICK[0.5405,0.0651]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_35_2951c740f1624169973a2090b7deaf91-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.5855]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855]" + }, + { + "from": "gpt", + "value": "CLICK[0.6493,0.8845]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845]" + }, + { + "from": "gpt", + "value": "TYPE[suppoet@software.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4747,0.3289]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com],CLICK[0.4747,0.3289]" + }, + { + "from": "gpt", + "value": "CLICK[0.3069,0.2645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com],CLICK[0.4747,0.3289],CLICK[0.3069,0.2645]" + }, + { + "from": "gpt", + "value": "TYPE[bug report]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com],CLICK[0.4747,0.3289],CLICK[0.3069,0.2645],TYPE[bug report]" + }, + { + "from": "gpt", + "value": "CLICK[0.3115,0.3405]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com],CLICK[0.4747,0.3289],CLICK[0.3069,0.2645],TYPE[bug report],CLICK[0.3115,0.3405]" + }, + { + "from": "gpt", + "value": "TYPE[found a nug in your latest version 3.5.1]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com],CLICK[0.4747,0.3289],CLICK[0.3069,0.2645],TYPE[bug report],CLICK[0.3115,0.3405],TYPE[found a nug in your latest version 3.5.1]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.0651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Create a message to \"support@software.com\" with subject \"Bug Report\" and body \"Found a bug in your latest version 3.5.1\". Don\"t forget to send it\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3105,0.5855],CLICK[0.6493,0.8845],TYPE[suppoet@software.com],CLICK[0.4747,0.3289],CLICK[0.3069,0.2645],TYPE[bug report],CLICK[0.3115,0.3405],TYPE[found a nug in your latest version 3.5.1],CLICK[0.6503,0.0651]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_38_b4cd8b55836a4177824de67afc78411e-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3174,0.5865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5865]" + }, + { + "from": "gpt", + "value": "CLICK[0.3082,0.0428]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5865],CLICK[0.3082,0.0428]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5865],CLICK[0.3082,0.0428],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3987,0.7826]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5865],CLICK[0.3082,0.0428],SWIPE[UP],CLICK[0.3987,0.7826]" + }, + { + "from": "gpt", + "value": "CLICK[0.4217,0.3418]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5865],CLICK[0.3082,0.0428],SWIPE[UP],CLICK[0.3987,0.7826],CLICK[0.4217,0.3418]" + }, + { + "from": "gpt", + "value": "CLICK[0.6572,0.0622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Select the first email in the \"Junk\" folder and move it to the \"Unread\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3174,0.5865],CLICK[0.3082,0.0428],SWIPE[UP],CLICK[0.3987,0.7826],CLICK[0.4217,0.3418],CLICK[0.6572,0.0622]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_41_d088642f5e8240e9b5c77e80e501a7da-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.6109]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3122,0.6109]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.0668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3122,0.6109],CLICK[0.3105,0.0668]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3122,0.6109],CLICK[0.3105,0.0668],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3691,0.5865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3122,0.6109],CLICK[0.3105,0.0668],SWIPE[UP],CLICK[0.3691,0.5865]" + }, + { + "from": "gpt", + "value": "CLICK[0.4441,0.2622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3122,0.6109],CLICK[0.3105,0.0668],SWIPE[UP],CLICK[0.3691,0.5865],CLICK[0.4441,0.2622]" + }, + { + "from": "gpt", + "value": "CLICK[0.5405,0.2339]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go to the Drafts mailbox. Check the recipient of the first email\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3122,0.6109],CLICK[0.3105,0.0668],SWIPE[UP],CLICK[0.3691,0.5865],CLICK[0.4441,0.2622],CLICK[0.5405,0.2339]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_45_4ff77c2788c941ad9066ea9caf2d3481-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.5681]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5681]" + }, + { + "from": "gpt", + "value": "TYPE[amazon]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5681],TYPE[amazon]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5681],TYPE[amazon],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5023,0.2513]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"Amazon\". Read the most recent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5681],TYPE[amazon],PRESS_ENTER,CLICK[0.5023,0.2513]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_47_f7bf7eb65936484cb0fa3b4a3bf0df81-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3059,0.6033]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.0605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3796,0.4391]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605],SWIPE[UP],CLICK[0.3796,0.4391]" + }, + { + "from": "gpt", + "value": "CLICK[0.4158,0.2697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605],SWIPE[UP],CLICK[0.3796,0.4391],CLICK[0.4158,0.2697]" + }, + { + "from": "gpt", + "value": "CLICK[0.3438,0.8984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605],SWIPE[UP],CLICK[0.3796,0.4391],CLICK[0.4158,0.2697],CLICK[0.3438,0.8984]" + }, + { + "from": "gpt", + "value": "TYPE[Acknowledged, Thank you]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605],SWIPE[UP],CLICK[0.3796,0.4391],CLICK[0.4158,0.2697],CLICK[0.3438,0.8984],TYPE[Acknowledged, Thank you]" + }, + { + "from": "gpt", + "value": "CLICK[0.6493,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Reply to the first email in the \"Starred\" with a message saying \"Acknowledged, Thank you.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3059,0.6033],CLICK[0.3122,0.0605],SWIPE[UP],CLICK[0.3796,0.4391],CLICK[0.4158,0.2697],CLICK[0.3438,0.8984],TYPE[Acknowledged, Thank you],CLICK[0.6493,0.0628]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_48_6f93f122a3f047e09527967e412a6dd6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3283,0.5842]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5842]" + }, + { + "from": "gpt", + "value": "CLICK[0.3082,0.0576]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5842],CLICK[0.3082,0.0576]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5842],CLICK[0.3082,0.0576],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3766,0.6632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5842],CLICK[0.3082,0.0576],SWIPE[UP],CLICK[0.3766,0.6632]" + }, + { + "from": "gpt", + "value": "CLICK[0.5283,0.3586]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5842],CLICK[0.3082,0.0576],SWIPE[UP],CLICK[0.3766,0.6632],CLICK[0.5283,0.3586]" + }, + { + "from": "gpt", + "value": "CLICK[0.5990,0.0645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Look for \"Spam\" folder. Delete the first email in it.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3283,0.5842],CLICK[0.3082,0.0576],SWIPE[UP],CLICK[0.3766,0.6632],CLICK[0.5283,0.3586],CLICK[0.5990,0.0645]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_52_9559c6a4df854088be4bd3445051c255-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3145,0.5849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5849]" + }, + { + "from": "gpt", + "value": "CLICK[0.4250,0.0714]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5849],CLICK[0.4250,0.0714]" + }, + { + "from": "gpt", + "value": "TYPE[payment]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5849],CLICK[0.4250,0.0714],TYPE[payment]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5849],CLICK[0.4250,0.0714],TYPE[payment],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3964,0.2592]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search for emails that contain the word \"Payment\". Access the first one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3145,0.5849],CLICK[0.4250,0.0714],TYPE[payment],PRESS_ENTER,CLICK[0.3964,0.2592]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_53_f76dcb4f03eb4e27a4f490bd85e6e6b5-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3312,0.5734]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734]" + }, + { + "from": "gpt", + "value": "CLICK[0.6470,0.8961]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961]" + }, + { + "from": "gpt", + "value": "TYPE[exampletest1234@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961],TYPE[exampletest1234@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.3928,0.3336]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961],TYPE[exampletest1234@gmail.com],CLICK[0.3928,0.3336]" + }, + { + "from": "gpt", + "value": "CLICK[0.3391,0.3141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961],TYPE[exampletest1234@gmail.com],CLICK[0.3928,0.3336],CLICK[0.3391,0.3141]" + }, + { + "from": "gpt", + "value": "TYPE[Test Email]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961],TYPE[exampletest1234@gmail.com],CLICK[0.3928,0.3336],CLICK[0.3391,0.3141],TYPE[Test Email]" + }, + { + "from": "gpt", + "value": "TYPE[This is a test email for task generation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961],TYPE[exampletest1234@gmail.com],CLICK[0.3928,0.3336],CLICK[0.3391,0.3141],TYPE[Test Email],TYPE[This is a test email for task generation]" + }, + { + "from": "gpt", + "value": "CLICK[0.6441,0.0668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose a new email to \"exampletest1234@gmail.com\". Use \"Test Email\" as the subject, and include in the content: \"This is a test email for task generation.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3312,0.5734],CLICK[0.6470,0.8961],TYPE[exampletest1234@gmail.com],CLICK[0.3928,0.3336],CLICK[0.3391,0.3141],TYPE[Test Email],TYPE[This is a test email for task generation],CLICK[0.6441,0.0668]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_55_8f853a2023a74374a5737bb0f02e0e49-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3414,0.5803]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803]" + }, + { + "from": "gpt", + "value": "CLICK[0.6211,0.8984]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984]" + }, + { + "from": "gpt", + "value": "TYPE[team@example.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4365,0.2691]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com],CLICK[0.4365,0.2691]" + }, + { + "from": "gpt", + "value": "CLICK[0.3289,0.2615]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com],CLICK[0.4365,0.2691],CLICK[0.3289,0.2615]" + }, + { + "from": "gpt", + "value": "TYPE[Project update]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com],CLICK[0.4365,0.2691],CLICK[0.3289,0.2615],TYPE[Project update]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.3336]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com],CLICK[0.4365,0.2691],CLICK[0.3289,0.2615],TYPE[Project update],CLICK[0.3076,0.3336]" + }, + { + "from": "gpt", + "value": "TYPE[The project is on track and will be completed by the end o fthis month]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com],CLICK[0.4365,0.2691],CLICK[0.3289,0.2615],TYPE[Project update],CLICK[0.3076,0.3336],TYPE[The project is on track and will be completed by the end o fthis month]" + }, + { + "from": "gpt", + "value": "CLICK[0.6457,0.0576]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"team@example.com\" with a subject \"Project Update\" and body \"The project is on track and will be completed by the end of this month.\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3414,0.5803],CLICK[0.6211,0.8984],TYPE[team@example.com],CLICK[0.4365,0.2691],CLICK[0.3289,0.2615],TYPE[Project update],CLICK[0.3076,0.3336],TYPE[The project is on track and will be completed by the end o fthis month],CLICK[0.6457,0.0576]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_19_59_f5d6b226ee4040719cfc9e2cff13f2ff-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.5832]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832]" + }, + { + "from": "gpt", + "value": "CLICK[0.6648,0.8977]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977]" + }, + { + "from": "gpt", + "value": "TYPE[testautomation101@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4457,0.3243]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243]" + }, + { + "from": "gpt", + "value": "TYPE[automationtest202@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4961,0.3618]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com],CLICK[0.4961,0.3618]" + }, + { + "from": "gpt", + "value": "CLICK[0.3059,0.3786]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com],CLICK[0.4961,0.3618],CLICK[0.3059,0.3786]" + }, + { + "from": "gpt", + "value": "TYPE[Gruop Test Email]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com],CLICK[0.4961,0.3618],CLICK[0.3059,0.3786],TYPE[Gruop Test Email]" + }, + { + "from": "gpt", + "value": "CLICK[0.3345,0.3809]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com],CLICK[0.4961,0.3618],CLICK[0.3059,0.3786],TYPE[Gruop Test Email],CLICK[0.3345,0.3809]" + }, + { + "from": "gpt", + "value": "TYPE[This is a test for task generation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com],CLICK[0.4961,0.3618],CLICK[0.3059,0.3786],TYPE[Gruop Test Email],CLICK[0.3345,0.3809],TYPE[This is a test for task generation]" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.0605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Compose an email to \"testautomation101@gmail.com\" and \"automationtest202@gmail.com\" with the subject \"Group Test Email\", and the body saying \"This is a test for task generation\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5832],CLICK[0.6648,0.8977],TYPE[testautomation101@gmail.com],CLICK[0.4457,0.3243],TYPE[automationtest202@gmail.com],CLICK[0.4961,0.3618],CLICK[0.3059,0.3786],TYPE[Gruop Test Email],CLICK[0.3345,0.3809],TYPE[This is a test for task generation],CLICK[0.6434,0.0605]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_10_2312aafe6b424f08ba634f857ad20615-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3191,0.5819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.5819]" + }, + { + "from": "gpt", + "value": "CLICK[0.2947,0.0766]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.5819],CLICK[0.2947,0.0766]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.5819],CLICK[0.2947,0.0766],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3888,0.6875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.5819],CLICK[0.2947,0.0766],SWIPE[UP],CLICK[0.3888,0.6875]" + }, + { + "from": "gpt", + "value": "CLICK[0.4434,0.4743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Open the second email present in the \"Trash\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3191,0.5819],CLICK[0.2947,0.0766],SWIPE[UP],CLICK[0.3888,0.6875],CLICK[0.4434,0.4743]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_13_8ff4094c6ad046818f110f8ada70c586-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3322,0.5947]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947]" + }, + { + "from": "gpt", + "value": "CLICK[0.3704,0.0737]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3704,0.0737]" + }, + { + "from": "gpt", + "value": "TYPE[webimar]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3704,0.0737],TYPE[webimar]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3704,0.0737],TYPE[webimar],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4188,0.2905]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3704,0.0737],TYPE[webimar],PRESS_ENTER,CLICK[0.4188,0.2905]" + }, + { + "from": "gpt", + "value": "CLICK[0.5437,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Emails containing the term \"Webinar\". Archive all the resultant emails.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5947],CLICK[0.3704,0.0737],TYPE[webimar],PRESS_ENTER,CLICK[0.4188,0.2905],CLICK[0.5437,0.0628]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_18_fdff73fe108d44a492a42f0e70f0e798-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.5941]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5941]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.0704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5941],CLICK[0.3974,0.0704]" + }, + { + "from": "gpt", + "value": "TYPE[BBC news]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5941],CLICK[0.3974,0.0704],TYPE[BBC news]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5941],CLICK[0.3974,0.0704],TYPE[BBC news],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3980,0.2559]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5941],CLICK[0.3974,0.0704],TYPE[BBC news],PRESS_ENTER,CLICK[0.3980,0.2559]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC News\". Read the newest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5941],CLICK[0.3974,0.0704],TYPE[BBC news],PRESS_ENTER,CLICK[0.3980,0.2559],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_19_cd82a996ce47429bbbc90a09ae96cc67-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3306,0.6033]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033]" + }, + { + "from": "gpt", + "value": "CLICK[0.3865,0.0783]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783]" + }, + { + "from": "gpt", + "value": "TYPE[proposal]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.7023,0.3895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895]" + }, + { + "from": "gpt", + "value": "CLICK[0.4480,0.3510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6049,0.9076]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510],SWIPE[UP],SWIPE[UP],CLICK[0.6049,0.9076]" + }, + { + "from": "gpt", + "value": "TYPE[manager@example.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510],SWIPE[UP],SWIPE[UP],CLICK[0.6049,0.9076],TYPE[manager@example.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.3612,0.2714]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510],SWIPE[UP],SWIPE[UP],CLICK[0.6049,0.9076],TYPE[manager@example.com],CLICK[0.3612,0.2714]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.0582]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Proposal\". Star the second one and Forward it to \"manager@example.com\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3306,0.6033],CLICK[0.3865,0.0783],TYPE[proposal],PRESS_ENTER,CLICK[0.7023,0.3895],CLICK[0.4480,0.3510],SWIPE[UP],SWIPE[UP],CLICK[0.6049,0.9076],TYPE[manager@example.com],CLICK[0.3612,0.2714],CLICK[0.6487,0.0582]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_22_3052b8a651044a129575be958a91b3cd-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.5727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5727]" + }, + { + "from": "gpt", + "value": "CLICK[0.3076,0.0720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5727],CLICK[0.3076,0.0720]" + }, + { + "from": "gpt", + "value": "CLICK[0.3865,0.7865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5727],CLICK[0.3076,0.0720],CLICK[0.3865,0.7865]" + }, + { + "from": "gpt", + "value": "CLICK[0.4118,0.4859]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5727],CLICK[0.3076,0.0720],CLICK[0.3865,0.7865],CLICK[0.4118,0.4859]" + }, + { + "from": "gpt", + "value": "CLICK[0.5391,0.0635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Archive the third email in the \"Starred\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3076,0.5727],CLICK[0.3076,0.0720],CLICK[0.3865,0.7865],CLICK[0.4118,0.4859],CLICK[0.5391,0.0635]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_24_d1af43dd8a6f4ce5b55e18a35a95a391-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3322,0.5780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5780]" + }, + { + "from": "gpt", + "value": "CLICK[0.3849,0.0704]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5780],CLICK[0.3849,0.0704]" + }, + { + "from": "gpt", + "value": "TYPE[BBC sports]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5780],CLICK[0.3849,0.0704],TYPE[BBC sports]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5780],CLICK[0.3849,0.0704],TYPE[BBC sports],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3911,0.2615]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5780],CLICK[0.3849,0.0704],TYPE[BBC sports],PRESS_ENTER,CLICK[0.3911,0.2615]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails from \"BBC sports\". Read the lastest email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5780],CLICK[0.3849,0.0704],TYPE[BBC sports],PRESS_ENTER,CLICK[0.3911,0.2615],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_25_21e3ababa580449184f949dc359b3fa0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3092,0.5987]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987]" + }, + { + "from": "gpt", + "value": "CLICK[0.6717,0.9168]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168]" + }, + { + "from": "gpt", + "value": "TYPE[testappium002@gmail.com]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com]" + }, + { + "from": "gpt", + "value": "CLICK[0.4280,0.3280]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com],CLICK[0.4280,0.3280]" + }, + { + "from": "gpt", + "value": "CLICK[0.2993,0.2645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com],CLICK[0.4280,0.3280],CLICK[0.2993,0.2645]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.2651]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com],CLICK[0.4280,0.3280],CLICK[0.2993,0.2645],CLICK[0.3997,0.2651]" + }, + { + "from": "gpt", + "value": "CLICK[0.3036,0.3296]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com],CLICK[0.4280,0.3280],CLICK[0.2993,0.2645],CLICK[0.3997,0.2651],CLICK[0.3036,0.3296]" + }, + { + "from": "gpt", + "value": "TYPE[This is a test email from the team]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com],CLICK[0.4280,0.3280],CLICK[0.2993,0.2645],CLICK[0.3997,0.2651],CLICK[0.3036,0.3296],TYPE[This is a test email from the team]" + }, + { + "from": "gpt", + "value": "CLICK[0.6470,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Write an email to testappium002@outlook.com. The subject is \"Test email\". The main content is \"this is a test email from the team\"\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3092,0.5987],CLICK[0.6717,0.9168],TYPE[testappium002@gmail.com],CLICK[0.4280,0.3280],CLICK[0.2993,0.2645],CLICK[0.3997,0.2651],CLICK[0.3036,0.3296],TYPE[This is a test email from the team],CLICK[0.6470,0.0628]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_26_ee44aad6c4634c7d8088a5fca723c1f7-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Read the first email in the inbox. Then mark it as Unread.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_28_c1ba156504a34433bd4d1f312825eafe-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Read the first email in the inbox. Then mark it as Unread.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_28_c1ba156504a34433bd4d1f312825eafe-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Read the first email in the inbox. Then mark it as Unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3253,0.5803]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_28_c1ba156504a34433bd4d1f312825eafe-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Read the first email in the inbox. Then mark it as Unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5803]" + }, + { + "from": "gpt", + "value": "CLICK[0.4079,0.6730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_28_c1ba156504a34433bd4d1f312825eafe-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Read the first email in the inbox. Then mark it as Unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5803],CLICK[0.4079,0.6730]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.0681]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_28_c1ba156504a34433bd4d1f312825eafe-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Read the first email in the inbox. Then mark it as Unread.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3253,0.5803],CLICK[0.4079,0.6730],CLICK[0.6503,0.0681]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_28_c1ba156504a34433bd4d1f312825eafe-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the second email in inbox.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_30_52e73573397f401a90e7a7d01503c60f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the second email in inbox.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_30_52e73573397f401a90e7a7d01503c60f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the second email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3289,0.5717]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_30_52e73573397f401a90e7a7d01503c60f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the second email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5717]" + }, + { + "from": "gpt", + "value": "CLICK[0.3974,0.7993]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_30_52e73573397f401a90e7a7d01503c60f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the second email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5717],CLICK[0.3974,0.7993]" + }, + { + "from": "gpt", + "value": "CLICK[0.5974,0.0635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_30_52e73573397f401a90e7a7d01503c60f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Delete the second email in inbox.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3289,0.5717],CLICK[0.3974,0.7993],CLICK[0.5974,0.0635]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_30_52e73573397f401a90e7a7d01503c60f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3322,0.5727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5727]" + }, + { + "from": "gpt", + "value": "CLICK[0.4072,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5727],CLICK[0.4072,0.0674]" + }, + { + "from": "gpt", + "value": "TYPE[verification]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5727],CLICK[0.4072,0.0674],TYPE[verification]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5727],CLICK[0.4072,0.0674],TYPE[verification],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4885,0.7365]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5727],CLICK[0.4072,0.0674],TYPE[verification],PRESS_ENTER,CLICK[0.4885,0.7365]" + }, + { + "from": "gpt", + "value": "CLICK[0.6003,0.0668]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Search emails that contain \"Verification\". Delete the third one.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3322,0.5727],CLICK[0.4072,0.0674],TYPE[verification],PRESS_ENTER,CLICK[0.4885,0.7365],CLICK[0.6003,0.0668]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_31_fb9434f53d09451cb8896954341acb47-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3007,0.5941]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941]" + }, + { + "from": "gpt", + "value": "CLICK[0.3092,0.0697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941],CLICK[0.3092,0.0697]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941],CLICK[0.3092,0.0697],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3645,0.6345]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941],CLICK[0.3092,0.0697],SWIPE[UP],CLICK[0.3645,0.6345]" + }, + { + "from": "gpt", + "value": "CLICK[0.4303,0.2783]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941],CLICK[0.3092,0.0697],SWIPE[UP],CLICK[0.3645,0.6345],CLICK[0.4303,0.2783]" + }, + { + "from": "gpt", + "value": "CLICK[0.7053,0.2076]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941],CLICK[0.3092,0.0697],SWIPE[UP],CLICK[0.3645,0.6345],CLICK[0.4303,0.2783],CLICK[0.7053,0.2076]" + }, + { + "from": "gpt", + "value": "CLICK[0.6470,0.0622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Drafts\" folder and send the first saved email draft.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3007,0.5941],CLICK[0.3092,0.0697],SWIPE[UP],CLICK[0.3645,0.6345],CLICK[0.4303,0.2783],CLICK[0.7053,0.2076],CLICK[0.6470,0.0622]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_3_89d2071006b84352a7a05976d53caeab-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.5803]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803]" + }, + { + "from": "gpt", + "value": "CLICK[0.4648,0.5819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803],CLICK[0.4648,0.5819]" + }, + { + "from": "gpt", + "value": "CLICK[0.5270,0.4270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803],CLICK[0.4648,0.5819],CLICK[0.5270,0.4270]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803],CLICK[0.4648,0.5819],CLICK[0.5270,0.4270],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3283,0.9181]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803],CLICK[0.4648,0.5819],CLICK[0.5270,0.4270],SWIPE[UP],CLICK[0.3283,0.9181]" + }, + { + "from": "gpt", + "value": "TYPE[understood]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803],CLICK[0.4648,0.5819],CLICK[0.5270,0.4270],SWIPE[UP],CLICK[0.3283,0.9181],TYPE[understood]" + }, + { + "from": "gpt", + "value": "CLICK[0.6411,0.0589]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Direct reply the first email with content \"Understood\".\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3128,0.5803],CLICK[0.4648,0.5819],CLICK[0.5270,0.4270],SWIPE[UP],CLICK[0.3283,0.9181],TYPE[understood],CLICK[0.6411,0.0589]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_5_9b551cfc70cf47f7b6680bb4c56d56fd-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3000,0.5819]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3000,0.5819]" + }, + { + "from": "gpt", + "value": "CLICK[0.3046,0.0697]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3000,0.5819],CLICK[0.3046,0.0697]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3000,0.5819],CLICK[0.3046,0.0697],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4010,0.5487]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3000,0.5819],CLICK[0.3046,0.0697],SWIPE[UP],CLICK[0.4010,0.5487]" + }, + { + "from": "gpt", + "value": "CLICK[0.4625,0.6875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3000,0.5819],CLICK[0.3046,0.0697],SWIPE[UP],CLICK[0.4010,0.5487],CLICK[0.4625,0.6875]" + }, + { + "from": "gpt", + "value": "CLICK[0.5951,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Go through the \"Sent\" folder and delete the fifth sent email.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3000,0.5819],CLICK[0.3046,0.0697],SWIPE[UP],CLICK[0.4010,0.5487],CLICK[0.4625,0.6875],CLICK[0.5951,0.0612]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_7_b2e009ed8d454b7da49eaf41557337eb-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.3069,0.6079]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3069,0.6079]" + }, + { + "from": "gpt", + "value": "CLICK[0.3168,0.0628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3069,0.6079],CLICK[0.3168,0.0628]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3069,0.6079],CLICK[0.3168,0.0628],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3872,0.8003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3069,0.6079],CLICK[0.3168,0.0628],SWIPE[UP],CLICK[0.3872,0.8003]" + }, + { + "from": "gpt", + "value": "CLICK[0.3951,0.3602]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3069,0.6079],CLICK[0.3168,0.0628],SWIPE[UP],CLICK[0.3872,0.8003],CLICK[0.3951,0.3602]" + }, + { + "from": "gpt", + "value": "CLICK[0.5990,0.0674]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Gmail. Erase the first email located in the \"Spam\" folder.\nHistory Actions: SWIPE[LEFT],SWIPE[LEFT],CLICK[0.3069,0.6079],CLICK[0.3168,0.0628],SWIPE[UP],CLICK[0.3872,0.8003],CLICK[0.3951,0.3602],CLICK[0.5990,0.0674]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_20_9_137ff9c706b841fa94b2abb40895695d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6849,0.8707]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707]" + }, + { + "from": "gpt", + "value": "CLICK[0.6747,0.0743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707],CLICK[0.6747,0.0743]" + }, + { + "from": "gpt", + "value": "CLICK[0.6447,0.0612]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707],CLICK[0.6747,0.0743],CLICK[0.6447,0.0612]" + }, + { + "from": "gpt", + "value": "TYPE[Rihanna]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707],CLICK[0.6747,0.0743],CLICK[0.6447,0.0612],TYPE[Rihanna]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707],CLICK[0.6747,0.0743],CLICK[0.6447,0.0612],TYPE[Rihanna],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.4056,0.3135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707],CLICK[0.6747,0.0743],CLICK[0.6447,0.0612],TYPE[Rihanna],PRESS_ENTER,CLICK[0.4056,0.3135]" + }, + { + "from": "gpt", + "value": "CLICK[0.4355,0.8368]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Rihanna\" and play the fifth song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6849,0.8707],CLICK[0.6747,0.0743],CLICK[0.6447,0.0612],TYPE[Rihanna],PRESS_ENTER,CLICK[0.4056,0.3135],CLICK[0.4355,0.8368]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_35_bbcd9109b86f47e88f2ee8a3deeea238-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6687,0.8546]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8546]" + }, + { + "from": "gpt", + "value": "CLICK[0.6470,0.0589]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8546],CLICK[0.6470,0.0589]" + }, + { + "from": "gpt", + "value": "TYPE[Maroon 5]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8546],CLICK[0.6470,0.0589],TYPE[Maroon 5]" + }, + { + "from": "gpt", + "value": "CLICK[0.4938,0.1082]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8546],CLICK[0.6470,0.0589],TYPE[Maroon 5],CLICK[0.4938,0.1082]" + }, + { + "from": "gpt", + "value": "CLICK[0.3368,0.3128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8546],CLICK[0.6470,0.0589],TYPE[Maroon 5],CLICK[0.4938,0.1082],CLICK[0.3368,0.3128]" + }, + { + "from": "gpt", + "value": "CLICK[0.4470,0.7273]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8546],CLICK[0.6470,0.0589],TYPE[Maroon 5],CLICK[0.4938,0.1082],CLICK[0.3368,0.3128],CLICK[0.4470,0.7273]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_36_9ac093bde64f4148861687a5d0347783-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6711,0.8655]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655],SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6095,0.5188]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[LEFT],CLICK[0.6095,0.5188]" + }, + { + "from": "gpt", + "value": "CLICK[0.4263,0.5757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fourth song from the list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6711,0.8655],SWIPE[UP],SWIPE[UP],SWIPE[UP],SWIPE[LEFT],CLICK[0.6095,0.5188],CLICK[0.4263,0.5757]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_37_e25c5fd0875c448fbc5594ac2617f3ad-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.8678]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.8678]" + }, + { + "from": "gpt", + "value": "CLICK[0.6434,0.0559]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.8678],CLICK[0.6434,0.0559]" + }, + { + "from": "gpt", + "value": "TYPE[Workout Beats]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.8678],CLICK[0.6434,0.0559],TYPE[Workout Beats]" + }, + { + "from": "gpt", + "value": "CLICK[0.4839,0.1089]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.8678],CLICK[0.6434,0.0559],TYPE[Workout Beats],CLICK[0.4839,0.1089]" + }, + { + "from": "gpt", + "value": "CLICK[0.4487,0.3036]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.8678],CLICK[0.6434,0.0559],TYPE[Workout Beats],CLICK[0.4839,0.1089],CLICK[0.4487,0.3036]" + }, + { + "from": "gpt", + "value": "CLICK[0.4171,0.5987]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Workout Beats\" and play the first song from the playlist.\nHistory Actions: SWIPE[LEFT],SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.8678],CLICK[0.6434,0.0559],TYPE[Workout Beats],CLICK[0.4839,0.1089],CLICK[0.4487,0.3036],CLICK[0.4171,0.5987]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_38_966763c273134349a7de19222f801390-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6655,0.8447]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8447]" + }, + { + "from": "gpt", + "value": "CLICK[0.5039,0.9727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8447],CLICK[0.5039,0.9727]" + }, + { + "from": "gpt", + "value": "CLICK[0.4095,0.4638]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8447],CLICK[0.5039,0.9727],CLICK[0.4095,0.4638]" + }, + { + "from": "gpt", + "value": "CLICK[0.4217,0.5372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Explore\". Choose \"New albums & singles\". Play the first song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8447],CLICK[0.5039,0.9727],CLICK[0.4095,0.4638],CLICK[0.4217,0.5372]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_39_2fe33f242e364ca7b10a4cdde477cff7-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6740,0.8701]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701]" + }, + { + "from": "gpt", + "value": "CLICK[0.6424,0.0599]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701],CLICK[0.6424,0.0599]" + }, + { + "from": "gpt", + "value": "TYPE[Divide]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701],CLICK[0.6424,0.0599],TYPE[Divide]" + }, + { + "from": "gpt", + "value": "CLICK[0.3941,0.1095]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701],CLICK[0.6424,0.0599],TYPE[Divide],CLICK[0.3941,0.1095]" + }, + { + "from": "gpt", + "value": "CLICK[0.4089,0.3095]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701],CLICK[0.6424,0.0599],TYPE[Divide],CLICK[0.3941,0.1095],CLICK[0.4089,0.3095]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701],CLICK[0.6424,0.0599],TYPE[Divide],CLICK[0.3941,0.1095],CLICK[0.4089,0.3095],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4141,0.4224]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the album \"Divide\" by \"Ed Sheeran\" and play the song \"Perfect\".\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6740,0.8701],CLICK[0.6424,0.0599],TYPE[Divide],CLICK[0.3941,0.1095],CLICK[0.4089,0.3095],SWIPE[UP],CLICK[0.4141,0.4224]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_41_869cfc179a5c418a9095fb7e07c54c4a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6862,0.8793]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6862,0.8793]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.0543]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6862,0.8793],CLICK[0.6503,0.0543]" + }, + { + "from": "gpt", + "value": "TYPE[Maroon 5]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6862,0.8793],CLICK[0.6503,0.0543],TYPE[Maroon 5]" + }, + { + "from": "gpt", + "value": "CLICK[0.4194,0.1128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6862,0.8793],CLICK[0.6503,0.0543],TYPE[Maroon 5],CLICK[0.4194,0.1128]" + }, + { + "from": "gpt", + "value": "CLICK[0.3352,0.3174]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6862,0.8793],CLICK[0.6503,0.0543],TYPE[Maroon 5],CLICK[0.4194,0.1128],CLICK[0.3352,0.3174]" + }, + { + "from": "gpt", + "value": "CLICK[0.4809,0.7382]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Maroon 5\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6862,0.8793],CLICK[0.6503,0.0543],TYPE[Maroon 5],CLICK[0.4194,0.1128],CLICK[0.3352,0.3174],CLICK[0.4809,0.7382]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_42_6701a86a97504767be805cdd6c9ea464-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6984,0.8539]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6984,0.8539]" + }, + { + "from": "gpt", + "value": "CLICK[0.6441,0.0553]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6984,0.8539],CLICK[0.6441,0.0553]" + }, + { + "from": "gpt", + "value": "TYPE[Ariana Grande]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6984,0.8539],CLICK[0.6441,0.0553],TYPE[Ariana Grande]" + }, + { + "from": "gpt", + "value": "CLICK[0.4464,0.1082]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6984,0.8539],CLICK[0.6441,0.0553],TYPE[Ariana Grande],CLICK[0.4464,0.1082]" + }, + { + "from": "gpt", + "value": "CLICK[0.3345,0.3220]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6984,0.8539],CLICK[0.6441,0.0553],TYPE[Ariana Grande],CLICK[0.4464,0.1082],CLICK[0.3345,0.3220]" + }, + { + "from": "gpt", + "value": "CLICK[0.4010,0.8086]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Ariana Grande\" and play the second song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6984,0.8539],CLICK[0.6441,0.0553],TYPE[Ariana Grande],CLICK[0.4464,0.1082],CLICK[0.3345,0.3220],CLICK[0.4010,0.8086]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_43_827224fb8d9241929dd62c761af93d0b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6694,0.8493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493]" + }, + { + "from": "gpt", + "value": "CLICK[0.6595,0.0559]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493],CLICK[0.6595,0.0559]" + }, + { + "from": "gpt", + "value": "TYPE[Calm Vibes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493],CLICK[0.6595,0.0559],TYPE[Calm Vibes]" + }, + { + "from": "gpt", + "value": "CLICK[0.4730,0.1118]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493],CLICK[0.6595,0.0559],TYPE[Calm Vibes],CLICK[0.4730,0.1118]" + }, + { + "from": "gpt", + "value": "CLICK[0.4388,0.3020]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493],CLICK[0.6595,0.0559],TYPE[Calm Vibes],CLICK[0.4730,0.1118],CLICK[0.4388,0.3020]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493],CLICK[0.6595,0.0559],TYPE[Calm Vibes],CLICK[0.4730,0.1118],CLICK[0.4388,0.3020],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4102,0.5727]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Calm Vibes\" and play the third song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6694,0.8493],CLICK[0.6595,0.0559],TYPE[Calm Vibes],CLICK[0.4730,0.1118],CLICK[0.4388,0.3020],SWIPE[UP],CLICK[0.4102,0.5727]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_44_be6ff72aeda242079cb8096d8c2e6516-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6839,0.8724]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6839,0.8724]" + }, + { + "from": "gpt", + "value": "CLICK[0.5875,0.9849]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6839,0.8724],CLICK[0.5875,0.9849]" + }, + { + "from": "gpt", + "value": "CLICK[0.5911,0.0484]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6839,0.8724],CLICK[0.5875,0.9849],CLICK[0.5911,0.0484]" + }, + { + "from": "gpt", + "value": "CLICK[0.4319,0.4033]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6839,0.8724],CLICK[0.5875,0.9849],CLICK[0.5911,0.0484],CLICK[0.4319,0.4033]" + }, + { + "from": "gpt", + "value": "CLICK[0.4250,0.5105]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music audio to the medium bitrate.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6839,0.8724],CLICK[0.5875,0.9849],CLICK[0.5911,0.0484],CLICK[0.4319,0.4033],CLICK[0.4250,0.5105]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_45_31eb02571747461aa65866b3b194e9e6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6701,0.8632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8632]" + }, + { + "from": "gpt", + "value": "CLICK[0.6395,0.0536]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8632],CLICK[0.6395,0.0536]" + }, + { + "from": "gpt", + "value": "TYPE[Hip Hop Dance Workout]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8632],CLICK[0.6395,0.0536],TYPE[Hip Hop Dance Workout]" + }, + { + "from": "gpt", + "value": "CLICK[0.4112,0.1174]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8632],CLICK[0.6395,0.0536],TYPE[Hip Hop Dance Workout],CLICK[0.4112,0.1174]" + }, + { + "from": "gpt", + "value": "CLICK[0.4089,0.1678]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8632],CLICK[0.6395,0.0536],TYPE[Hip Hop Dance Workout],CLICK[0.4112,0.1174],CLICK[0.4089,0.1678]" + }, + { + "from": "gpt", + "value": "CLICK[0.4556,0.2362]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the video \"Hip Hop Dance Workout\" and watch the second video from the search results.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8632],CLICK[0.6395,0.0536],TYPE[Hip Hop Dance Workout],CLICK[0.4112,0.1174],CLICK[0.4089,0.1678],CLICK[0.4556,0.2362]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_47_59c3a15f88af4cd6af1f64891bc65909-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6757,0.8684]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6757,0.8684]" + }, + { + "from": "gpt", + "value": "CLICK[0.6470,0.0589]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6757,0.8684],CLICK[0.6470,0.0589]" + }, + { + "from": "gpt", + "value": "TYPE[Beyonce]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6757,0.8684],CLICK[0.6470,0.0589],TYPE[Beyonce]" + }, + { + "from": "gpt", + "value": "CLICK[0.4171,0.1135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6757,0.8684],CLICK[0.6470,0.0589],TYPE[Beyonce],CLICK[0.4171,0.1135]" + }, + { + "from": "gpt", + "value": "CLICK[0.3322,0.3043]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6757,0.8684],CLICK[0.6470,0.0589],TYPE[Beyonce],CLICK[0.4171,0.1135],CLICK[0.3322,0.3043]" + }, + { + "from": "gpt", + "value": "CLICK[0.4424,0.7273]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the artist \"Beyonce\" and play the first song from her top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6757,0.8684],CLICK[0.6470,0.0589],TYPE[Beyonce],CLICK[0.4171,0.1135],CLICK[0.3322,0.3043],CLICK[0.4424,0.7273]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_48_03cb1f64bc304d2eb0df00b407279473-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6687,0.8493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8493]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8493],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8493],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3882,0.5428]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8493],SWIPE[UP],SWIPE[UP],CLICK[0.3882,0.5428]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8493],SWIPE[UP],SWIPE[UP],CLICK[0.3882,0.5428],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4158,0.5441]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Home\". Choose \"Recommend List\". Play the fifth song from the first album.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6687,0.8493],SWIPE[UP],SWIPE[UP],CLICK[0.3882,0.5428],SWIPE[UP],CLICK[0.4158,0.5441]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_50_4ea63ea48fa94ae99186ec39fc93dd01-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6809,0.8655]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6809,0.8655]" + }, + { + "from": "gpt", + "value": "CLICK[0.6487,0.0474]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6809,0.8655],CLICK[0.6487,0.0474]" + }, + { + "from": "gpt", + "value": "TYPE[Green Day]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6809,0.8655],CLICK[0.6487,0.0474],TYPE[Green Day]" + }, + { + "from": "gpt", + "value": "CLICK[0.4678,0.1141]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6809,0.8655],CLICK[0.6487,0.0474],TYPE[Green Day],CLICK[0.4678,0.1141]" + }, + { + "from": "gpt", + "value": "CLICK[0.4434,0.3049]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6809,0.8655],CLICK[0.6487,0.0474],TYPE[Green Day],CLICK[0.4678,0.1141],CLICK[0.4434,0.3049]" + }, + { + "from": "gpt", + "value": "CLICK[0.3957,0.7266]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the band \"Green Day\" and play the first song from their top hits list.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6809,0.8655],CLICK[0.6487,0.0474],TYPE[Green Day],CLICK[0.4678,0.1141],CLICK[0.4434,0.3049],CLICK[0.3957,0.7266]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_51_0aaaf4f065e04cd1b3acde7f829b782e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6655,0.8622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8622]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8622],CLICK[0.6503,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[Chill Vibes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8622],CLICK[0.6503,0.0530],TYPE[Chill Vibes]" + }, + { + "from": "gpt", + "value": "CLICK[0.4118,0.1095]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8622],CLICK[0.6503,0.0530],TYPE[Chill Vibes],CLICK[0.4118,0.1095]" + }, + { + "from": "gpt", + "value": "CLICK[0.3691,0.6155]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Search the playlist \"Chill Vibes\" and play the first song from the playlist.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6655,0.8622],CLICK[0.6503,0.0530],TYPE[Chill Vibes],CLICK[0.4118,0.1095],CLICK[0.3691,0.6155]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_52_ab014e22cfb14bba9d359faa5937c15c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6701,0.8747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8747]" + }, + { + "from": "gpt", + "value": "CLICK[0.5852,0.9773]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8747],CLICK[0.5852,0.9773]" + }, + { + "from": "gpt", + "value": "CLICK[0.6020,0.0566]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8747],CLICK[0.5852,0.9773],CLICK[0.6020,0.0566]" + }, + { + "from": "gpt", + "value": "CLICK[0.4095,0.4875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8747],CLICK[0.5852,0.9773],CLICK[0.6020,0.0566],CLICK[0.4095,0.4875]" + }, + { + "from": "gpt", + "value": "CLICK[0.4572,0.5395]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Youtube Music. Open \"Settings\". Change the quality of the music video to 360p.\nHistory Actions: SWIPE[UP],SWIPE[UP],SWIPE[UP],CLICK[0.6701,0.8747],CLICK[0.5852,0.9773],CLICK[0.6020,0.0566],CLICK[0.4095,0.4875],CLICK[0.4572,0.5395]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_5_6_22_53_a96d410b641d4c6a883c41601365f837-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3166,0.8047]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047]" + }, + { + "from": "gpt", + "value": "CLICK[0.2845,0.9767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767]" + }, + { + "from": "gpt", + "value": "PRESS_BACK" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK" + }, + { + "from": "gpt", + "value": "CLICK[0.3551,0.9760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760]" + }, + { + "from": "gpt", + "value": "TYPE[Work]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work]" + }, + { + "from": "gpt", + "value": "CLICK[0.6473,0.6311]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311]" + }, + { + "from": "gpt", + "value": "CLICK[0.6926,0.9459]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459]" + }, + { + "from": "gpt", + "value": "TYPE[Prepare for the meeting]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting]" + }, + { + "from": "gpt", + "value": "CLICK[0.4713,0.9713]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713]" + }, + { + "from": "gpt", + "value": "CLICK[0.5189,0.9270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713],CLICK[0.5189,0.9270]" + }, + { + "from": "gpt", + "value": "CLICK[0.6010,0.2622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713],CLICK[0.5189,0.9270],CLICK[0.6010,0.2622]" + }, + { + "from": "gpt", + "value": "CLICK[0.6051,0.3557]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713],CLICK[0.5189,0.9270],CLICK[0.6010,0.2622],CLICK[0.6051,0.3557]" + }, + { + "from": "gpt", + "value": "CLICK[0.5858,0.5601]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713],CLICK[0.5189,0.9270],CLICK[0.6010,0.2622],CLICK[0.6051,0.3557],CLICK[0.5858,0.5601]" + }, + { + "from": "gpt", + "value": "CLICK[0.6686,0.7541]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713],CLICK[0.5189,0.9270],CLICK[0.6010,0.2622],CLICK[0.6051,0.3557],CLICK[0.5858,0.5601],CLICK[0.6686,0.7541]" + }, + { + "from": "gpt", + "value": "CLICK[0.7064,0.9270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Work'. Then create a new task to 'Prepare for the meeting' and set the reminder time at today 4PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8047],CLICK[0.2845,0.9767],PRESS_BACK,CLICK[0.3551,0.9760],TYPE[Work],CLICK[0.6473,0.6311],CLICK[0.6926,0.9459],TYPE[Prepare for the meeting],CLICK[0.4713,0.9713],CLICK[0.5189,0.9270],CLICK[0.6010,0.2622],CLICK[0.6051,0.3557],CLICK[0.5858,0.5601],CLICK[0.6686,0.7541],CLICK[0.7064,0.9270]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_50_47c67579498849b9b1220f3303530e7c-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3091,0.7409]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.7409]" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.9584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.7409],CLICK[0.7071,0.9584]" + }, + { + "from": "gpt", + "value": "TYPE[Send materials to the boss]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.7409],CLICK[0.7071,0.9584],TYPE[Send materials to the boss]" + }, + { + "from": "gpt", + "value": "CLICK[0.3466,0.9676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.7409],CLICK[0.7071,0.9584],TYPE[Send materials to the boss],CLICK[0.3466,0.9676]" + }, + { + "from": "gpt", + "value": "CLICK[0.3919,0.8338]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.7409],CLICK[0.7071,0.9584],TYPE[Send materials to the boss],CLICK[0.3466,0.9676],CLICK[0.3919,0.8338]" + }, + { + "from": "gpt", + "value": "CLICK[0.7095,0.9277]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'Send materials to the boss' in the 'Work' list and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.7409],CLICK[0.7071,0.9584],TYPE[Send materials to the boss],CLICK[0.3466,0.9676],CLICK[0.3919,0.8338],CLICK[0.7095,0.9277]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_55_af14a467b098422aa000b8009d161067-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3091,0.8169]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169]" + }, + { + "from": "gpt", + "value": "CLICK[0.4905,0.3780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.9584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584]" + }, + { + "from": "gpt", + "value": "TYPE[Weekly report]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report]" + }, + { + "from": "gpt", + "value": "CLICK[0.5858,0.9676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report],CLICK[0.5858,0.9676]" + }, + { + "from": "gpt", + "value": "CLICK[0.6155,0.9291]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report],CLICK[0.5858,0.9676],CLICK[0.6155,0.9291]" + }, + { + "from": "gpt", + "value": "CLICK[0.4490,0.5034]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report],CLICK[0.5858,0.9676],CLICK[0.6155,0.9291],CLICK[0.4490,0.5034]" + }, + { + "from": "gpt", + "value": "CLICK[0.5625,0.5010]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report],CLICK[0.5858,0.9676],CLICK[0.6155,0.9291],CLICK[0.4490,0.5034],CLICK[0.5625,0.5010]" + }, + { + "from": "gpt", + "value": "CLICK[0.6419,0.6385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report],CLICK[0.5858,0.9676],CLICK[0.6155,0.9291],CLICK[0.4490,0.5034],CLICK[0.5625,0.5010],CLICK[0.6419,0.6385]" + }, + { + "from": "gpt", + "value": "CLICK[0.7108,0.9253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Weekly report' and repeat the task weekly on Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8169],CLICK[0.4905,0.3780],CLICK[0.6993,0.9584],TYPE[Weekly report],CLICK[0.5858,0.9676],CLICK[0.6155,0.9291],CLICK[0.4490,0.5034],CLICK[0.5625,0.5010],CLICK[0.6419,0.6385],CLICK[0.7108,0.9253]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_14_59_2105397782c5492e80969cf2fba1fa08-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check My Day and what is the next task to do?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_11_bdbdd6e0056a4b9798e1a2560fc02497-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check My Day and what is the next task to do?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_11_bdbdd6e0056a4b9798e1a2560fc02497-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check My Day and what is the next task to do?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3176,0.8470]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_11_bdbdd6e0056a4b9798e1a2560fc02497-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check My Day and what is the next task to do?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3176,0.8470]" + }, + { + "from": "gpt", + "value": "CLICK[0.2828,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_11_bdbdd6e0056a4b9798e1a2560fc02497-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check My Day and what is the next task to do?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3176,0.8470],CLICK[0.2828,0.0530]" + }, + { + "from": "gpt", + "value": "CLICK[0.4412,0.1030]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_11_bdbdd6e0056a4b9798e1a2560fc02497-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check My Day and what is the next task to do?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3176,0.8470],CLICK[0.2828,0.0530],CLICK[0.4412,0.1030]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_11_bdbdd6e0056a4b9798e1a2560fc02497-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3206,0.8017]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017]" + }, + { + "from": "gpt", + "value": "CLICK[0.5095,0.4257]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257]" + }, + { + "from": "gpt", + "value": "CLICK[0.7078,0.9530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530]" + }, + { + "from": "gpt", + "value": "TYPE[Call the client]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client]" + }, + { + "from": "gpt", + "value": "CLICK[0.4598,0.9682]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client],CLICK[0.4598,0.9682]" + }, + { + "from": "gpt", + "value": "CLICK[0.5436,0.9230]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client],CLICK[0.4598,0.9682],CLICK[0.5436,0.9230]" + }, + { + "from": "gpt", + "value": "CLICK[0.6179,0.2605]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client],CLICK[0.4598,0.9682],CLICK[0.5436,0.9230],CLICK[0.6179,0.2605]" + }, + { + "from": "gpt", + "value": "CLICK[0.5973,0.5233]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client],CLICK[0.4598,0.9682],CLICK[0.5436,0.9230],CLICK[0.6179,0.2605],CLICK[0.5973,0.5233]" + }, + { + "from": "gpt", + "value": "CLICK[0.6655,0.7584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client],CLICK[0.4598,0.9682],CLICK[0.5436,0.9230],CLICK[0.6179,0.2605],CLICK[0.5973,0.5233],CLICK[0.6655,0.7584]" + }, + { + "from": "gpt", + "value": "CLICK[0.7034,0.9277]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Call the client' and set the reminder time at today 3PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.8017],CLICK[0.5095,0.4257],CLICK[0.7078,0.9530],TYPE[Call the client],CLICK[0.4598,0.9682],CLICK[0.5436,0.9230],CLICK[0.6179,0.2605],CLICK[0.5973,0.5233],CLICK[0.6655,0.7584],CLICK[0.7034,0.9277]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_13_e7bdbc94d90f405f83f0f9e16da9d70a-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3128,0.8301]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301]" + }, + { + "from": "gpt", + "value": "CLICK[0.2892,0.0608]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608]" + }, + { + "from": "gpt", + "value": "CLICK[0.5044,0.4726]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726]" + }, + { + "from": "gpt", + "value": "CLICK[0.6949,0.9578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578]" + }, + { + "from": "gpt", + "value": "TYPE[Prepare for the exam]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578],TYPE[Prepare for the exam]" + }, + { + "from": "gpt", + "value": "CLICK[0.3483,0.9682]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578],TYPE[Prepare for the exam],CLICK[0.3483,0.9682]" + }, + { + "from": "gpt", + "value": "CLICK[0.3858,0.9253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578],TYPE[Prepare for the exam],CLICK[0.3483,0.9682],CLICK[0.3858,0.9253]" + }, + { + "from": "gpt", + "value": "CLICK[0.5932,0.5963]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578],TYPE[Prepare for the exam],CLICK[0.3483,0.9682],CLICK[0.3858,0.9253],CLICK[0.5932,0.5963]" + }, + { + "from": "gpt", + "value": "CLICK[0.6341,0.7301]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578],TYPE[Prepare for the exam],CLICK[0.3483,0.9682],CLICK[0.3858,0.9253],CLICK[0.5932,0.5963],CLICK[0.6341,0.7301]" + }, + { + "from": "gpt", + "value": "CLICK[0.7047,0.9236]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Prepare for the exam' and set the due date to this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3128,0.8301],CLICK[0.2892,0.0608],CLICK[0.5044,0.4726],CLICK[0.6949,0.9578],TYPE[Prepare for the exam],CLICK[0.3483,0.9682],CLICK[0.3858,0.9253],CLICK[0.5932,0.5963],CLICK[0.6341,0.7301],CLICK[0.7047,0.9236]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_16_8a141580dfda4e18ba78fceea520108b-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3230,0.8622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3230,0.8622]" + }, + { + "from": "gpt", + "value": "CLICK[0.7216,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3230,0.8622],CLICK[0.7216,0.0530]" + }, + { + "from": "gpt", + "value": "TYPE[Read linear algebra textbook chapter 2]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3230,0.8622],CLICK[0.7216,0.0530],TYPE[Read linear algebra textbook chapter 2]" + }, + { + "from": "gpt", + "value": "CLICK[0.5128,0.1115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3230,0.8622],CLICK[0.7216,0.0530],TYPE[Read linear algebra textbook chapter 2],CLICK[0.5128,0.1115]" + }, + { + "from": "gpt", + "value": "CLICK[0.2868,0.1162]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read linear algebra textbook chapter 2' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3230,0.8622],CLICK[0.7216,0.0530],TYPE[Read linear algebra textbook chapter 2],CLICK[0.5128,0.1115],CLICK[0.2868,0.1162]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_17_a917e0cc1a2049dcbb61690f274a55df-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3135,0.8517]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8517]" + }, + { + "from": "gpt", + "value": "CLICK[0.7186,0.0561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8517],CLICK[0.7186,0.0561]" + }, + { + "from": "gpt", + "value": "TYPE[Prepare for the exam]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8517],CLICK[0.7186,0.0561],TYPE[Prepare for the exam]" + }, + { + "from": "gpt", + "value": "CLICK[0.4980,0.1206]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8517],CLICK[0.7186,0.0561],TYPE[Prepare for the exam],CLICK[0.4980,0.1206]" + }, + { + "from": "gpt", + "value": "CLICK[0.7216,0.9723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8517],CLICK[0.7186,0.0561],TYPE[Prepare for the exam],CLICK[0.4980,0.1206],CLICK[0.7216,0.9723]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.5649]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Prepare for the exam' and delete it.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8517],CLICK[0.7186,0.0561],TYPE[Prepare for the exam],CLICK[0.4980,0.1206],CLICK[0.7216,0.9723],CLICK[0.6632,0.5649]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_18_f986470c39424bb1b96355b9ab5d1609-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3074,0.8608]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608]" + }, + { + "from": "gpt", + "value": "PRESS_BACK" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.2959]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959]" + }, + { + "from": "gpt", + "value": "CLICK[0.6963,0.9544]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544]" + }, + { + "from": "gpt", + "value": "TYPE[grocery shopping]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping]" + }, + { + "from": "gpt", + "value": "CLICK[0.4696,0.9669]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping],CLICK[0.4696,0.9669]" + }, + { + "from": "gpt", + "value": "CLICK[0.5304,0.9260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping],CLICK[0.4696,0.9669],CLICK[0.5304,0.9260]" + }, + { + "from": "gpt", + "value": "CLICK[0.6101,0.2635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping],CLICK[0.4696,0.9669],CLICK[0.5304,0.9260],CLICK[0.6101,0.2635]" + }, + { + "from": "gpt", + "value": "CLICK[0.5973,0.3527]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping],CLICK[0.4696,0.9669],CLICK[0.5304,0.9260],CLICK[0.6101,0.2635],CLICK[0.5973,0.3527]" + }, + { + "from": "gpt", + "value": "CLICK[0.6655,0.7541]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping],CLICK[0.4696,0.9669],CLICK[0.5304,0.9260],CLICK[0.6101,0.2635],CLICK[0.5973,0.3527],CLICK[0.6655,0.7541]" + }, + { + "from": "gpt", + "value": "CLICK[0.7078,0.9270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new common task named 'grocery shopping' and set the reminder time at today 6PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3074,0.8608],PRESS_BACK,CLICK[0.4872,0.2959],CLICK[0.6963,0.9544],TYPE[grocery shopping],CLICK[0.4696,0.9669],CLICK[0.5304,0.9260],CLICK[0.6101,0.2635],CLICK[0.5973,0.3527],CLICK[0.6655,0.7541],CLICK[0.7078,0.9270]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_1_7e8698784f384a139398f7d2c5cbb3e2-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3182,0.8777]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.4220]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220]" + }, + { + "from": "gpt", + "value": "CLICK[0.6919,0.9530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530]" + }, + { + "from": "gpt", + "value": "TYPE[Send the report to the team]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530],TYPE[Send the report to the team]" + }, + { + "from": "gpt", + "value": "CLICK[0.3473,0.9645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530],TYPE[Send the report to the team],CLICK[0.3473,0.9645]" + }, + { + "from": "gpt", + "value": "CLICK[0.3899,0.9230]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530],TYPE[Send the report to the team],CLICK[0.3473,0.9645],CLICK[0.3899,0.9230]" + }, + { + "from": "gpt", + "value": "CLICK[0.4057,0.6395]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530],TYPE[Send the report to the team],CLICK[0.3473,0.9645],CLICK[0.3899,0.9230],CLICK[0.4057,0.6395]" + }, + { + "from": "gpt", + "value": "CLICK[0.6280,0.7307]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530],TYPE[Send the report to the team],CLICK[0.3473,0.9645],CLICK[0.3899,0.9230],CLICK[0.4057,0.6395],CLICK[0.6280,0.7307]" + }, + { + "from": "gpt", + "value": "CLICK[0.7125,0.9216]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the report to the team' and set the due date to next Monday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8777],CLICK[0.4949,0.4220],CLICK[0.6919,0.9530],TYPE[Send the report to the team],CLICK[0.3473,0.9645],CLICK[0.3899,0.9230],CLICK[0.4057,0.6395],CLICK[0.6280,0.7307],CLICK[0.7125,0.9216]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_20_875aaab7cecd4fd68fb7272739516e51-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.8432]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432]" + }, + { + "from": "gpt", + "value": "CLICK[0.2929,0.0507]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507]" + }, + { + "from": "gpt", + "value": "CLICK[0.4956,0.4780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780]" + }, + { + "from": "gpt", + "value": "CLICK[0.7010,0.9568]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568]" + }, + { + "from": "gpt", + "value": "TYPE[Write an essay]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568],TYPE[Write an essay]" + }, + { + "from": "gpt", + "value": "CLICK[0.3459,0.9669]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568],TYPE[Write an essay],CLICK[0.3459,0.9669]" + }, + { + "from": "gpt", + "value": "CLICK[0.3828,0.9270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568],TYPE[Write an essay],CLICK[0.3459,0.9669],CLICK[0.3828,0.9270]" + }, + { + "from": "gpt", + "value": "CLICK[0.4966,0.6372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568],TYPE[Write an essay],CLICK[0.3459,0.9669],CLICK[0.3828,0.9270],CLICK[0.4966,0.6372]" + }, + { + "from": "gpt", + "value": "CLICK[0.6372,0.7324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568],TYPE[Write an essay],CLICK[0.3459,0.9669],CLICK[0.3828,0.9270],CLICK[0.4966,0.6372],CLICK[0.6372,0.7324]" + }, + { + "from": "gpt", + "value": "CLICK[0.7088,0.9152]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Write an essay' and set the due date to next Wednesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8432],CLICK[0.2929,0.0507],CLICK[0.4956,0.4780],CLICK[0.7010,0.9568],TYPE[Write an essay],CLICK[0.3459,0.9669],CLICK[0.3828,0.9270],CLICK[0.4966,0.6372],CLICK[0.6372,0.7324],CLICK[0.7088,0.9152]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_22_52bc5bb02f5b4a4ea96ee38632f448b2-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3115,0.9230]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230]" + }, + { + "from": "gpt", + "value": "CLICK[0.4919,0.4267]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267]" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.9429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429]" + }, + { + "from": "gpt", + "value": "TYPE[Prepare for the presentation]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429],TYPE[Prepare for the presentation]" + }, + { + "from": "gpt", + "value": "CLICK[0.3635,0.9699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429],TYPE[Prepare for the presentation],CLICK[0.3635,0.9699]" + }, + { + "from": "gpt", + "value": "CLICK[0.3628,0.9176]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429],TYPE[Prepare for the presentation],CLICK[0.3635,0.9699],CLICK[0.3628,0.9176]" + }, + { + "from": "gpt", + "value": "CLICK[0.5449,0.6372]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429],TYPE[Prepare for the presentation],CLICK[0.3635,0.9699],CLICK[0.3628,0.9176],CLICK[0.5449,0.6372]" + }, + { + "from": "gpt", + "value": "CLICK[0.6365,0.7318]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429],TYPE[Prepare for the presentation],CLICK[0.3635,0.9699],CLICK[0.3628,0.9176],CLICK[0.5449,0.6372],CLICK[0.6365,0.7318]" + }, + { + "from": "gpt", + "value": "CLICK[0.7149,0.9270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare for the presentation' and set the due date to next Thursday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3115,0.9230],CLICK[0.4919,0.4267],CLICK[0.7071,0.9429],TYPE[Prepare for the presentation],CLICK[0.3635,0.9699],CLICK[0.3628,0.9176],CLICK[0.5449,0.6372],CLICK[0.6365,0.7318],CLICK[0.7149,0.9270]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_23_b656cafedba54d69aac3e867d19b645b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3135,0.8831]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831]" + }, + { + "from": "gpt", + "value": "CLICK[0.2929,0.0514]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514]" + }, + { + "from": "gpt", + "value": "CLICK[0.4703,0.1074]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074]" + }, + { + "from": "gpt", + "value": "CLICK[0.7034,0.9628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628]" + }, + { + "from": "gpt", + "value": "TYPE[Buy a flower for mom]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom]" + }, + { + "from": "gpt", + "value": "CLICK[0.5649,0.9652]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom],CLICK[0.5649,0.9652]" + }, + { + "from": "gpt", + "value": "CLICK[0.6395,0.9307]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom],CLICK[0.5649,0.9652],CLICK[0.6395,0.9307]" + }, + { + "from": "gpt", + "value": "CLICK[0.6155,0.2635]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom],CLICK[0.5649,0.9652],CLICK[0.6395,0.9307],CLICK[0.6155,0.2635]" + }, + { + "from": "gpt", + "value": "CLICK[0.4544,0.5973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom],CLICK[0.5649,0.9652],CLICK[0.6395,0.9307],CLICK[0.6155,0.2635],CLICK[0.4544,0.5973]" + }, + { + "from": "gpt", + "value": "CLICK[0.6611,0.7584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom],CLICK[0.5649,0.9652],CLICK[0.6395,0.9307],CLICK[0.6155,0.2635],CLICK[0.4544,0.5973],CLICK[0.6611,0.7584]" + }, + { + "from": "gpt", + "value": "CLICK[0.7057,0.9199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Open My Day and add a new task 'Buy a flower for mom' and set the reminder time at today 7PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8831],CLICK[0.2929,0.0514],CLICK[0.4703,0.1074],CLICK[0.7034,0.9628],TYPE[Buy a flower for mom],CLICK[0.5649,0.9652],CLICK[0.6395,0.9307],CLICK[0.6155,0.2635],CLICK[0.4544,0.5973],CLICK[0.6611,0.7584],CLICK[0.7057,0.9199]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_25_b8e3d9b722ab41eaa508de4913ff4192-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3166,0.8199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199]" + }, + { + "from": "gpt", + "value": "CLICK[0.2828,0.0578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578]" + }, + { + "from": "gpt", + "value": "CLICK[0.4858,0.4764]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764]" + }, + { + "from": "gpt", + "value": "CLICK[0.7047,0.9608]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608]" + }, + { + "from": "gpt", + "value": "TYPE[Practice coding]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608],TYPE[Practice coding]" + }, + { + "from": "gpt", + "value": "CLICK[0.5743,0.9693]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608],TYPE[Practice coding],CLICK[0.5743,0.9693]" + }, + { + "from": "gpt", + "value": "CLICK[0.6142,0.6855]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608],TYPE[Practice coding],CLICK[0.5743,0.9693],CLICK[0.6142,0.6855]" + }, + { + "from": "gpt", + "value": "CLICK[0.7017,0.9193]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608],TYPE[Practice coding],CLICK[0.5743,0.9693],CLICK[0.6142,0.6855],CLICK[0.7017,0.9193]" + }, + { + "from": "gpt", + "value": "PRESS_BACK" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608],TYPE[Practice coding],CLICK[0.5743,0.9693],CLICK[0.6142,0.6855],CLICK[0.7017,0.9193],PRESS_BACK" + }, + { + "from": "gpt", + "value": "CLICK[0.7088,0.1199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Practice coding' and set daily repeat and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8199],CLICK[0.2828,0.0578],CLICK[0.4858,0.4764],CLICK[0.7047,0.9608],TYPE[Practice coding],CLICK[0.5743,0.9693],CLICK[0.6142,0.6855],CLICK[0.7017,0.9193],PRESS_BACK,CLICK[0.7088,0.1199]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_27_4b71937ffc19456d97ab948affec5fac-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3206,0.9399]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399]" + }, + { + "from": "gpt", + "value": "CLICK[0.2851,0.0547]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547]" + }, + { + "from": "gpt", + "value": "CLICK[0.4936,0.4220]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220]" + }, + { + "from": "gpt", + "value": "CLICK[0.6956,0.9578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578]" + }, + { + "from": "gpt", + "value": "TYPE[fill the form]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form]" + }, + { + "from": "gpt", + "value": "CLICK[0.4618,0.9676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676]" + }, + { + "from": "gpt", + "value": "CLICK[0.5466,0.9321]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321]" + }, + { + "from": "gpt", + "value": "CLICK[0.6142,0.2611]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321],CLICK[0.6142,0.2611]" + }, + { + "from": "gpt", + "value": "CLICK[0.5466,0.5973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321],CLICK[0.6142,0.2611],CLICK[0.5466,0.5973]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.7571]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321],CLICK[0.6142,0.2611],CLICK[0.5466,0.5973],CLICK[0.6632,0.7571]" + }, + { + "from": "gpt", + "value": "CLICK[0.7017,0.9236]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321],CLICK[0.6142,0.2611],CLICK[0.5466,0.5973],CLICK[0.6632,0.7571],CLICK[0.7017,0.9236]" + }, + { + "from": "gpt", + "value": "PRESS_BACK" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321],CLICK[0.6142,0.2611],CLICK[0.5466,0.5973],CLICK[0.6632,0.7571],CLICK[0.7017,0.9236],PRESS_BACK" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.1199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'fill the form' with reminder time at 5PM today and set it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3206,0.9399],CLICK[0.2851,0.0547],CLICK[0.4936,0.4220],CLICK[0.6956,0.9578],TYPE[fill the form],CLICK[0.4618,0.9676],CLICK[0.5466,0.9321],CLICK[0.6142,0.2611],CLICK[0.5466,0.5973],CLICK[0.6632,0.7571],CLICK[0.7017,0.9236],PRESS_BACK,CLICK[0.7071,0.1199]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_29_4dae6e778f2548a1b16ed6264ec68023-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3213,0.8740]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740]" + }, + { + "from": "gpt", + "value": "CLICK[0.4889,0.2997]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997]" + }, + { + "from": "gpt", + "value": "CLICK[0.7017,0.9584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584]" + }, + { + "from": "gpt", + "value": "TYPE[lunch at subway]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway]" + }, + { + "from": "gpt", + "value": "CLICK[0.4726,0.9713]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway],CLICK[0.4726,0.9713]" + }, + { + "from": "gpt", + "value": "CLICK[0.5564,0.9284]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway],CLICK[0.4726,0.9713],CLICK[0.5564,0.9284]" + }, + { + "from": "gpt", + "value": "CLICK[0.6057,0.2628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway],CLICK[0.4726,0.9713],CLICK[0.5564,0.9284],CLICK[0.6057,0.2628]" + }, + { + "from": "gpt", + "value": "CLICK[0.5003,0.4166]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway],CLICK[0.4726,0.9713],CLICK[0.5564,0.9284],CLICK[0.6057,0.2628],CLICK[0.5003,0.4166]" + }, + { + "from": "gpt", + "value": "CLICK[0.6733,0.7584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway],CLICK[0.4726,0.9713],CLICK[0.5564,0.9284],CLICK[0.6057,0.2628],CLICK[0.5003,0.4166],CLICK[0.6733,0.7584]" + }, + { + "from": "gpt", + "value": "CLICK[0.7132,0.9230]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task named 'lunch at subway' and set the reminder time at 12PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3213,0.8740],CLICK[0.4889,0.2997],CLICK[0.7017,0.9584],TYPE[lunch at subway],CLICK[0.4726,0.9713],CLICK[0.5564,0.9284],CLICK[0.6057,0.2628],CLICK[0.5003,0.4166],CLICK[0.6733,0.7584],CLICK[0.7132,0.9230]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_31_0e2490f04ef946ad911bb99127cf5c94-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3145,0.9084]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9084]" + }, + { + "from": "gpt", + "value": "CLICK[0.4574,0.2061]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9084],CLICK[0.4574,0.2061]" + }, + { + "from": "gpt", + "value": "CLICK[0.3345,0.1821]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9084],CLICK[0.4574,0.2061],CLICK[0.3345,0.1821]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.3652]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Check Planned tab and see the tasks for this week.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9084],CLICK[0.4574,0.2061],CLICK[0.3345,0.1821],CLICK[0.4074,0.3652]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_33_112d634d0281404f954594e213596950-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3122,0.8547]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547]" + }, + { + "from": "gpt", + "value": "CLICK[0.5010,0.4274]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547],CLICK[0.5010,0.4274]" + }, + { + "from": "gpt", + "value": "CLICK[0.6902,0.9507]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547],CLICK[0.5010,0.4274],CLICK[0.6902,0.9507]" + }, + { + "from": "gpt", + "value": "TYPE[Send the email to the team]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547],CLICK[0.5010,0.4274],CLICK[0.6902,0.9507],TYPE[Send the email to the team]" + }, + { + "from": "gpt", + "value": "CLICK[0.3304,0.9662]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547],CLICK[0.5010,0.4274],CLICK[0.6902,0.9507],TYPE[Send the email to the team],CLICK[0.3304,0.9662]" + }, + { + "from": "gpt", + "value": "CLICK[0.3889,0.8338]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547],CLICK[0.5010,0.4274],CLICK[0.6902,0.9507],TYPE[Send the email to the team],CLICK[0.3304,0.9662],CLICK[0.3889,0.8338]" + }, + { + "from": "gpt", + "value": "CLICK[0.7064,0.9206]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Send the email to the team' and set the due date to tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3122,0.8547],CLICK[0.5010,0.4274],CLICK[0.6902,0.9507],TYPE[Send the email to the team],CLICK[0.3304,0.9662],CLICK[0.3889,0.8338],CLICK[0.7064,0.9206]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_34_843b8872edb24a7384f384a18c9448eb-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3145,0.9007]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007]" + }, + { + "from": "gpt", + "value": "CLICK[0.4990,0.4743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743]" + }, + { + "from": "gpt", + "value": "CLICK[0.7057,0.9598]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598]" + }, + { + "from": "gpt", + "value": "TYPE[Read the research paper]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper]" + }, + { + "from": "gpt", + "value": "CLICK[0.4605,0.9713]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713]" + }, + { + "from": "gpt", + "value": "CLICK[0.5473,0.9284]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713],CLICK[0.5473,0.9284]" + }, + { + "from": "gpt", + "value": "CLICK[0.5135,0.6270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713],CLICK[0.5473,0.9284],CLICK[0.5135,0.6270]" + }, + { + "from": "gpt", + "value": "CLICK[0.4267,0.4764]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713],CLICK[0.5473,0.9284],CLICK[0.5135,0.6270],CLICK[0.4267,0.4764]" + }, + { + "from": "gpt", + "value": "CLICK[0.5020,0.6142]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713],CLICK[0.5473,0.9284],CLICK[0.5135,0.6270],CLICK[0.4267,0.4764],CLICK[0.5020,0.6142]" + }, + { + "from": "gpt", + "value": "CLICK[0.6666,0.7561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713],CLICK[0.5473,0.9284],CLICK[0.5135,0.6270],CLICK[0.4267,0.4764],CLICK[0.5020,0.6142],CLICK[0.6666,0.7561]" + }, + { + "from": "gpt", + "value": "CLICK[0.7108,0.9284]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read the research paper' and set the reminder time at 10:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.9007],CLICK[0.4990,0.4743],CLICK[0.7057,0.9598],TYPE[Read the research paper],CLICK[0.4605,0.9713],CLICK[0.5473,0.9284],CLICK[0.5135,0.6270],CLICK[0.4267,0.4764],CLICK[0.5020,0.6142],CLICK[0.6666,0.7561],CLICK[0.7108,0.9284]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_36_50ab5f7b326e4b1a876289872e593fff-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.8439]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8439]" + }, + { + "from": "gpt", + "value": "CLICK[0.7142,0.0561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8439],CLICK[0.7142,0.0561]" + }, + { + "from": "gpt", + "value": "TYPE[Read the research paper]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8439],CLICK[0.7142,0.0561],TYPE[Read the research paper]" + }, + { + "from": "gpt", + "value": "CLICK[0.7162,0.1152]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Read the research paper' and mark it as important.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8439],CLICK[0.7142,0.0561],TYPE[Read the research paper],CLICK[0.7162,0.1152]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_38_40d1c1b47f1541fabbe7e5f7e49daafe-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.8409]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409]" + }, + { + "from": "gpt", + "value": "CLICK[0.5226,0.4233]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.9561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561]" + }, + { + "from": "gpt", + "value": "TYPE[prepare the budget]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget]" + }, + { + "from": "gpt", + "value": "CLICK[0.3466,0.9699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699]" + }, + { + "from": "gpt", + "value": "CLICK[0.3966,0.9314]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314]" + }, + { + "from": "gpt", + "value": "CLICK[0.5878,0.6348]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348]" + }, + { + "from": "gpt", + "value": "CLICK[0.6287,0.7318]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318]" + }, + { + "from": "gpt", + "value": "CLICK[0.5166,0.9676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318],CLICK[0.5166,0.9676]" + }, + { + "from": "gpt", + "value": "CLICK[0.5895,0.9291]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318],CLICK[0.5166,0.9676],CLICK[0.5895,0.9291]" + }, + { + "from": "gpt", + "value": "CLICK[0.5172,0.6564]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318],CLICK[0.5166,0.9676],CLICK[0.5895,0.9291],CLICK[0.5172,0.6564]" + }, + { + "from": "gpt", + "value": "CLICK[0.4172,0.5672]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318],CLICK[0.5166,0.9676],CLICK[0.5895,0.9291],CLICK[0.5172,0.6564],CLICK[0.4172,0.5672]" + }, + { + "from": "gpt", + "value": "CLICK[0.6716,0.7578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318],CLICK[0.5166,0.9676],CLICK[0.5895,0.9291],CLICK[0.5172,0.6564],CLICK[0.4172,0.5672],CLICK[0.6716,0.7578]" + }, + { + "from": "gpt", + "value": "CLICK[0.7108,0.9247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Work' list named 'Prepare the budget' and set the due date to next Friday and set the reminder time at next Thursday 8 PM.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3105,0.8409],CLICK[0.5226,0.4233],CLICK[0.7041,0.9561],TYPE[prepare the budget],CLICK[0.3466,0.9699],CLICK[0.3966,0.9314],CLICK[0.5878,0.6348],CLICK[0.6287,0.7318],CLICK[0.5166,0.9676],CLICK[0.5895,0.9291],CLICK[0.5172,0.6564],CLICK[0.4172,0.5672],CLICK[0.6716,0.7578],CLICK[0.7108,0.9247]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_40_40b224b2359249428d8d58925d596f7a-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3135,0.8584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8584]" + }, + { + "from": "gpt", + "value": "CLICK[0.2922,0.0537]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8584],CLICK[0.2922,0.0537]" + }, + { + "from": "gpt", + "value": "CLICK[0.7172,0.0554]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8584],CLICK[0.2922,0.0537],CLICK[0.7172,0.0554]" + }, + { + "from": "gpt", + "value": "TYPE[Send the email to the team]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8584],CLICK[0.2922,0.0537],CLICK[0.7172,0.0554],TYPE[Send the email to the team]" + }, + { + "from": "gpt", + "value": "CLICK[0.2828,0.1139]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Send the email to the team' and mark it as completed.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3135,0.8584],CLICK[0.2922,0.0537],CLICK[0.7172,0.0554],TYPE[Send the email to the team],CLICK[0.2828,0.1139]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_42_7c478bb1384f44d691986b9ecdb4f000-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3098,0.7669]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669]" + }, + { + "from": "gpt", + "value": "CLICK[0.4851,0.4726]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669],CLICK[0.4851,0.4726]" + }, + { + "from": "gpt", + "value": "CLICK[0.7047,0.9530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669],CLICK[0.4851,0.4726],CLICK[0.7047,0.9530]" + }, + { + "from": "gpt", + "value": "TYPE[Study English]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669],CLICK[0.4851,0.4726],CLICK[0.7047,0.9530],TYPE[Study English]" + }, + { + "from": "gpt", + "value": "CLICK[0.5666,0.9730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669],CLICK[0.4851,0.4726],CLICK[0.7047,0.9530],TYPE[Study English],CLICK[0.5666,0.9730]" + }, + { + "from": "gpt", + "value": "CLICK[0.6125,0.6740]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669],CLICK[0.4851,0.4726],CLICK[0.7047,0.9530],TYPE[Study English],CLICK[0.5666,0.9730],CLICK[0.6125,0.6740]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.9253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Study English' and set the daily repeat.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3098,0.7669],CLICK[0.4851,0.4726],CLICK[0.7047,0.9530],TYPE[Study English],CLICK[0.5666,0.9730],CLICK[0.6125,0.6740],CLICK[0.7041,0.9253]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_43_9b67ea96a5a3401eb95cbbf65d2421c6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3051,0.8155]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155]" + }, + { + "from": "gpt", + "value": "CLICK[0.3544,0.9753]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753]" + }, + { + "from": "gpt", + "value": "TYPE[Personal]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal]" + }, + { + "from": "gpt", + "value": "CLICK[0.6480,0.6334]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.9568]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568]" + }, + { + "from": "gpt", + "value": "TYPE[Go for a walk]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk]" + }, + { + "from": "gpt", + "value": "CLICK[0.4703,0.9699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk],CLICK[0.4703,0.9699]" + }, + { + "from": "gpt", + "value": "CLICK[0.5480,0.9277]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk],CLICK[0.4703,0.9699],CLICK[0.5480,0.9277]" + }, + { + "from": "gpt", + "value": "CLICK[0.6111,0.2611]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk],CLICK[0.4703,0.9699],CLICK[0.5480,0.9277],CLICK[0.6111,0.2611]" + }, + { + "from": "gpt", + "value": "CLICK[0.5003,0.6101]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk],CLICK[0.4703,0.9699],CLICK[0.5480,0.9277],CLICK[0.6111,0.2611],CLICK[0.5003,0.6101]" + }, + { + "from": "gpt", + "value": "CLICK[0.6595,0.7584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk],CLICK[0.4703,0.9699],CLICK[0.5480,0.9277],CLICK[0.6111,0.2611],CLICK[0.5003,0.6101],CLICK[0.6595,0.7584]" + }, + { + "from": "gpt", + "value": "CLICK[0.7095,0.9301]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Personal' and create a new task named 'Go for a walk' and set the reminder time at 6PM today.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3051,0.8155],CLICK[0.3544,0.9753],TYPE[Personal],CLICK[0.6480,0.6334],CLICK[0.6993,0.9568],TYPE[Go for a walk],CLICK[0.4703,0.9699],CLICK[0.5480,0.9277],CLICK[0.6111,0.2611],CLICK[0.5003,0.6101],CLICK[0.6595,0.7584],CLICK[0.7095,0.9301]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_46_7e2435de3c824a4dbd1a1f6222e2186f-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3081,0.7625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625]" + }, + { + "from": "gpt", + "value": "CLICK[0.7010,0.9568]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568]" + }, + { + "from": "gpt", + "value": "TYPE[Call mom]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom]" + }, + { + "from": "gpt", + "value": "CLICK[0.4851,0.9682]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682]" + }, + { + "from": "gpt", + "value": "CLICK[0.5213,0.9223]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682],CLICK[0.5213,0.9223]" + }, + { + "from": "gpt", + "value": "CLICK[0.5142,0.6203]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682],CLICK[0.5213,0.9223],CLICK[0.5142,0.6203]" + }, + { + "from": "gpt", + "value": "CLICK[0.4274,0.5618]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682],CLICK[0.5213,0.9223],CLICK[0.5142,0.6203],CLICK[0.4274,0.5618]" + }, + { + "from": "gpt", + "value": "CLICK[0.4966,0.6111]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682],CLICK[0.5213,0.9223],CLICK[0.5142,0.6203],CLICK[0.4274,0.5618],CLICK[0.4966,0.6111]" + }, + { + "from": "gpt", + "value": "CLICK[0.6696,0.7554]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682],CLICK[0.5213,0.9223],CLICK[0.5142,0.6203],CLICK[0.4274,0.5618],CLICK[0.4966,0.6111],CLICK[0.6696,0.7554]" + }, + { + "from": "gpt", + "value": "CLICK[0.7095,0.9236]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Call mom' and set the reminder time at 8:30PM tomorrow.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3081,0.7625],CLICK[0.7010,0.9568],TYPE[Call mom],CLICK[0.4851,0.9682],CLICK[0.5213,0.9223],CLICK[0.5142,0.6203],CLICK[0.4274,0.5618],CLICK[0.4966,0.6111],CLICK[0.6696,0.7554],CLICK[0.7095,0.9236]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_49_cef3388eee4242e087aa823a3593a3e1-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3182,0.8091]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091]" + }, + { + "from": "gpt", + "value": "CLICK[0.2892,0.0591]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591]" + }, + { + "from": "gpt", + "value": "CLICK[0.3507,0.9736]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736]" + }, + { + "from": "gpt", + "value": "TYPE[Shopping]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping]" + }, + { + "from": "gpt", + "value": "CLICK[0.6257,0.6311]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.9530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530]" + }, + { + "from": "gpt", + "value": "TYPE[Buy cakes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes]" + }, + { + "from": "gpt", + "value": "CLICK[0.4757,0.9699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes],CLICK[0.4757,0.9699]" + }, + { + "from": "gpt", + "value": "CLICK[0.5149,0.9193]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes],CLICK[0.4757,0.9699],CLICK[0.5149,0.9193]" + }, + { + "from": "gpt", + "value": "CLICK[0.5551,0.6196]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes],CLICK[0.4757,0.9699],CLICK[0.5149,0.9193],CLICK[0.5551,0.6196]" + }, + { + "from": "gpt", + "value": "CLICK[0.5848,0.5564]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes],CLICK[0.4757,0.9699],CLICK[0.5149,0.9193],CLICK[0.5551,0.6196],CLICK[0.5848,0.5564]" + }, + { + "from": "gpt", + "value": "CLICK[0.6642,0.7584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes],CLICK[0.4757,0.9699],CLICK[0.5149,0.9193],CLICK[0.5551,0.6196],CLICK[0.5848,0.5564],CLICK[0.6642,0.7584]" + }, + { + "from": "gpt", + "value": "CLICK[0.7132,0.9236]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Shopping' and create a new task named 'Buy cakes' and set the reminder time at 4PM this Friday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3182,0.8091],CLICK[0.2892,0.0591],CLICK[0.3507,0.9736],TYPE[Shopping],CLICK[0.6257,0.6311],CLICK[0.6993,0.9530],TYPE[Buy cakes],CLICK[0.4757,0.9699],CLICK[0.5149,0.9193],CLICK[0.5551,0.6196],CLICK[0.5848,0.5564],CLICK[0.6642,0.7584],CLICK[0.7132,0.9236]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_50_195ff1deee424a38a5d202a5d9cfebc0-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3145,0.7892]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892]" + }, + { + "from": "gpt", + "value": "CLICK[0.7155,0.0524]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524]" + }, + { + "from": "gpt", + "value": "TYPE[Buy cakes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes]" + }, + { + "from": "gpt", + "value": "CLICK[0.2922,0.1145]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes],CLICK[0.2922,0.1145]" + }, + { + "from": "gpt", + "value": "CLICK[0.2882,0.0514]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes],CLICK[0.2922,0.1145],CLICK[0.2882,0.0514]" + }, + { + "from": "gpt", + "value": "CLICK[0.4703,0.5679]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes],CLICK[0.2922,0.1145],CLICK[0.2882,0.0514],CLICK[0.4703,0.5679]" + }, + { + "from": "gpt", + "value": "CLICK[0.7149,0.0578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes],CLICK[0.2922,0.1145],CLICK[0.2882,0.0514],CLICK[0.4703,0.5679],CLICK[0.7149,0.0578]" + }, + { + "from": "gpt", + "value": "CLICK[0.6027,0.3919]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes],CLICK[0.2922,0.1145],CLICK[0.2882,0.0514],CLICK[0.4703,0.5679],CLICK[0.7149,0.0578],CLICK[0.6027,0.3919]" + }, + { + "from": "gpt", + "value": "CLICK[0.6280,0.5642]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Search for the task 'Buy cakes' and mark it as completed. Then delete the task list named 'Shopping'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3145,0.7892],CLICK[0.7155,0.0524],TYPE[Buy cakes],CLICK[0.2922,0.1145],CLICK[0.2882,0.0514],CLICK[0.4703,0.5679],CLICK[0.7149,0.0578],CLICK[0.6027,0.3919],CLICK[0.6280,0.5642]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_53_3b11c7da03f24efba2dbfa1ad9038eb0-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3091,0.8041]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041]" + }, + { + "from": "gpt", + "value": "CLICK[0.4926,0.5226]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226]" + }, + { + "from": "gpt", + "value": "CLICK[0.6980,0.9537]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537]" + }, + { + "from": "gpt", + "value": "TYPE[Read a book]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book]" + }, + { + "from": "gpt", + "value": "CLICK[0.4659,0.9706]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706]" + }, + { + "from": "gpt", + "value": "CLICK[0.5395,0.9270]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270]" + }, + { + "from": "gpt", + "value": "CLICK[0.6034,0.2767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270],CLICK[0.6034,0.2767]" + }, + { + "from": "gpt", + "value": "CLICK[0.4551,0.4395]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270],CLICK[0.6034,0.2767],CLICK[0.4551,0.4395]" + }, + { + "from": "gpt", + "value": "CLICK[0.6672,0.7554]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270],CLICK[0.6034,0.2767],CLICK[0.4551,0.4395],CLICK[0.6672,0.7554]" + }, + { + "from": "gpt", + "value": "CLICK[0.6855,0.9747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270],CLICK[0.6034,0.2767],CLICK[0.4551,0.4395],CLICK[0.6672,0.7554],CLICK[0.6855,0.9747]" + }, + { + "from": "gpt", + "value": "CLICK[0.6358,0.6885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270],CLICK[0.6034,0.2767],CLICK[0.4551,0.4395],CLICK[0.6672,0.7554],CLICK[0.6855,0.9747],CLICK[0.6358,0.6885]" + }, + { + "from": "gpt", + "value": "CLICK[0.7101,0.9253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Personal' list named 'Read a book' and set the reminder time at 11PM today and repeat daily.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3091,0.8041],CLICK[0.4926,0.5226],CLICK[0.6980,0.9537],TYPE[Read a book],CLICK[0.4659,0.9706],CLICK[0.5395,0.9270],CLICK[0.6034,0.2767],CLICK[0.4551,0.4395],CLICK[0.6672,0.7554],CLICK[0.6855,0.9747],CLICK[0.6358,0.6885],CLICK[0.7101,0.9253]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_54_c2b3ca0da1dc4d3fa917c41f83c1ba51-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3152,0.8439]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439]" + }, + { + "from": "gpt", + "value": "CLICK[0.3497,0.9753]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753]" + }, + { + "from": "gpt", + "value": "TYPE[Study]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study]" + }, + { + "from": "gpt", + "value": "CLICK[0.6503,0.6324]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324]" + }, + { + "from": "gpt", + "value": "CLICK[0.2858,0.0561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561]" + }, + { + "from": "gpt", + "value": "CLICK[0.7193,0.9747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561],CLICK[0.7193,0.9747]" + }, + { + "from": "gpt", + "value": "TYPE[Hard Life]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561],CLICK[0.7193,0.9747],TYPE[Hard Life]" + }, + { + "from": "gpt", + "value": "CLICK[0.6203,0.5770]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561],CLICK[0.7193,0.9747],TYPE[Hard Life],CLICK[0.6203,0.5770]" + }, + { + "from": "gpt", + "value": "CLICK[0.6348,0.4750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561],CLICK[0.7193,0.9747],TYPE[Hard Life],CLICK[0.6203,0.5770],CLICK[0.6348,0.4750]" + }, + { + "from": "gpt", + "value": "CLICK[0.6334,0.5318]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561],CLICK[0.7193,0.9747],TYPE[Hard Life],CLICK[0.6203,0.5770],CLICK[0.6348,0.4750],CLICK[0.6334,0.5318]" + }, + { + "from": "gpt", + "value": "CLICK[0.6324,0.6003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task list named 'Study' and create a new task list group named 'Hard Life'. Add the task list 'Study' and 'Work' to the group 'Hard Life'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3152,0.8439],CLICK[0.3497,0.9753],TYPE[Study],CLICK[0.6503,0.6324],CLICK[0.2858,0.0561],CLICK[0.7193,0.9747],TYPE[Hard Life],CLICK[0.6203,0.5770],CLICK[0.6348,0.4750],CLICK[0.6334,0.5318],CLICK[0.6324,0.6003]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_5_1b50725bcd724f2e94da809cfdbd4128-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3166,0.8416]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416]" + }, + { + "from": "gpt", + "value": "CLICK[0.4780,0.4774]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774]" + }, + { + "from": "gpt", + "value": "CLICK[0.7017,0.9622]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622]" + }, + { + "from": "gpt", + "value": "TYPE[Read linear algebra textbook chapter 2]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622],TYPE[Read linear algebra textbook chapter 2]" + }, + { + "from": "gpt", + "value": "CLICK[0.3514,0.9723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622],TYPE[Read linear algebra textbook chapter 2],CLICK[0.3514,0.9723]" + }, + { + "from": "gpt", + "value": "CLICK[0.3689,0.9307]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622],TYPE[Read linear algebra textbook chapter 2],CLICK[0.3514,0.9723],CLICK[0.3689,0.9307]" + }, + { + "from": "gpt", + "value": "CLICK[0.4557,0.6419]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622],TYPE[Read linear algebra textbook chapter 2],CLICK[0.3514,0.9723],CLICK[0.3689,0.9307],CLICK[0.4557,0.6419]" + }, + { + "from": "gpt", + "value": "CLICK[0.6372,0.7318]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622],TYPE[Read linear algebra textbook chapter 2],CLICK[0.3514,0.9723],CLICK[0.3689,0.9307],CLICK[0.4557,0.6419],CLICK[0.6372,0.7318]" + }, + { + "from": "gpt", + "value": "CLICK[0.7125,0.9247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Microsoft To Do. Create a new task in the 'Study' list named 'Read linear algebra textbook chapter 2' and set the due date to next Tuesday.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3166,0.8416],CLICK[0.4780,0.4774],CLICK[0.7017,0.9622],TYPE[Read linear algebra textbook chapter 2],CLICK[0.3514,0.9723],CLICK[0.3689,0.9307],CLICK[0.4557,0.6419],CLICK[0.6372,0.7318],CLICK[0.7125,0.9247]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_15_7_6bcfbdaa983344318a201834291d1d72-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4057,0.8801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4057,0.8801]" + }, + { + "from": "gpt", + "value": "CLICK[0.3544,0.1767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4057,0.8801],CLICK[0.3544,0.1767]" + }, + { + "from": "gpt", + "value": "CLICK[0.3736,0.4919]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4057,0.8801],CLICK[0.3544,0.1767],CLICK[0.3736,0.4919]" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.8909]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4057,0.8801],CLICK[0.3544,0.1767],CLICK[0.3736,0.4919],CLICK[0.7071,0.8909]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around the current location. Sort by ratings. What is the first non-sponsored restaurant?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4057,0.8801],CLICK[0.3544,0.1767],CLICK[0.3736,0.4919],CLICK[0.7071,0.8909],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_40_91e7c66fd14345a8915295d7c1f0ca57-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4098,0.8791]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791]" + }, + { + "from": "gpt", + "value": "CLICK[0.4419,0.1213]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213]" + }, + { + "from": "gpt", + "value": "CLICK[0.4429,0.1199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199]" + }, + { + "from": "gpt", + "value": "TYPE[Bryant Park, NY]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY]" + }, + { + "from": "gpt", + "value": "CLICK[0.4544,0.0578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578]" + }, + { + "from": "gpt", + "value": "CLICK[0.4220,0.5003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003]" + }, + { + "from": "gpt", + "value": "CLICK[0.3814,0.4949]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003],CLICK[0.3814,0.4949]" + }, + { + "from": "gpt", + "value": "CLICK[0.6973,0.8348]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003],CLICK[0.3814,0.4949],CLICK[0.6973,0.8348]" + }, + { + "from": "gpt", + "value": "CLICK[0.3098,0.5027]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003],CLICK[0.3814,0.4949],CLICK[0.6973,0.8348],CLICK[0.3098,0.5027]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003],CLICK[0.3814,0.4949],CLICK[0.6973,0.8348],CLICK[0.3098,0.5027],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6679,0.8777]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003],CLICK[0.3814,0.4949],CLICK[0.6973,0.8348],CLICK[0.3098,0.5027],SWIPE[UP],CLICK[0.6679,0.8777]" + }, + { + "from": "gpt", + "value": "CLICK[0.6534,0.9524]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the restaurants around 'Bryant Park, NY', and sort by distance. Filter the results by available takeout.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8791],CLICK[0.4419,0.1213],CLICK[0.4429,0.1199],TYPE[Bryant Park, NY],CLICK[0.4544,0.0578],CLICK[0.4220,0.5003],CLICK[0.3814,0.4949],CLICK[0.6973,0.8348],CLICK[0.3098,0.5027],SWIPE[UP],CLICK[0.6679,0.8777],CLICK[0.6534,0.9524]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_42_423a9867be43419f9fec526df44c46d1-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4152,0.8855]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.0976]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855],CLICK[0.4949,0.0976]" + }, + { + "from": "gpt", + "value": "TYPE[McDonald's]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855],CLICK[0.4949,0.0976],TYPE[McDonald's]" + }, + { + "from": "gpt", + "value": "CLICK[0.4743,0.1743]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855],CLICK[0.4949,0.0976],TYPE[McDonald's],CLICK[0.4743,0.1743]" + }, + { + "from": "gpt", + "value": "CLICK[0.3767,0.4943]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855],CLICK[0.4949,0.0976],TYPE[McDonald's],CLICK[0.4743,0.1743],CLICK[0.3767,0.4943]" + }, + { + "from": "gpt", + "value": "CLICK[0.7024,0.8368]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855],CLICK[0.4949,0.0976],TYPE[McDonald's],CLICK[0.4743,0.1743],CLICK[0.3767,0.4943],CLICK[0.7024,0.8368]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest McDonald's. Where is it?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4152,0.8855],CLICK[0.4949,0.0976],TYPE[McDonald's],CLICK[0.4743,0.1743],CLICK[0.3767,0.4943],CLICK[0.7024,0.8368],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_48_9908e31f5c34417a9d4cb32da5c0683a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3983,0.8791]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791]" + }, + { + "from": "gpt", + "value": "CLICK[0.4973,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791],CLICK[0.4973,0.1253]" + }, + { + "from": "gpt", + "value": "TYPE[Starbucks]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791],CLICK[0.4973,0.1253],TYPE[Starbucks]" + }, + { + "from": "gpt", + "value": "CLICK[0.4666,0.1838]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791],CLICK[0.4973,0.1253],TYPE[Starbucks],CLICK[0.4666,0.1838]" + }, + { + "from": "gpt", + "value": "CLICK[0.3797,0.4949]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791],CLICK[0.4973,0.1253],TYPE[Starbucks],CLICK[0.4666,0.1838],CLICK[0.3797,0.4949]" + }, + { + "from": "gpt", + "value": "CLICK[0.7034,0.8385]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791],CLICK[0.4973,0.1253],TYPE[Starbucks],CLICK[0.4666,0.1838],CLICK[0.3797,0.4949],CLICK[0.7034,0.8385]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for the nearest Starbucks. What's the distance?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3983,0.8791],CLICK[0.4973,0.1253],TYPE[Starbucks],CLICK[0.4666,0.1838],CLICK[0.3797,0.4949],CLICK[0.7034,0.8385],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_51_fa7640d7ad5a4803a16d483bc65375a9-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4105,0.8976]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4105,0.8976]" + }, + { + "from": "gpt", + "value": "CLICK[0.5818,0.1784]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4105,0.8976],CLICK[0.5818,0.1784]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4105,0.8976],CLICK[0.5818,0.1784],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5179,0.6757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4105,0.8976],CLICK[0.5818,0.1784],SWIPE[UP],CLICK[0.5179,0.6757]" + }, + { + "from": "gpt", + "value": "CLICK[0.5105,0.5780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4105,0.8976],CLICK[0.5818,0.1784],SWIPE[UP],CLICK[0.5179,0.6757],CLICK[0.5105,0.5780]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for movers around the current location. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4105,0.8976],CLICK[0.5818,0.1784],SWIPE[UP],CLICK[0.5179,0.6757],CLICK[0.5105,0.5780],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_55_9e5bcbcdef0d43e08496ece8553a01f9-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4081,0.8807]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4081,0.8807]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.1784]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4081,0.8807],CLICK[0.6878,0.1784]" + }, + { + "from": "gpt", + "value": "CLICK[0.4767,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4081,0.8807],CLICK[0.6878,0.1784],CLICK[0.4767,0.1689]" + }, + { + "from": "gpt", + "value": "CLICK[0.5189,0.4764]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4081,0.8807],CLICK[0.6878,0.1784],CLICK[0.4767,0.1689],CLICK[0.5189,0.4764]" + }, + { + "from": "gpt", + "value": "CLICK[0.5618,0.5811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for plumbers and filter by yelp guaranteed. What is the phone number of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4081,0.8807],CLICK[0.6878,0.1784],CLICK[0.4767,0.1689],CLICK[0.5189,0.4764],CLICK[0.5618,0.5811]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_16_57_6c615b70b86d42d18590110e33e96200-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4027,0.8953]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953]" + }, + { + "from": "gpt", + "value": "CLICK[0.3537,0.1767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767]" + }, + { + "from": "gpt", + "value": "CLICK[0.3834,0.4848]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767],CLICK[0.3834,0.4848]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.8909]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767],CLICK[0.3834,0.4848],CLICK[0.7041,0.8909]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767],CLICK[0.3834,0.4848],CLICK[0.7041,0.8909],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767],CLICK[0.3834,0.4848],CLICK[0.7041,0.8909],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5510,0.6149]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767],CLICK[0.3834,0.4848],CLICK[0.7041,0.8909],SWIPE[UP],SWIPE[UP],CLICK[0.5510,0.6149]" + }, + { + "from": "gpt", + "value": "CLICK[0.3943,0.7493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4027,0.8953],CLICK[0.3537,0.1767],CLICK[0.3834,0.4848],CLICK[0.7041,0.8909],SWIPE[UP],SWIPE[UP],CLICK[0.5510,0.6149],CLICK[0.3943,0.7493]" + }, + { + "from": "gpt", + "value": "TASK_IMPOSSIBLE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_10_e032efbbbc9044f990bc83b80c80210e-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4128,0.8875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875]" + }, + { + "from": "gpt", + "value": "CLICK[0.4956,0.1247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247]" + }, + { + "from": "gpt", + "value": "TYPE[Restaurants]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants]" + }, + { + "from": "gpt", + "value": "CLICK[0.4297,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253]" + }, + { + "from": "gpt", + "value": "TYPE[Times Square]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.3821,0.4902]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER,CLICK[0.3821,0.4902]" + }, + { + "from": "gpt", + "value": "CLICK[0.7017,0.8916]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER,CLICK[0.3821,0.4902],CLICK[0.7017,0.8916]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER,CLICK[0.3821,0.4902],CLICK[0.7017,0.8916],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER,CLICK[0.3821,0.4902],CLICK[0.7017,0.8916],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5841,0.5949]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER,CLICK[0.3821,0.4902],CLICK[0.7017,0.8916],SWIPE[UP],SWIPE[UP],CLICK[0.5841,0.5949]" + }, + { + "from": "gpt", + "value": "CLICK[0.5088,0.8584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for restaurants around Times Square. Sort by ratings and select the first non-sponsored recommendation. Open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8875],CLICK[0.4956,0.1247],TYPE[Restaurants],CLICK[0.4297,0.1253],TYPE[Times Square],PRESS_ENTER,CLICK[0.3821,0.4902],CLICK[0.7017,0.8916],SWIPE[UP],SWIPE[UP],CLICK[0.5841,0.5949],CLICK[0.5088,0.8584]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_12_17624c6b489f4331acd296a23a1aa85f-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4172,0.8584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584]" + }, + { + "from": "gpt", + "value": "CLICK[0.4787,0.1291]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291]" + }, + { + "from": "gpt", + "value": "TYPE[Steakhouse]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse]" + }, + { + "from": "gpt", + "value": "CLICK[0.4635,0.1807]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse],CLICK[0.4635,0.1807]" + }, + { + "from": "gpt", + "value": "CLICK[0.3044,0.4949]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse],CLICK[0.4635,0.1807],CLICK[0.3044,0.4949]" + }, + { + "from": "gpt", + "value": "CLICK[0.3791,0.1912]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse],CLICK[0.4635,0.1807],CLICK[0.3044,0.4949],CLICK[0.3791,0.1912]" + }, + { + "from": "gpt", + "value": "CLICK[0.4291,0.1861]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse],CLICK[0.4635,0.1807],CLICK[0.3044,0.4949],CLICK[0.3791,0.1912],CLICK[0.4291,0.1861]" + }, + { + "from": "gpt", + "value": "CLICK[0.5034,0.1861]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse],CLICK[0.4635,0.1807],CLICK[0.3044,0.4949],CLICK[0.3791,0.1912],CLICK[0.4291,0.1861],CLICK[0.5034,0.1861]" + }, + { + "from": "gpt", + "value": "CLICK[0.6588,0.9500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Steakhouse' around the current location. Filter by price from $$ to $$$$.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4172,0.8584],CLICK[0.4787,0.1291],TYPE[Steakhouse],CLICK[0.4635,0.1807],CLICK[0.3044,0.4949],CLICK[0.3791,0.1912],CLICK[0.4291,0.1861],CLICK[0.5034,0.1861],CLICK[0.6588,0.9500]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_15_c1be206877134eaf9034adcf9dedbe60-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for home cleaning services. What is the review rating of the first sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_2_33ec76b698004f258610c54f497c7978-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for home cleaning services. What is the review rating of the first sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_2_33ec76b698004f258610c54f497c7978-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for home cleaning services. What is the review rating of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.8753]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_2_33ec76b698004f258610c54f497c7978-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for home cleaning services. What is the review rating of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4051,0.8753]" + }, + { + "from": "gpt", + "value": "SWIPE[LEFT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_2_33ec76b698004f258610c54f497c7978-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for home cleaning services. What is the review rating of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4051,0.8753],SWIPE[LEFT]" + }, + { + "from": "gpt", + "value": "CLICK[0.6101,0.1797]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_2_33ec76b698004f258610c54f497c7978-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for home cleaning services. What is the review rating of the first sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4051,0.8753],SWIPE[LEFT],CLICK[0.6101,0.1797]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_2_33ec76b698004f258610c54f497c7978-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.8838]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838]" + }, + { + "from": "gpt", + "value": "CLICK[0.4834,0.1247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247]" + }, + { + "from": "gpt", + "value": "TYPE[Pizza]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza]" + }, + { + "from": "gpt", + "value": "CLICK[0.4736,0.1176]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176]" + }, + { + "from": "gpt", + "value": "TYPE[Wall Street]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street]" + }, + { + "from": "gpt", + "value": "PRESS_ENTER" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER" + }, + { + "from": "gpt", + "value": "CLICK[0.5179,0.8277]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER,CLICK[0.5179,0.8277]" + }, + { + "from": "gpt", + "value": "CLICK[0.3061,0.8500]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER,CLICK[0.5179,0.8277],CLICK[0.3061,0.8500]" + }, + { + "from": "gpt", + "value": "CLICK[0.6618,0.6155]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER,CLICK[0.5179,0.8277],CLICK[0.3061,0.8500],CLICK[0.6618,0.6155]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER,CLICK[0.5179,0.8277],CLICK[0.3061,0.8500],CLICK[0.6618,0.6155],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6601,0.8355]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER,CLICK[0.5179,0.8277],CLICK[0.3061,0.8500],CLICK[0.6618,0.6155],SWIPE[UP],CLICK[0.6601,0.8355]" + }, + { + "from": "gpt", + "value": "CLICK[0.6679,0.9544]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Pizza' around Wall Street. Filter by available online waitlist. Sort by ratings.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8838],CLICK[0.4834,0.1247],TYPE[Pizza],CLICK[0.4736,0.1176],TYPE[Wall Street],PRESS_ENTER,CLICK[0.5179,0.8277],CLICK[0.3061,0.8500],CLICK[0.6618,0.6155],SWIPE[UP],CLICK[0.6601,0.8355],CLICK[0.6679,0.9544]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_50_8e50dda246434698ad00b5fe62f2378f-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4111,0.8723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723]" + }, + { + "from": "gpt", + "value": "CLICK[0.4304,0.1247]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723],CLICK[0.4304,0.1247]" + }, + { + "from": "gpt", + "value": "TYPE[Sushi]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723],CLICK[0.4304,0.1247],TYPE[Sushi]" + }, + { + "from": "gpt", + "value": "CLICK[0.4980,0.1767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723],CLICK[0.4304,0.1247],TYPE[Sushi],CLICK[0.4980,0.1767]" + }, + { + "from": "gpt", + "value": "CLICK[0.3730,0.4973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723],CLICK[0.4304,0.1247],TYPE[Sushi],CLICK[0.4980,0.1767],CLICK[0.3730,0.4973]" + }, + { + "from": "gpt", + "value": "CLICK[0.7034,0.8368]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723],CLICK[0.4304,0.1247],TYPE[Sushi],CLICK[0.4980,0.1767],CLICK[0.3730,0.4973],CLICK[0.7034,0.8368]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Sushi' around the current location. Sort by distance. What is the average rating of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4111,0.8723],CLICK[0.4304,0.1247],TYPE[Sushi],CLICK[0.4980,0.1767],CLICK[0.3730,0.4973],CLICK[0.7034,0.8368],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_54_301f0efdcd1b474db0550193b025d3ab-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4044,0.8699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699]" + }, + { + "from": "gpt", + "value": "CLICK[0.4659,0.1253]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253]" + }, + { + "from": "gpt", + "value": "CLICK[0.4696,0.1260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260]" + }, + { + "from": "gpt", + "value": "TYPE[Empire States]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States]" + }, + { + "from": "gpt", + "value": "CLICK[0.4757,0.0662]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States],CLICK[0.4757,0.0662]" + }, + { + "from": "gpt", + "value": "TYPE[Burgers]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States],CLICK[0.4757,0.0662],TYPE[Burgers]" + }, + { + "from": "gpt", + "value": "CLICK[0.4689,0.1828]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States],CLICK[0.4757,0.0662],TYPE[Burgers],CLICK[0.4689,0.1828]" + }, + { + "from": "gpt", + "value": "CLICK[0.5220,0.7922]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States],CLICK[0.4757,0.0662],TYPE[Burgers],CLICK[0.4689,0.1828],CLICK[0.5220,0.7922]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States],CLICK[0.4757,0.0662],TYPE[Burgers],CLICK[0.4689,0.1828],CLICK[0.5220,0.7922],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5088,0.8953]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Burgers' around Empire States. Open the menu tab of the sponsored result.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4044,0.8699],CLICK[0.4659,0.1253],CLICK[0.4696,0.1260],TYPE[Empire States],CLICK[0.4757,0.0662],TYPE[Burgers],CLICK[0.4689,0.1828],CLICK[0.5220,0.7922],SWIPE[UP],CLICK[0.5088,0.8953]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_17_58_621c3bfbcbfb4b4bb632ca8f71a3e4e6-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4128,0.8838]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838]" + }, + { + "from": "gpt", + "value": "CLICK[0.4449,0.1260]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260]" + }, + { + "from": "gpt", + "value": "TYPE[Coffee]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee]" + }, + { + "from": "gpt", + "value": "CLICK[0.4226,0.1774]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee],CLICK[0.4226,0.1774]" + }, + { + "from": "gpt", + "value": "CLICK[0.3713,0.4926]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee],CLICK[0.4226,0.1774],CLICK[0.3713,0.4926]" + }, + { + "from": "gpt", + "value": "CLICK[0.7034,0.8409]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee],CLICK[0.4226,0.1774],CLICK[0.3713,0.4926],CLICK[0.7034,0.8409]" + }, + { + "from": "gpt", + "value": "CLICK[0.3105,0.4973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee],CLICK[0.4226,0.1774],CLICK[0.3713,0.4926],CLICK[0.7034,0.8409],CLICK[0.3105,0.4973]" + }, + { + "from": "gpt", + "value": "CLICK[0.6611,0.8524]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee],CLICK[0.4226,0.1774],CLICK[0.3713,0.4926],CLICK[0.7034,0.8409],CLICK[0.3105,0.4973],CLICK[0.6611,0.8524]" + }, + { + "from": "gpt", + "value": "CLICK[0.6679,0.9530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Coffee' around the current location. Sort by distance and filter by available reservations.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4128,0.8838],CLICK[0.4449,0.1260],TYPE[Coffee],CLICK[0.4226,0.1774],CLICK[0.3713,0.4926],CLICK[0.7034,0.8409],CLICK[0.3105,0.4973],CLICK[0.6611,0.8524],CLICK[0.6679,0.9530]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_13_83b7e91493814637bfede5eebfdb0d2d-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4098,0.8730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730]" + }, + { + "from": "gpt", + "value": "CLICK[0.4696,0.1128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128]" + }, + { + "from": "gpt", + "value": "TYPE[Chinese]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese]" + }, + { + "from": "gpt", + "value": "CLICK[0.4037,0.1767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese],CLICK[0.4037,0.1767]" + }, + { + "from": "gpt", + "value": "CLICK[0.3791,0.4932]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese],CLICK[0.4037,0.1767],CLICK[0.3791,0.4932]" + }, + { + "from": "gpt", + "value": "CLICK[0.6973,0.8845]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese],CLICK[0.4037,0.1767],CLICK[0.3791,0.4932],CLICK[0.6973,0.8845]" + }, + { + "from": "gpt", + "value": "CLICK[0.5243,0.7885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese],CLICK[0.4037,0.1767],CLICK[0.3791,0.4932],CLICK[0.6973,0.8845],CLICK[0.5243,0.7885]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese],CLICK[0.4037,0.1767],CLICK[0.3791,0.4932],CLICK[0.6973,0.8845],CLICK[0.5243,0.7885],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5088,0.7294]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Chinese food' around the current location. Sort by ratings. Open the menu tab of the first recommendation.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4098,0.8730],CLICK[0.4696,0.1128],TYPE[Chinese],CLICK[0.4037,0.1767],CLICK[0.3791,0.4932],CLICK[0.6973,0.8845],CLICK[0.5243,0.7885],SWIPE[UP],CLICK[0.5088,0.7294]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_17_9cecfe11a8f9473b9169e04bbf7e1a27-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4243,0.9145]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145]" + }, + { + "from": "gpt", + "value": "CLICK[0.5142,0.1199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199]" + }, + { + "from": "gpt", + "value": "TYPE[Italian]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian]" + }, + { + "from": "gpt", + "value": "CLICK[0.4642,0.1851]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian],CLICK[0.4642,0.1851]" + }, + { + "from": "gpt", + "value": "CLICK[0.3750,0.4848]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian],CLICK[0.4642,0.1851],CLICK[0.3750,0.4848]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.8361]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian],CLICK[0.4642,0.1851],CLICK[0.3750,0.4848],CLICK[0.7041,0.8361]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian],CLICK[0.4642,0.1851],CLICK[0.3750,0.4848],CLICK[0.7041,0.8361],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5149,0.8348]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian],CLICK[0.4642,0.1851],CLICK[0.3750,0.4848],CLICK[0.7041,0.8361],SWIPE[UP],CLICK[0.5149,0.8348]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.7125]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Italian' around the current location. Sort by distance. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4243,0.9145],CLICK[0.5142,0.1199],TYPE[Italian],CLICK[0.4642,0.1851],CLICK[0.3750,0.4848],CLICK[0.7041,0.8361],SWIPE[UP],CLICK[0.5149,0.8348],CLICK[0.3997,0.7125]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_20_11dc16a004a24c7fb3079d02c65d0675-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4088,0.8939]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939]" + }, + { + "from": "gpt", + "value": "CLICK[0.4659,0.1206]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206]" + }, + { + "from": "gpt", + "value": "TYPE[Thai]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206],TYPE[Thai]" + }, + { + "from": "gpt", + "value": "CLICK[0.4649,0.1875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206],TYPE[Thai],CLICK[0.4649,0.1875]" + }, + { + "from": "gpt", + "value": "CLICK[0.3814,0.4956]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206],TYPE[Thai],CLICK[0.4649,0.1875],CLICK[0.3814,0.4956]" + }, + { + "from": "gpt", + "value": "CLICK[0.7034,0.8909]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206],TYPE[Thai],CLICK[0.4649,0.1875],CLICK[0.3814,0.4956],CLICK[0.7034,0.8909]" + }, + { + "from": "gpt", + "value": "CLICK[0.5480,0.7855]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206],TYPE[Thai],CLICK[0.4649,0.1875],CLICK[0.3814,0.4956],CLICK[0.7034,0.8909],CLICK[0.5480,0.7855]" + }, + { + "from": "gpt", + "value": "CLICK[0.4990,0.7223]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Thai' around the current location. Sort by ratings. Check the open hours of the first recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4088,0.8939],CLICK[0.4659,0.1206],TYPE[Thai],CLICK[0.4649,0.1875],CLICK[0.3814,0.4956],CLICK[0.7034,0.8909],CLICK[0.5480,0.7855],CLICK[0.4990,0.7223]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_24_8dbe59c5ec3c453bba3a76ace2a85c32-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4074,0.8770]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770]" + }, + { + "from": "gpt", + "value": "CLICK[0.4797,0.1182]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182]" + }, + { + "from": "gpt", + "value": "TYPE[breakfast]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast]" + }, + { + "from": "gpt", + "value": "CLICK[0.4236,0.1828]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828]" + }, + { + "from": "gpt", + "value": "CLICK[0.2966,0.4956]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956]" + }, + { + "from": "gpt", + "value": "CLICK[0.3321,0.1875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875]" + }, + { + "from": "gpt", + "value": "CLICK[0.3743,0.1905]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.6162]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905],CLICK[0.6632,0.6162]" + }, + { + "from": "gpt", + "value": "CLICK[0.6595,0.9490]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905],CLICK[0.6632,0.6162],CLICK[0.6595,0.9490]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905],CLICK[0.6632,0.6162],CLICK[0.6595,0.9490],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5220,0.8409]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905],CLICK[0.6632,0.6162],CLICK[0.6595,0.9490],SWIPE[UP],CLICK[0.5220,0.8409]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905],CLICK[0.6632,0.6162],CLICK[0.6595,0.9490],SWIPE[UP],CLICK[0.5220,0.8409],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5142,0.7318]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'breakfirst' around the current location. Filter by price from $ to $$. Select the nearest restaurant and open the menu tab.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4074,0.8770],CLICK[0.4797,0.1182],TYPE[breakfast],CLICK[0.4236,0.1828],CLICK[0.2966,0.4956],CLICK[0.3321,0.1875],CLICK[0.3743,0.1905],CLICK[0.6632,0.6162],CLICK[0.6595,0.9490],SWIPE[UP],CLICK[0.5220,0.8409],SWIPE[UP],CLICK[0.5142,0.7318]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_26_643b28a28a984f9b9738c2f7a26f8266-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3997,0.8801]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801]" + }, + { + "from": "gpt", + "value": "CLICK[0.4804,0.1230]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230]" + }, + { + "from": "gpt", + "value": "TYPE[Popeyes]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes]" + }, + { + "from": "gpt", + "value": "CLICK[0.4588,0.1767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes],CLICK[0.4588,0.1767]" + }, + { + "from": "gpt", + "value": "CLICK[0.3730,0.4858]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes],CLICK[0.4588,0.1767],CLICK[0.3730,0.4858]" + }, + { + "from": "gpt", + "value": "CLICK[0.7064,0.8392]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes],CLICK[0.4588,0.1767],CLICK[0.3730,0.4858],CLICK[0.7064,0.8392]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes],CLICK[0.4588,0.1767],CLICK[0.3730,0.4858],CLICK[0.7064,0.8392],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5111,0.6378]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes],CLICK[0.4588,0.1767],CLICK[0.3730,0.4858],CLICK[0.7064,0.8392],SWIPE[UP],CLICK[0.5111,0.6378]" + }, + { + "from": "gpt", + "value": "CLICK[0.3919,0.7101]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Popeyes' around the current location. Select the nearest one and check the open hours.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.3997,0.8801],CLICK[0.4804,0.1230],TYPE[Popeyes],CLICK[0.4588,0.1767],CLICK[0.3730,0.4858],CLICK[0.7064,0.8392],SWIPE[UP],CLICK[0.5111,0.6378],CLICK[0.3919,0.7101]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_28_5575f393d28a437bb3e9e298fdd7501b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4213,0.8760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760]" + }, + { + "from": "gpt", + "value": "CLICK[0.4304,0.1236]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236]" + }, + { + "from": "gpt", + "value": "TYPE[Tacos]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos]" + }, + { + "from": "gpt", + "value": "CLICK[0.4405,0.1699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos],CLICK[0.4405,0.1699]" + }, + { + "from": "gpt", + "value": "CLICK[0.3912,0.4912]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos],CLICK[0.4405,0.1699],CLICK[0.3912,0.4912]" + }, + { + "from": "gpt", + "value": "CLICK[0.7010,0.8875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos],CLICK[0.4405,0.1699],CLICK[0.3912,0.4912],CLICK[0.7010,0.8875]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos],CLICK[0.4405,0.1699],CLICK[0.3912,0.4912],CLICK[0.7010,0.8875],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4912,0.8169]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos],CLICK[0.4405,0.1699],CLICK[0.3912,0.4912],CLICK[0.7010,0.8875],SWIPE[UP],CLICK[0.4912,0.8169]" + }, + { + "from": "gpt", + "value": "CLICK[0.3865,0.7733]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Tacos' around the current location. Sort by ratings. What is the phone number of the first non-sponsored recommendation?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4213,0.8760],CLICK[0.4304,0.1236],TYPE[Tacos],CLICK[0.4405,0.1699],CLICK[0.3912,0.4912],CLICK[0.7010,0.8875],SWIPE[UP],CLICK[0.4912,0.8169],CLICK[0.3865,0.7733]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_3_57b0f0bcf4954b2985f11274d4f3426d-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4068,0.8885]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885]" + }, + { + "from": "gpt", + "value": "CLICK[0.4804,0.1182]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885],CLICK[0.4804,0.1182]" + }, + { + "from": "gpt", + "value": "TYPE[Ice Cream]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885],CLICK[0.4804,0.1182],TYPE[Ice Cream]" + }, + { + "from": "gpt", + "value": "CLICK[0.4605,0.1821]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885],CLICK[0.4804,0.1182],TYPE[Ice Cream],CLICK[0.4605,0.1821]" + }, + { + "from": "gpt", + "value": "CLICK[0.3051,0.4943]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885],CLICK[0.4804,0.1182],TYPE[Ice Cream],CLICK[0.4605,0.1821],CLICK[0.3051,0.4943]" + }, + { + "from": "gpt", + "value": "CLICK[0.3382,0.1882]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885],CLICK[0.4804,0.1182],TYPE[Ice Cream],CLICK[0.4605,0.1821],CLICK[0.3051,0.4943],CLICK[0.3382,0.1882]" + }, + { + "from": "gpt", + "value": "CLICK[0.6510,0.9514]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Yelp. Search for 'Ice Cream' around the current location. Filter by price at $.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.4068,0.8885],CLICK[0.4804,0.1182],TYPE[Ice Cream],CLICK[0.4605,0.1821],CLICK[0.3051,0.4943],CLICK[0.3382,0.1882],CLICK[0.6510,0.9514]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_19_18_9_f5942a4e17e3405fb9266aa1bb137b45-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6885,0.3882]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3882]" + }, + { + "from": "gpt", + "value": "CLICK[0.4997,0.4696]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3882],CLICK[0.4997,0.4696]" + }, + { + "from": "gpt", + "value": "TYPE[Empire State]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3882],CLICK[0.4997,0.4696],TYPE[Empire State]" + }, + { + "from": "gpt", + "value": "CLICK[0.4973,0.2152]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3882],CLICK[0.4997,0.4696],TYPE[Empire State],CLICK[0.4973,0.2152]" + }, + { + "from": "gpt", + "value": "CLICK[0.6909,0.8186]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Empire State by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3882],CLICK[0.4997,0.4696],TYPE[Empire State],CLICK[0.4973,0.2152],CLICK[0.6909,0.8186]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_0_0ee3d2dcc9544b19b10e68692e84072d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address by public transit.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_11_ce6e2c0195264b17b062f4d6a361207f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address by public transit.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6841,0.3828]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_11_ce6e2c0195264b17b062f4d6a361207f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3828]" + }, + { + "from": "gpt", + "value": "CLICK[0.5095,0.4672]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_11_ce6e2c0195264b17b062f4d6a361207f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3828],CLICK[0.5095,0.4672]" + }, + { + "from": "gpt", + "value": "CLICK[0.5088,0.1689]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_11_ce6e2c0195264b17b062f4d6a361207f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3828],CLICK[0.5095,0.4672],CLICK[0.5088,0.1689]" + }, + { + "from": "gpt", + "value": "CLICK[0.4736,0.3429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_11_ce6e2c0195264b17b062f4d6a361207f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address by public transit.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3828],CLICK[0.5095,0.4672],CLICK[0.5088,0.1689],CLICK[0.4736,0.3429]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_11_ce6e2c0195264b17b062f4d6a361207f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a nearest Citi Bike station. Find out how many empty spaces are available.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_12_e8f905cd8f2f4741983380eff5a97fc9-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a nearest Citi Bike station. Find out how many empty spaces are available.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6818,0.3828]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_12_e8f905cd8f2f4741983380eff5a97fc9-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a nearest Citi Bike station. Find out how many empty spaces are available.\nHistory Actions: SWIPE[UP],CLICK[0.6818,0.3828]" + }, + { + "from": "gpt", + "value": "CLICK[0.5003,0.5993]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_12_e8f905cd8f2f4741983380eff5a97fc9-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a nearest Citi Bike station. Find out how many empty spaces are available.\nHistory Actions: SWIPE[UP],CLICK[0.6818,0.3828],CLICK[0.5003,0.5993]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_12_e8f905cd8f2f4741983380eff5a97fc9-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest subway station. Find out what lines are available.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_13_dab2917f9fdd498b8009e68b8d8b3bed-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest subway station. Find out what lines are available.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6932,0.3828]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_13_dab2917f9fdd498b8009e68b8d8b3bed-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest subway station. Find out what lines are available.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3828]" + }, + { + "from": "gpt", + "value": "CLICK[0.3166,0.6517]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_13_dab2917f9fdd498b8009e68b8d8b3bed-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest subway station. Find out what lines are available.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3828],CLICK[0.3166,0.6517]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_13_dab2917f9fdd498b8009e68b8d8b3bed-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest bus stop. Find out what lines are available.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_14_f40be1322266496ba63cfd600bbb868a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest bus stop. Find out what lines are available.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.3757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_14_f40be1322266496ba63cfd600bbb868a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest bus stop. Find out what lines are available.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3757]" + }, + { + "from": "gpt", + "value": "CLICK[0.4105,0.6601]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_14_f40be1322266496ba63cfd600bbb868a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for the nearest bus stop. Find out what lines are available.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3757],CLICK[0.4105,0.6601]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_14_f40be1322266496ba63cfd600bbb868a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6939,0.3797]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: SWIPE[UP],CLICK[0.6939,0.3797]" + }, + { + "from": "gpt", + "value": "CLICK[0.4973,0.4649]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: SWIPE[UP],CLICK[0.6939,0.3797],CLICK[0.4973,0.4649]" + }, + { + "from": "gpt", + "value": "TYPE[Times Square]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: SWIPE[UP],CLICK[0.6939,0.3797],CLICK[0.4973,0.4649],TYPE[Times Square]" + }, + { + "from": "gpt", + "value": "CLICK[0.4882,0.3044]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: SWIPE[UP],CLICK[0.6939,0.3797],CLICK[0.4973,0.4649],TYPE[Times Square],CLICK[0.4882,0.3044]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.8132]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Times Square. What is the fastest route by subway?\nHistory Actions: SWIPE[UP],CLICK[0.6939,0.3797],CLICK[0.4973,0.4649],TYPE[Times Square],CLICK[0.4882,0.3044],CLICK[0.7003,0.8132]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_15_d1f3d38d7d8b4d38a34db95fc8f05ff7-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.3811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811]" + }, + { + "from": "gpt", + "value": "CLICK[0.6226,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811],CLICK[0.6226,0.5189]" + }, + { + "from": "gpt", + "value": "CLICK[0.6557,0.0578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811],CLICK[0.6226,0.5189],CLICK[0.6557,0.0578]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811],CLICK[0.6226,0.5189],CLICK[0.6557,0.0578],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811],CLICK[0.6226,0.5189],CLICK[0.6557,0.0578],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811],CLICK[0.6226,0.5189],CLICK[0.6557,0.0578],SWIPE[UP],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.5696,0.9561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the working place and set the departure time to 8:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6801,0.3811],CLICK[0.6226,0.5189],CLICK[0.6557,0.0578],SWIPE[UP],SWIPE[UP],SWIPE[DOWN],CLICK[0.5696,0.9561]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_16_34eb393968e947a7860d6d1af5fedd0d-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6841,0.3780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780]" + }, + { + "from": "gpt", + "value": "CLICK[0.5010,0.4625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780],CLICK[0.5010,0.4625]" + }, + { + "from": "gpt", + "value": "TYPE[Dumbo]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780],CLICK[0.5010,0.4625],TYPE[Dumbo]" + }, + { + "from": "gpt", + "value": "CLICK[0.4943,0.3003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780],CLICK[0.5010,0.4625],TYPE[Dumbo],CLICK[0.4943,0.3003]" + }, + { + "from": "gpt", + "value": "CLICK[0.6939,0.8176]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780],CLICK[0.5010,0.4625],TYPE[Dumbo],CLICK[0.4943,0.3003],CLICK[0.6939,0.8176]" + }, + { + "from": "gpt", + "value": "CLICK[0.5365,0.4027]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780],CLICK[0.5010,0.4625],TYPE[Dumbo],CLICK[0.4943,0.3003],CLICK[0.6939,0.8176],CLICK[0.5365,0.4027]" + }, + { + "from": "gpt", + "value": "CLICK[0.6834,0.5003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to Dumbo. Select the first recommended route and start nevigation.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3780],CLICK[0.5010,0.4625],TYPE[Dumbo],CLICK[0.4943,0.3003],CLICK[0.6939,0.8176],CLICK[0.5365,0.4027],CLICK[0.6834,0.5003]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_18_f860846252124706a88a1f1d0e077503-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6865,0.3767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767]" + }, + { + "from": "gpt", + "value": "CLICK[0.5020,0.4666]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666]" + }, + { + "from": "gpt", + "value": "CLICK[0.5189,0.1730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730]" + }, + { + "from": "gpt", + "value": "CLICK[0.4841,0.3429]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730],CLICK[0.4841,0.3429]" + }, + { + "from": "gpt", + "value": "CLICK[0.6595,0.0537]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730],CLICK[0.4841,0.3429],CLICK[0.6595,0.0537]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730],CLICK[0.4841,0.3429],CLICK[0.6595,0.0537],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730],CLICK[0.4841,0.3429],CLICK[0.6595,0.0537],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730],CLICK[0.4841,0.3429],CLICK[0.6595,0.0537],SWIPE[UP],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "CLICK[0.5720,0.9568]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from current location to the school address and set the departure time to 10:00 AM tomorrow.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3767],CLICK[0.5020,0.4666],CLICK[0.5189,0.1730],CLICK[0.4841,0.3429],CLICK[0.6595,0.0537],SWIPE[UP],SWIPE[UP],SWIPE[DOWN],CLICK[0.5720,0.9568]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_20_4abbc9ee431142d0a98a77b6a748e72a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.3791]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791]" + }, + { + "from": "gpt", + "value": "CLICK[0.4267,0.5280]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280]" + }, + { + "from": "gpt", + "value": "CLICK[0.5003,0.1169]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169]" + }, + { + "from": "gpt", + "value": "CLICK[0.4905,0.6402]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402]" + }, + { + "from": "gpt", + "value": "CLICK[0.6601,0.0507]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402],CLICK[0.6601,0.0507]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402],CLICK[0.6601,0.0507],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402],CLICK[0.6601,0.0507],SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402],CLICK[0.6601,0.0507],SWIPE[UP],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402],CLICK[0.6601,0.0507],SWIPE[UP],SWIPE[UP],SWIPE[DOWN],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5666,0.9544]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from the working place to home and set the departure time to 6:00 PM next monday.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3791],CLICK[0.4267,0.5280],CLICK[0.5003,0.1169],CLICK[0.4905,0.6402],CLICK[0.6601,0.0507],SWIPE[UP],SWIPE[UP],SWIPE[DOWN],SWIPE[UP],CLICK[0.5666,0.9544]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_22_22bc16f26d27451e84ff558f543d803b-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest Citi Bike station that has spaces available.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_25_1bb93c3f84cd4fe990bfeef543fe368c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest Citi Bike station that has spaces available.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6909,0.3834]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_25_1bb93c3f84cd4fe990bfeef543fe368c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest Citi Bike station that has spaces available.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3834]" + }, + { + "from": "gpt", + "value": "CLICK[0.4943,0.5973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_25_1bb93c3f84cd4fe990bfeef543fe368c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest Citi Bike station that has spaces available.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3834],CLICK[0.4943,0.5973]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_25_1bb93c3f84cd4fe990bfeef543fe368c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest Citi Bike station that has spaces available.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3834],CLICK[0.4943,0.5973],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_25_1bb93c3f84cd4fe990bfeef543fe368c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check the current issues of the subway lines. Read more if line 1 has any issues.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_27_962363f8d8cd4f60bf2ea2d0f578d023-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check the current issues of the subway lines. Read more if line 1 has any issues.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6919,0.3834]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_27_962363f8d8cd4f60bf2ea2d0f578d023-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check the current issues of the subway lines. Read more if line 1 has any issues.\nHistory Actions: SWIPE[UP],CLICK[0.6919,0.3834]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.5939]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_27_962363f8d8cd4f60bf2ea2d0f578d023-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check the current issues of the subway lines. Read more if line 1 has any issues.\nHistory Actions: SWIPE[UP],CLICK[0.6919,0.3834],CLICK[0.6872,0.5939]" + }, + { + "from": "gpt", + "value": "CLICK[0.6686,0.8791]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_27_962363f8d8cd4f60bf2ea2d0f578d023-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check the current issues of the subway lines. Read more if line 1 has any issues.\nHistory Actions: SWIPE[UP],CLICK[0.6919,0.3834],CLICK[0.6872,0.5939],CLICK[0.6686,0.8791]" + }, + { + "from": "gpt", + "value": "CLICK[0.6155,0.1128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_27_962363f8d8cd4f60bf2ea2d0f578d023-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check the current issues of the subway lines. Read more if line 1 has any issues.\nHistory Actions: SWIPE[UP],CLICK[0.6919,0.3834],CLICK[0.6872,0.5939],CLICK[0.6686,0.8791],CLICK[0.6155,0.1128]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_27_962363f8d8cd4f60bf2ea2d0f578d023-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6848,0.3791]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP],CLICK[0.6848,0.3791]" + }, + { + "from": "gpt", + "value": "CLICK[0.4821,0.4659]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP],CLICK[0.6848,0.3791],CLICK[0.4821,0.4659]" + }, + { + "from": "gpt", + "value": "TYPE[starbucks]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP],CLICK[0.6848,0.3791],CLICK[0.4821,0.4659],TYPE[starbucks]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.1723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP],CLICK[0.6848,0.3791],CLICK[0.4821,0.4659],TYPE[starbucks],CLICK[0.6632,0.1723]" + }, + { + "from": "gpt", + "value": "CLICK[0.6841,0.0632]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP],CLICK[0.6848,0.3791],CLICK[0.4821,0.4659],TYPE[starbucks],CLICK[0.6632,0.1723],CLICK[0.6841,0.0632]" + }, + { + "from": "gpt", + "value": "CLICK[0.7010,0.8145]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for Starbucks and find out how long it takes to walk there.\nHistory Actions: SWIPE[UP],CLICK[0.6848,0.3791],CLICK[0.4821,0.4659],TYPE[starbucks],CLICK[0.6632,0.1723],CLICK[0.6841,0.0632],CLICK[0.7010,0.8145]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_28_bbccf8b53d224d089fe0fbdc66665fd7-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6855,0.3750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750]" + }, + { + "from": "gpt", + "value": "CLICK[0.6963,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189]" + }, + { + "from": "gpt", + "value": "TYPE[Columbia University]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189],TYPE[Columbia University]" + }, + { + "from": "gpt", + "value": "CLICK[0.4990,0.2973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189],TYPE[Columbia University],CLICK[0.4990,0.2973]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.9628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189],TYPE[Columbia University],CLICK[0.4990,0.2973],CLICK[0.6993,0.9628]" + }, + { + "from": "gpt", + "value": "TYPE[CU]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189],TYPE[Columbia University],CLICK[0.4990,0.2973],CLICK[0.6993,0.9628],TYPE[CU]" + }, + { + "from": "gpt", + "value": "CLICK[0.6358,0.5618]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189],TYPE[Columbia University],CLICK[0.4990,0.2973],CLICK[0.6993,0.9628],TYPE[CU],CLICK[0.6358,0.5618]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.7986]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Columbia University as a favorite place and name it CU.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3750],CLICK[0.6963,0.5189],TYPE[Columbia University],CLICK[0.4990,0.2973],CLICK[0.6993,0.9628],TYPE[CU],CLICK[0.6358,0.5618],CLICK[0.6993,0.7986]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_2_95185451ded04c7880fd032a29e86bc4-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a lyft ride from current location.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_37_7a7f99a9f23e4e97a2fd02d6a47df325-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a lyft ride from current location.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6885,0.3750]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_37_7a7f99a9f23e4e97a2fd02d6a47df325-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a lyft ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3750]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.6625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_37_7a7f99a9f23e4e97a2fd02d6a47df325-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a lyft ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3750],CLICK[0.6895,0.6625]" + }, + { + "from": "gpt", + "value": "CLICK[0.4051,0.7277]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_37_7a7f99a9f23e4e97a2fd02d6a47df325-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a lyft ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3750],CLICK[0.6895,0.6625],CLICK[0.4051,0.7277]" + }, + { + "from": "gpt", + "value": "CLICK[0.6240,0.5547]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_37_7a7f99a9f23e4e97a2fd02d6a47df325-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a lyft ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6885,0.3750],CLICK[0.6895,0.6625],CLICK[0.4051,0.7277],CLICK[0.6240,0.5547]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_37_7a7f99a9f23e4e97a2fd02d6a47df325-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a Uber ride from current location.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_38_8babea8816b540838b81819ebe759837-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a Uber ride from current location.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6855,0.3851]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_38_8babea8816b540838b81819ebe759837-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a Uber ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3851]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.6601]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_38_8babea8816b540838b81819ebe759837-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a Uber ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3851],CLICK[0.6878,0.6601]" + }, + { + "from": "gpt", + "value": "CLICK[0.4081,0.7264]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_38_8babea8816b540838b81819ebe759837-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Check out the waiting time for a Uber ride from current location.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3851],CLICK[0.6878,0.6601],CLICK[0.4081,0.7264]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_38_8babea8816b540838b81819ebe759837-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.3821]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3821]" + }, + { + "from": "gpt", + "value": "CLICK[0.6909,0.5233]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3821],CLICK[0.6909,0.5233]" + }, + { + "from": "gpt", + "value": "CLICK[0.4990,0.4672]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3821],CLICK[0.6909,0.5233],CLICK[0.4990,0.4672]" + }, + { + "from": "gpt", + "value": "TYPE[Rockefeller Center]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3821],CLICK[0.6909,0.5233],CLICK[0.4990,0.4672],TYPE[Rockefeller Center]" + }, + { + "from": "gpt", + "value": "CLICK[0.4936,0.1990]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3821],CLICK[0.6909,0.5233],CLICK[0.4990,0.4672],TYPE[Rockefeller Center],CLICK[0.4936,0.1990]" + }, + { + "from": "gpt", + "value": "CLICK[0.6949,0.8024]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Rockefeller Center as a favorite place.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3821],CLICK[0.6909,0.5233],CLICK[0.4990,0.4672],TYPE[Rockefeller Center],CLICK[0.4936,0.1990],CLICK[0.6949,0.8024]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_39_75120cb62add47048413c0801f6fb03e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Open the maps of the Brooklyn Buses.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_41_b148d010e69047829c4334cb453f79c1-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Open the maps of the Brooklyn Buses.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.3791]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_41_b148d010e69047829c4334cb453f79c1-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Open the maps of the Brooklyn Buses.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3791]" + }, + { + "from": "gpt", + "value": "CLICK[0.5949,0.5973]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_41_b148d010e69047829c4334cb453f79c1-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Open the maps of the Brooklyn Buses.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3791],CLICK[0.5949,0.5973]" + }, + { + "from": "gpt", + "value": "CLICK[0.7155,0.0584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_41_b148d010e69047829c4334cb453f79c1-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Open the maps of the Brooklyn Buses.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3791],CLICK[0.5949,0.5973],CLICK[0.7155,0.0584]" + }, + { + "from": "gpt", + "value": "CLICK[0.4172,0.5155]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_41_b148d010e69047829c4334cb453f79c1-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Open the maps of the Brooklyn Buses.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3791],CLICK[0.5949,0.5973],CLICK[0.7155,0.0584],CLICK[0.4172,0.5155]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_41_b148d010e69047829c4334cb453f79c1-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest bus station.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_42_c803dd8804be4a1f88fa94a86182f640-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest bus station.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6841,0.3821]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_42_c803dd8804be4a1f88fa94a86182f640-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest bus station.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3821]" + }, + { + "from": "gpt", + "value": "CLICK[0.4098,0.6608]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_42_c803dd8804be4a1f88fa94a86182f640-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest bus station.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3821],CLICK[0.4098,0.6608]" + }, + { + "from": "gpt", + "value": "CLICK[0.5088,0.6757]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_42_c803dd8804be4a1f88fa94a86182f640-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest bus station.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3821],CLICK[0.4098,0.6608],CLICK[0.5088,0.6757]" + }, + { + "from": "gpt", + "value": "CLICK[0.6595,0.0561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_42_c803dd8804be4a1f88fa94a86182f640-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest bus station.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3821],CLICK[0.4098,0.6608],CLICK[0.5088,0.6757],CLICK[0.6595,0.0561]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_42_c803dd8804be4a1f88fa94a86182f640-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.3811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811]" + }, + { + "from": "gpt", + "value": "CLICK[0.3159,0.6003]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003]" + }, + { + "from": "gpt", + "value": "CLICK[0.5135,0.4318]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003],CLICK[0.5135,0.4318]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.0537]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003],CLICK[0.5135,0.4318],CLICK[0.7041,0.0537]" + }, + { + "from": "gpt", + "value": "CLICK[0.6480,0.1598]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003],CLICK[0.5135,0.4318],CLICK[0.7041,0.0537],CLICK[0.6480,0.1598]" + }, + { + "from": "gpt", + "value": "TYPE[MET]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003],CLICK[0.5135,0.4318],CLICK[0.7041,0.0537],CLICK[0.6480,0.1598],TYPE[MET]" + }, + { + "from": "gpt", + "value": "CLICK[0.4895,0.5649]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003],CLICK[0.5135,0.4318],CLICK[0.7041,0.0537],CLICK[0.6480,0.1598],TYPE[MET],CLICK[0.4895,0.5649]" + }, + { + "from": "gpt", + "value": "CLICK[0.6986,0.8091]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved bus station and search for a route from it to the MET.\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3811],CLICK[0.3159,0.6003],CLICK[0.5135,0.4318],CLICK[0.7041,0.0537],CLICK[0.6480,0.1598],TYPE[MET],CLICK[0.4895,0.5649],CLICK[0.6986,0.8091]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_43_5da5d5db72ae4759ad55b5599f94479b-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.3780]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3780]" + }, + { + "from": "gpt", + "value": "CLICK[0.3166,0.6595]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3780],CLICK[0.3166,0.6595]" + }, + { + "from": "gpt", + "value": "CLICK[0.5074,0.6777]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3780],CLICK[0.3166,0.6595],CLICK[0.5074,0.6777]" + }, + { + "from": "gpt", + "value": "CLICK[0.6588,0.0493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3780],CLICK[0.3166,0.6595],CLICK[0.5074,0.6777],CLICK[0.6588,0.0493]" + }, + { + "from": "gpt", + "value": "CLICK[0.3767,0.4395]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3780],CLICK[0.3166,0.6595],CLICK[0.5074,0.6777],CLICK[0.6588,0.0493],CLICK[0.3767,0.4395]" + }, + { + "from": "gpt", + "value": "CLICK[0.6679,0.6348]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the nearest subway station.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3780],CLICK[0.3166,0.6595],CLICK[0.5074,0.6777],CLICK[0.6588,0.0493],CLICK[0.3767,0.4395],CLICK[0.6679,0.6348]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_49_607396136fc74d49aa40df725140f26c-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6909,0.3804]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804]" + }, + { + "from": "gpt", + "value": "CLICK[0.3199,0.6017]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017]" + }, + { + "from": "gpt", + "value": "CLICK[0.5057,0.5510]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017],CLICK[0.5057,0.5510]" + }, + { + "from": "gpt", + "value": "CLICK[0.7071,0.0514]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017],CLICK[0.5057,0.5510],CLICK[0.7071,0.0514]" + }, + { + "from": "gpt", + "value": "CLICK[0.6334,0.1584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017],CLICK[0.5057,0.5510],CLICK[0.7071,0.0514],CLICK[0.6334,0.1584]" + }, + { + "from": "gpt", + "value": "TYPE[MOMA]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017],CLICK[0.5057,0.5510],CLICK[0.7071,0.0514],CLICK[0.6334,0.1584],TYPE[MOMA]" + }, + { + "from": "gpt", + "value": "CLICK[0.4882,0.2091]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017],CLICK[0.5057,0.5510],CLICK[0.7071,0.0514],CLICK[0.6334,0.1584],TYPE[MOMA],CLICK[0.4882,0.2091]" + }, + { + "from": "gpt", + "value": "CLICK[0.7024,0.8139]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the saved subway station and search for a route from it to MOMA.\nHistory Actions: SWIPE[UP],CLICK[0.6909,0.3804],CLICK[0.3199,0.6017],CLICK[0.5057,0.5510],CLICK[0.7071,0.0514],CLICK[0.6334,0.1584],TYPE[MOMA],CLICK[0.4882,0.2091],CLICK[0.7024,0.8139]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_51_9c1e6ed47cef486dbbf19529a09bedf0-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest E-scooter station. How long does it take to walk there?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_53_52c41c2923c54a8988156b42df474e7a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest E-scooter station. How long does it take to walk there?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.3797]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_53_52c41c2923c54a8988156b42df474e7a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest E-scooter station. How long does it take to walk there?\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3797]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.6618]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_53_52c41c2923c54a8988156b42df474e7a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest E-scooter station. How long does it take to walk there?\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3797],CLICK[0.6872,0.6618]" + }, + { + "from": "gpt", + "value": "CLICK[0.5095,0.7233]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_53_52c41c2923c54a8988156b42df474e7a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Find the nearest E-scooter station. How long does it take to walk there?\nHistory Actions: SWIPE[UP],CLICK[0.6895,0.3797],CLICK[0.6872,0.6618],CLICK[0.5095,0.7233]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_53_52c41c2923c54a8988156b42df474e7a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to Boston.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_54_2efe872096454071aa0b80012e458c24-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to Boston.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.3797]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_54_2efe872096454071aa0b80012e458c24-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to Boston.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3797]" + }, + { + "from": "gpt", + "value": "CLICK[0.2966,0.0530]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_54_2efe872096454071aa0b80012e458c24-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to Boston.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3797],CLICK[0.2966,0.0530]" + }, + { + "from": "gpt", + "value": "CLICK[0.6939,0.2466]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_54_2efe872096454071aa0b80012e458c24-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to Boston.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3797],CLICK[0.2966,0.0530],CLICK[0.6939,0.2466]" + }, + { + "from": "gpt", + "value": "CLICK[0.5034,0.5071]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_54_2efe872096454071aa0b80012e458c24-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to Boston.\nHistory Actions: SWIPE[UP],CLICK[0.6878,0.3797],CLICK[0.2966,0.0530],CLICK[0.6939,0.2466],CLICK[0.5034,0.5071]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_54_2efe872096454071aa0b80012e458c24-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to New York.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_55_de1ca1d238be43ccae5d011539d07376-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to New York.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.3811]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_55_de1ca1d238be43ccae5d011539d07376-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to New York.\nHistory Actions: SWIPE[UP],CLICK[0.6993,0.3811]" + }, + { + "from": "gpt", + "value": "CLICK[0.2936,0.0493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_55_de1ca1d238be43ccae5d011539d07376-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to New York.\nHistory Actions: SWIPE[UP],CLICK[0.6993,0.3811],CLICK[0.2936,0.0493]" + }, + { + "from": "gpt", + "value": "CLICK[0.6926,0.2514]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_55_de1ca1d238be43ccae5d011539d07376-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to New York.\nHistory Actions: SWIPE[UP],CLICK[0.6993,0.3811],CLICK[0.2936,0.0493],CLICK[0.6926,0.2514]" + }, + { + "from": "gpt", + "value": "CLICK[0.4841,0.3382]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_55_de1ca1d238be43ccae5d011539d07376-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Change the city to New York.\nHistory Actions: SWIPE[UP],CLICK[0.6993,0.3811],CLICK[0.2936,0.0493],CLICK[0.6926,0.2514],CLICK[0.4841,0.3382]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_55_de1ca1d238be43ccae5d011539d07376-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6855,0.3797]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3797]" + }, + { + "from": "gpt", + "value": "CLICK[0.6280,0.5189]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3797],CLICK[0.6280,0.5189]" + }, + { + "from": "gpt", + "value": "CLICK[0.4834,0.1213]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3797],CLICK[0.6280,0.5189],CLICK[0.4834,0.1213]" + }, + { + "from": "gpt", + "value": "CLICK[0.5142,0.2368]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3797],CLICK[0.6280,0.5189],CLICK[0.4834,0.1213],CLICK[0.5142,0.2368]" + }, + { + "from": "gpt", + "value": "CLICK[0.4821,0.4412]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from school to the working place.\nHistory Actions: SWIPE[UP],CLICK[0.6855,0.3797],CLICK[0.6280,0.5189],CLICK[0.4834,0.1213],CLICK[0.5142,0.2368],CLICK[0.4821,0.4412]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_56_9c6931ac563a406799cc79a9cb87dbad-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6841,0.3919]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3919]" + }, + { + "from": "gpt", + "value": "CLICK[0.3027,0.5226]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3919],CLICK[0.3027,0.5226]" + }, + { + "from": "gpt", + "value": "TYPE[255 W 94th St]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3919],CLICK[0.3027,0.5226],TYPE[255 W 94th St]" + }, + { + "from": "gpt", + "value": "CLICK[0.5172,0.1997]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3919],CLICK[0.3027,0.5226],TYPE[255 W 94th St],CLICK[0.5172,0.1997]" + }, + { + "from": "gpt", + "value": "CLICK[0.6993,0.8007]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save 255 W 94th St as the home address.\nHistory Actions: SWIPE[UP],CLICK[0.6841,0.3919],CLICK[0.3027,0.5226],TYPE[255 W 94th St],CLICK[0.5172,0.1997],CLICK[0.6993,0.8007]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_5_4eb4a259ddb24bd0a6d2063fc429d4d6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6872,0.3865]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865]" + }, + { + "from": "gpt", + "value": "CLICK[0.6980,0.5240]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240]" + }, + { + "from": "gpt", + "value": "CLICK[0.4997,0.4027]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027]" + }, + { + "from": "gpt", + "value": "TYPE[Courant Institute]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027],TYPE[Courant Institute]" + }, + { + "from": "gpt", + "value": "CLICK[0.5027,0.1983]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027],TYPE[Courant Institute],CLICK[0.5027,0.1983]" + }, + { + "from": "gpt", + "value": "CLICK[0.7003,0.9608]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027],TYPE[Courant Institute],CLICK[0.5027,0.1983],CLICK[0.7003,0.9608]" + }, + { + "from": "gpt", + "value": "TYPE[School]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027],TYPE[Courant Institute],CLICK[0.5027,0.1983],CLICK[0.7003,0.9608],TYPE[School]" + }, + { + "from": "gpt", + "value": "CLICK[0.6378,0.5625]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027],TYPE[Courant Institute],CLICK[0.5027,0.1983],CLICK[0.7003,0.9608],TYPE[School],CLICK[0.6378,0.5625]" + }, + { + "from": "gpt", + "value": "CLICK[0.7047,0.8024]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Save the Courant Institute as the school address.\nHistory Actions: SWIPE[UP],CLICK[0.6872,0.3865],CLICK[0.6980,0.5240],CLICK[0.4997,0.4027],TYPE[Courant Institute],CLICK[0.5027,0.1983],CLICK[0.7003,0.9608],TYPE[School],CLICK[0.6378,0.5625],CLICK[0.7047,0.8024]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_14_9_f6d574a2e3bc4552960585438b823a10-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6932,0.3804]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804]" + }, + { + "from": "gpt", + "value": "CLICK[0.4811,0.4611]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804],CLICK[0.4811,0.4611]" + }, + { + "from": "gpt", + "value": "TYPE[Washington Square Park]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804],CLICK[0.4811,0.4611],TYPE[Washington Square Park]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.2135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804],CLICK[0.4811,0.4611],TYPE[Washington Square Park],CLICK[0.4949,0.2135]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.8929]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804],CLICK[0.4811,0.4611],TYPE[Washington Square Park],CLICK[0.4949,0.2135],CLICK[0.4872,0.8929]" + }, + { + "from": "gpt", + "value": "TYPE[Rockefeller Center]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804],CLICK[0.4811,0.4611],TYPE[Washington Square Park],CLICK[0.4949,0.2135],CLICK[0.4872,0.8929],TYPE[Rockefeller Center]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.1723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from Rockefeller Center to the Washington Square Park.\nHistory Actions: SWIPE[UP],CLICK[0.6932,0.3804],CLICK[0.4811,0.4611],TYPE[Washington Square Park],CLICK[0.4949,0.2135],CLICK[0.4872,0.8929],TYPE[Rockefeller Center],CLICK[0.4949,0.1723]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_10_b2af4e1f133f4cdd8623b23fd69e1f94-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6865,0.3851]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851]" + }, + { + "from": "gpt", + "value": "CLICK[0.5051,0.4696]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696]" + }, + { + "from": "gpt", + "value": "CLICK[0.5027,0.1767]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767]" + }, + { + "from": "gpt", + "value": "CLICK[0.4649,0.4412]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767],CLICK[0.4649,0.4412]" + }, + { + "from": "gpt", + "value": "CLICK[0.6632,0.0561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767],CLICK[0.4649,0.4412],CLICK[0.6632,0.0561]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767],CLICK[0.4649,0.4412],CLICK[0.6632,0.0561],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[DOWN]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767],CLICK[0.4649,0.4412],CLICK[0.6632,0.0561],SWIPE[UP],SWIPE[DOWN]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767],CLICK[0.4649,0.4412],CLICK[0.6632,0.0561],SWIPE[UP],SWIPE[DOWN],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.5672,0.9459]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Citymapper. Search for a route from home to the saved CU address and set the departure time to 3:00 PM this Saturday.\nHistory Actions: SWIPE[UP],CLICK[0.6865,0.3851],CLICK[0.5051,0.4696],CLICK[0.5027,0.1767],CLICK[0.4649,0.4412],CLICK[0.6632,0.0561],SWIPE[UP],SWIPE[DOWN],SWIPE[UP],CLICK[0.5672,0.9459]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_20_15_12_6b56ebad54134bd1adf4baeba8e1d02a-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6926,0.3889]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889]" + }, + { + "from": "gpt", + "value": "CLICK[0.4997,0.1399]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4997,0.1399]" + }, + { + "from": "gpt", + "value": "CLICK[0.7162,0.9730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4997,0.1399],CLICK[0.7162,0.9730]" + }, + { + "from": "gpt", + "value": "CLICK[0.3260,0.8422]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4997,0.1399],CLICK[0.7162,0.9730],CLICK[0.3260,0.8422]" + }, + { + "from": "gpt", + "value": "CLICK[0.7149,0.9645]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4997,0.1399],CLICK[0.7162,0.9730],CLICK[0.3260,0.8422],CLICK[0.7149,0.9645]" + }, + { + "from": "gpt", + "value": "TYPE[how about this]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4997,0.1399],CLICK[0.7162,0.9730],CLICK[0.3260,0.8422],CLICK[0.7149,0.9645],TYPE[how about this]" + }, + { + "from": "gpt", + "value": "CLICK[0.7132,0.9676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the first photo in the phone to the group 'lunch' and send a message 'how about this'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4997,0.1399],CLICK[0.7162,0.9730],CLICK[0.3260,0.8422],CLICK[0.7149,0.9645],TYPE[how about this],CLICK[0.7132,0.9676]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_12_7c04b458b1bf4b299742b88d08603676-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch'?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_14_0d03683044b74c6282b0145da2ad63ef-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch'?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_14_0d03683044b74c6282b0145da2ad63ef-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch'?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6885,0.3943]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_14_0d03683044b74c6282b0145da2ad63ef-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch'?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3943]" + }, + { + "from": "gpt", + "value": "CLICK[0.4997,0.1206]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_14_0d03683044b74c6282b0145da2ad63ef-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch'?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3943],CLICK[0.4997,0.1206]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_14_0d03683044b74c6282b0145da2ad63ef-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6926,0.3875]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3875]" + }, + { + "from": "gpt", + "value": "CLICK[0.4872,0.1345]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3875],CLICK[0.4872,0.1345]" + }, + { + "from": "gpt", + "value": "CLICK[0.7118,0.9699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3875],CLICK[0.4872,0.1345],CLICK[0.7118,0.9699]" + }, + { + "from": "gpt", + "value": "CLICK[0.6358,0.9338]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3875],CLICK[0.4872,0.1345],CLICK[0.7118,0.9699],CLICK[0.6358,0.9338]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.9662]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3875],CLICK[0.4872,0.1345],CLICK[0.7118,0.9699],CLICK[0.6358,0.9338],CLICK[0.7041,0.9662]" + }, + { + "from": "gpt", + "value": "CLICK[0.7101,0.9723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the current location to the group 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3875],CLICK[0.4872,0.1345],CLICK[0.7118,0.9699],CLICK[0.6358,0.9338],CLICK[0.7041,0.9662],CLICK[0.7101,0.9723]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_15_d8d525d43fc24f8db82b3c71645d190a-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6885,0.3959]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3959]" + }, + { + "from": "gpt", + "value": "CLICK[0.4949,0.2997]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3959],CLICK[0.4949,0.2997]" + }, + { + "from": "gpt", + "value": "CLICK[0.7125,0.9730]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3959],CLICK[0.4949,0.2997],CLICK[0.7125,0.9730]" + }, + { + "from": "gpt", + "value": "CLICK[0.4611,0.9392]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3959],CLICK[0.4949,0.2997],CLICK[0.7125,0.9730],CLICK[0.4611,0.9392]" + }, + { + "from": "gpt", + "value": "CLICK[0.4895,0.2959]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3959],CLICK[0.4949,0.2997],CLICK[0.7125,0.9730],CLICK[0.4611,0.9392],CLICK[0.4895,0.2959]" + }, + { + "from": "gpt", + "value": "CLICK[0.7118,0.9682]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send the file 'alarmLog.txt' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6885,0.3959],CLICK[0.4949,0.2997],CLICK[0.7125,0.9730],CLICK[0.4611,0.9392],CLICK[0.4895,0.2959],CLICK[0.7118,0.9682]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_16_3a8ca8c769fc461fae1e9df6638e1f3f-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.3959]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3959]" + }, + { + "from": "gpt", + "value": "CLICK[0.5142,0.3037]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3959],CLICK[0.5142,0.3037]" + }, + { + "from": "gpt", + "value": "CLICK[0.6733,0.0561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3959],CLICK[0.5142,0.3037],CLICK[0.6733,0.0561]" + }, + { + "from": "gpt", + "value": "CLICK[0.6264,0.5395]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Voice call Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3959],CLICK[0.5142,0.3037],CLICK[0.6733,0.0561],CLICK[0.6264,0.5395]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_17_f763590921644f2e9507c7cc70c681e2-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.3895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895]" + }, + { + "from": "gpt", + "value": "CLICK[0.4966,0.2199]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.4966,0.2199]" + }, + { + "from": "gpt", + "value": "CLICK[0.6287,0.0584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.4966,0.2199],CLICK[0.6287,0.0584]" + }, + { + "from": "gpt", + "value": "CLICK[0.5064,0.8976]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Video call Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.4966,0.2199],CLICK[0.6287,0.0584],CLICK[0.5064,0.8976]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_18_68795fdd2cf94e749b8cf5cc260fcb3d-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6855,0.3912]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6855,0.3912]" + }, + { + "from": "gpt", + "value": "CLICK[0.5003,0.2145]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6855,0.3912],CLICK[0.5003,0.2145]" + }, + { + "from": "gpt", + "value": "CLICK[0.7240,0.0584]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6855,0.3912],CLICK[0.5003,0.2145],CLICK[0.7240,0.0584]" + }, + { + "from": "gpt", + "value": "CLICK[0.6125,0.3037]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6855,0.3912],CLICK[0.5003,0.2145],CLICK[0.7240,0.0584],CLICK[0.6125,0.3037]" + }, + { + "from": "gpt", + "value": "CLICK[0.4720,0.4443]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Mute the notifications for 1 hour from Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6855,0.3912],CLICK[0.5003,0.2145],CLICK[0.7240,0.0584],CLICK[0.6125,0.3037],CLICK[0.4720,0.4443]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_19_f136b15194ee408e9f5f00569c625095-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Open the emoji picker in the group chat 'lunch'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_09745085094a481c9ac0906ed8e7e796-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Open the emoji picker in the group chat 'lunch'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_09745085094a481c9ac0906ed8e7e796-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Open the emoji picker in the group chat 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6909,0.3912]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_09745085094a481c9ac0906ed8e7e796-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Open the emoji picker in the group chat 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.3912]" + }, + { + "from": "gpt", + "value": "CLICK[0.4851,0.3037]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_09745085094a481c9ac0906ed8e7e796-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Open the emoji picker in the group chat 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.3912],CLICK[0.4851,0.3037]" + }, + { + "from": "gpt", + "value": "CLICK[0.2936,0.9699]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_09745085094a481c9ac0906ed8e7e796-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Open the emoji picker in the group chat 'lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.3912],CLICK[0.4851,0.3037],CLICK[0.2936,0.9699]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_09745085094a481c9ac0906ed8e7e796-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6878,0.3895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895]" + }, + { + "from": "gpt", + "value": "CLICK[0.6655,0.9561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561]" + }, + { + "from": "gpt", + "value": "CLICK[0.4726,0.1291]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291]" + }, + { + "from": "gpt", + "value": "CLICK[0.6780,0.8361]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361]" + }, + { + "from": "gpt", + "value": "CLICK[0.3821,0.2338]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338]" + }, + { + "from": "gpt", + "value": "CLICK[0.3260,0.1436]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436]" + }, + { + "from": "gpt", + "value": "CLICK[0.6980,0.9693]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693]" + }, + { + "from": "gpt", + "value": "CLICK[0.4389,0.9693]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693]" + }, + { + "from": "gpt", + "value": "TYPE[good lunch]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch]" + }, + { + "from": "gpt", + "value": "CLICK[0.7064,0.9706]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch],CLICK[0.7064,0.9706]" + }, + { + "from": "gpt", + "value": "CLICK[0.7078,0.9628]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch],CLICK[0.7064,0.9706],CLICK[0.7078,0.9628]" + }, + { + "from": "gpt", + "value": "CLICK[0.4943,0.4473]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch],CLICK[0.7064,0.9706],CLICK[0.7078,0.9628],CLICK[0.4943,0.4473]" + }, + { + "from": "gpt", + "value": "CLICK[0.4980,0.7493]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-15.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch],CLICK[0.7064,0.9706],CLICK[0.7078,0.9628],CLICK[0.4943,0.4473],CLICK[0.4980,0.7493]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.9338]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-16.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch],CLICK[0.7064,0.9706],CLICK[0.7078,0.9628],CLICK[0.4943,0.4473],CLICK[0.4980,0.7493],CLICK[0.6801,0.9338]" + }, + { + "from": "gpt", + "value": "CLICK[0.7108,0.9676]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-17.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a new story with the latest photo in the phone and add the text as 'good lunch'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6878,0.3895],CLICK[0.6655,0.9561],CLICK[0.4726,0.1291],CLICK[0.6780,0.8361],CLICK[0.3821,0.2338],CLICK[0.3260,0.1436],CLICK[0.6980,0.9693],CLICK[0.4389,0.9693],TYPE[good lunch],CLICK[0.7064,0.9706],CLICK[0.7078,0.9628],CLICK[0.4943,0.4473],CLICK[0.4980,0.7493],CLICK[0.6801,0.9338],CLICK[0.7108,0.9676]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_20_5d6f66282db64af99e721c134692e464-18.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch' and who is it from?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_24_ba5e5178117d45c7b39054d96be0c423-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch' and who is it from?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_24_ba5e5178117d45c7b39054d96be0c423-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch' and who is it from?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6902,0.3905]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_24_ba5e5178117d45c7b39054d96be0c423-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch' and who is it from?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6902,0.3905]" + }, + { + "from": "gpt", + "value": "CLICK[0.5051,0.1399]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_24_ba5e5178117d45c7b39054d96be0c423-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from the group 'lunch' and who is it from?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6902,0.3905],CLICK[0.5051,0.1399]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_24_ba5e5178117d45c7b39054d96be0c423-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6932,0.3889]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3889]" + }, + { + "from": "gpt", + "value": "CLICK[0.5081,0.1321]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3889],CLICK[0.5081,0.1321]" + }, + { + "from": "gpt", + "value": "SWIPE[RIGHT]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3889],CLICK[0.5081,0.1321],SWIPE[RIGHT]" + }, + { + "from": "gpt", + "value": "TYPE[good]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3889],CLICK[0.5081,0.1321],SWIPE[RIGHT],TYPE[good]" + }, + { + "from": "gpt", + "value": "CLICK[0.7125,0.9713]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Reply to the last message from the group 'lunch' with 'good'.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3889],CLICK[0.5081,0.1321],SWIPE[RIGHT],TYPE[good],CLICK[0.7125,0.9713]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_25_a7b222bdbbf14370b10da959f789a54b-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.3895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3895]" + }, + { + "from": "gpt", + "value": "CLICK[0.6801,0.0547]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3895],CLICK[0.6801,0.0547]" + }, + { + "from": "gpt", + "value": "TYPE[1pm]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3895],CLICK[0.6801,0.0547],TYPE[1pm]" + }, + { + "from": "gpt", + "value": "CLICK[0.5172,0.1868]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Search for the message '1pm' and select the latest message.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3895],CLICK[0.6801,0.0547],TYPE[1pm],CLICK[0.5172,0.1868]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_26_7368ef9581c64efabfdde15452a686f6-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6794,0.3895]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895]" + }, + { + "from": "gpt", + "value": "CLICK[0.4980,0.2135]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895],CLICK[0.4980,0.2135]" + }, + { + "from": "gpt", + "value": "CLICK[0.7149,0.0554]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895],CLICK[0.4980,0.2135],CLICK[0.7149,0.0554]" + }, + { + "from": "gpt", + "value": "CLICK[0.6209,0.1537]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895],CLICK[0.4980,0.2135],CLICK[0.7149,0.0554],CLICK[0.6209,0.1537]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895],CLICK[0.4980,0.2135],CLICK[0.7149,0.0554],CLICK[0.6209,0.1537],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.3966,0.9152]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895],CLICK[0.4980,0.2135],CLICK[0.7149,0.0554],CLICK[0.6209,0.1537],SWIPE[UP],CLICK[0.3966,0.9152]" + }, + { + "from": "gpt", + "value": "CLICK[0.6480,0.5696]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Block the chat with Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6794,0.3895],CLICK[0.4980,0.2135],CLICK[0.7149,0.0554],CLICK[0.6209,0.1537],SWIPE[UP],CLICK[0.3966,0.9152],CLICK[0.6480,0.5696]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_27_2f636591b50d4378b810b4a39440e21e-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6919,0.3936]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936]" + }, + { + "from": "gpt", + "value": "CLICK[0.4973,0.1213]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213]" + }, + { + "from": "gpt", + "value": "TYPE[I will delete the group]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group]" + }, + { + "from": "gpt", + "value": "CLICK[0.7172,0.9760]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760]" + }, + { + "from": "gpt", + "value": "CLICK[0.7142,0.0547]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547]" + }, + { + "from": "gpt", + "value": "CLICK[0.6270,0.1561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547],CLICK[0.6270,0.1561]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547],CLICK[0.6270,0.1561],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.4736,0.8578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547],CLICK[0.6270,0.1561],SWIPE[UP],CLICK[0.4736,0.8578]" + }, + { + "from": "gpt", + "value": "CLICK[0.6166,0.5726]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547],CLICK[0.6270,0.1561],SWIPE[UP],CLICK[0.4736,0.8578],CLICK[0.6166,0.5726]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.1128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547],CLICK[0.6270,0.1561],SWIPE[UP],CLICK[0.4736,0.8578],CLICK[0.6166,0.5726],CLICK[0.7041,0.1128]" + }, + { + "from": "gpt", + "value": "CLICK[0.6811,0.9662]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-13.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will delete the group' in 'lunch' group and leave the group 'lunch' and transfer the admin to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6919,0.3936],CLICK[0.4973,0.1213],TYPE[I will delete the group],CLICK[0.7172,0.9760],CLICK[0.7142,0.0547],CLICK[0.6270,0.1561],SWIPE[UP],CLICK[0.4736,0.8578],CLICK[0.6166,0.5726],CLICK[0.7041,0.1128],CLICK[0.6811,0.9662]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_29_abcc563b6c0c414a96bbdf0cb414be11-14.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6926,0.3889]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889]" + }, + { + "from": "gpt", + "value": "CLICK[0.4966,0.2128]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4966,0.2128]" + }, + { + "from": "gpt", + "value": "TYPE[lunch at 1pm?]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4966,0.2128],TYPE[lunch at 1pm?]" + }, + { + "from": "gpt", + "value": "CLICK[0.7108,0.9723]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'lunch at 1pm?' to Johnson.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6926,0.3889],CLICK[0.4966,0.2128],TYPE[lunch at 1pm?],CLICK[0.7108,0.9723]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_4_c0e80d21552445d3bd0e2ad2eb8c0cf8-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6895,0.3936]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3936]" + }, + { + "from": "gpt", + "value": "CLICK[0.4821,0.2115]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3936],CLICK[0.4821,0.2115]" + }, + { + "from": "gpt", + "value": "TYPE[he is available]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3936],CLICK[0.4821,0.2115],TYPE[he is available]" + }, + { + "from": "gpt", + "value": "CLICK[0.7095,0.9713]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'he is available' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6895,0.3936],CLICK[0.4821,0.2115],TYPE[he is available],CLICK[0.7095,0.9713]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_5_29e2217a6ed84579b5691dd8077b6551-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6909,0.4720]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.4720]" + }, + { + "from": "gpt", + "value": "CLICK[0.4672,0.1213]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.4720],CLICK[0.4672,0.1213]" + }, + { + "from": "gpt", + "value": "TYPE[I will be there]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.4720],CLICK[0.4672,0.1213],TYPE[I will be there]" + }, + { + "from": "gpt", + "value": "CLICK[0.7057,0.9662]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Send a message 'I will be there' to Bruce.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6909,0.4720],CLICK[0.4672,0.1213],TYPE[I will be there],CLICK[0.7057,0.9662]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_6_86eaa76400234482a23d8f0866483fc0-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from Bruce?\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_8_b6779b7b382e4a558ea0fb0fb21f41b3-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from Bruce?\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_8_b6779b7b382e4a558ea0fb0fb21f41b3-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from Bruce?\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6932,0.3919]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_8_b6779b7b382e4a558ea0fb0fb21f41b3-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from Bruce?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3919]" + }, + { + "from": "gpt", + "value": "CLICK[0.4966,0.1375]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_8_b6779b7b382e4a558ea0fb0fb21f41b3-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. What is the last message from Bruce?\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6932,0.3919],CLICK[0.4966,0.1375]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_8_b6779b7b382e4a558ea0fb0fb21f41b3-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: " + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-1.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP]" + }, + { + "from": "gpt", + "value": "SWIPE[UP]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-2.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP]" + }, + { + "from": "gpt", + "value": "CLICK[0.6963,0.3936]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-3.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936]" + }, + { + "from": "gpt", + "value": "CLICK[0.7226,0.0578]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-4.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578]" + }, + { + "from": "gpt", + "value": "CLICK[0.6240,0.1098]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-5.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098]" + }, + { + "from": "gpt", + "value": "CLICK[0.7041,0.4182]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-6.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182]" + }, + { + "from": "gpt", + "value": "CLICK[0.7101,0.5787]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-7.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182],CLICK[0.7101,0.5787]" + }, + { + "from": "gpt", + "value": "CLICK[0.7078,0.9561]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-8.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182],CLICK[0.7101,0.5787],CLICK[0.7078,0.9561]" + }, + { + "from": "gpt", + "value": "TYPE[lunch]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-9.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182],CLICK[0.7101,0.5787],CLICK[0.7078,0.9561],TYPE[lunch]" + }, + { + "from": "gpt", + "value": "CLICK[0.6855,0.9608]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-10.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182],CLICK[0.7101,0.5787],CLICK[0.7078,0.9561],TYPE[lunch],CLICK[0.6855,0.9608]" + }, + { + "from": "gpt", + "value": "TYPE[this is the group]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-11.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182],CLICK[0.7101,0.5787],CLICK[0.7078,0.9561],TYPE[lunch],CLICK[0.6855,0.9608],TYPE[this is the group]" + }, + { + "from": "gpt", + "value": "CLICK[0.7088,0.9747]" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-12.png" + }, + { + "conversations": [ + { + "from": "human", + "value": "Given the task instruction, please specify the action to take on the current screen.\nTask: Open Signal. Create a group named 'lunch' with Johnson and Bruce. And send a message 'this is the group' to the group.\nHistory Actions: SWIPE[UP],SWIPE[UP],CLICK[0.6963,0.3936],CLICK[0.7226,0.0578],CLICK[0.6240,0.1098],CLICK[0.7041,0.4182],CLICK[0.7101,0.5787],CLICK[0.7078,0.9561],TYPE[lunch],CLICK[0.6855,0.9608],TYPE[this is the group],CLICK[0.7088,0.9747]" + }, + { + "from": "gpt", + "value": "TASK_COMPLETE" + } + ], + "image": "/data/zbz5349/ICLR_2024/ACL_2025/DataSet/AMEX/AMEX/screenshot/2024_6_21_14_9_adda7bc387a54ccc841cb9802ead7493-13.png" + } +] \ No newline at end of file