% Key papers for Software engineering agent benchmarks (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages. @inproceedings{aaai2026_u2udata_100107, title = {{U2UData+: A Scalable Swarm UAVs Autonomous Flight Dataset for Embodied Long-horizon Tasks}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2026}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/37158}, } @inproceedings{aaai2026_mcptox_108543, title = {{MCPTox: A Benchmark for Tool Poisoning on Real-World MCP Servers}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2026}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/40895}, } @inproceedings{neurips2023_de_66584, title = {{De novo Drug Design using Reinforcement Learning with Multiple GPT Agents}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2023}, url = {https://nips.cc/virtual/2023/poster/73033}, } @inproceedings{icml2025_itbench_48311, title = {{ITBench: Evaluating AI Agents across Diverse Real-World IT Automation Tasks}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2025}, url = {https://icml.cc/virtual/2025/poster/44303}, } @inproceedings{neurips2025_wasp_76400, title = {{WASP: Benchmarking Web Agent Security Against Prompt Injection Attacks}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2025}, url = {https://openreview.net/forum?id=Ip1cCUAllL}, } @inproceedings{icml2025_cradle_46167, title = {{Cradle: Empowering Foundation Agents towards General Computer Control}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2025}, url = {https://icml.cc/virtual/2025/poster/46393}, } @inproceedings{neurips2025_os_75923, title = {{OS-Harm: A Benchmark for Measuring Safety of Computer Use Agents}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2025}, url = {https://openreview.net/forum?id=Di30GwhQSX}, } @inproceedings{iclr2025_swe_83640, title = {{SWE-bench Multimodal: Do AI Systems Generalize to Visual Software Domains?}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, url = {https://iclr.cc/virtual/2025/poster/28177}, } @inproceedings{aaai2026_gittaskbench_108181, title = {{GitTaskBench: A Benchmark for Code Agents Solving Real-World Tasks Through Code Repository Leveraging}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2026}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/40533}, } @inproceedings{icml2025_re_46027, title = {{RE-Bench: Evaluating Frontier AI R\&D Capabilities of Language Model Agents against Human Experts}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2025}, url = {https://icml.cc/virtual/2025/poster/46519}, } @inproceedings{neurips2025_stratus_78509, title = {{STRATUS: A Multi-agent System for Autonomous Reliability Engineering of Modern Clouds}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2025}, url = {https://openreview.net/forum?id=fYW1PKawwJ}, } @inproceedings{iclr2026_cybergym_84430, title = {{CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2026}, url = {https://openreview.net/forum?id=2YvbLQEdYt}, } @inproceedings{aaai2026_is_108528, title = {{IS-Bench: Evaluating Interactive Safety of VLM-Driven Embodied Agents in Daily Household Tasks}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2026}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/40880}, } @inproceedings{icml2024_anytool_45450, title = {{AnyTool: Self-Reflective, Hierarchical Agents for Large-Scale API Calls}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2024}, url = {https://icml.cc/virtual/2024/poster/33001}, } @inproceedings{icml2025_syncmind_46188, title = {{SyncMind: Measuring Agent Out-of-Sync Recovery in Collaborative Software Engineering}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2025}, url = {https://icml.cc/virtual/2025/poster/46372}, } @inproceedings{icml2025_training_46530, title = {{Training Software Engineering Agents and Verifiers with SWE-Gym}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2025}, url = {https://icml.cc/virtual/2025/poster/46038}, } @inproceedings{icml2026_is_54570, title = {{Is Vibe Coding Safe? Benchmarking Vulnerability of Agent-Generated Code in Real-World Tasks}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2026}, url = {https://openreview.net/forum?id=qG8g00zRZa}, } @inproceedings{neurips2025_repo2run_78511, title = {{Repo2Run: Automated Building Executable Environment for Code Repository at Scale}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2025}, url = {https://openreview.net/forum?id=fZsd3KLMje}, } @inproceedings{neurips2025_bountybench_79430, title = {{BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2025}, url = {https://openreview.net/forum?id=pIsP4lMlFd}, } @inproceedings{iclr2025_eia_84036, title = {{EIA: ENVIRONMENTAL INJECTION ATTACK ON GENERALIST WEB AGENTS FOR PRIVACY LEAKAGE}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, url = {https://iclr.cc/virtual/2025/poster/27789}, } @inproceedings{iclr2026_st_86182, title = {{ST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2026}, url = {https://openreview.net/forum?id=MuCDzH0ctf}, }