% Key papers for Code generation benchmarks (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages. @inproceedings{neurips2023_self_68156, title = {{Self-Refine: Iterative Refinement with Self-Feedback}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2023}, url = {https://nips.cc/virtual/2023/poster/71632}, } @inproceedings{iclr2023_codegen_33024, title = {{CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2023}, url = {https://iclr.cc/virtual/2023/poster/11335}, } @inproceedings{aaai2023_repair_4507, title = {{Repair Is Nearly Generation: Multilingual Program Repair with LLMs}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2023}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/25642}, } @inproceedings{aaai2024_can_5369, title = {{Can LLM Replace Stack Overflow? A Study on Robustness and Reliability of Large Language Model Code Generation}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2024}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/30185}, } @inproceedings{icml2023_ds_42444, title = {{DS-1000: A Natural and Reliable Benchmark for Data Science Code Generation}}, booktitle = {International Conference on Machine Learning (ICML)}, year = {2023}, url = {https://icml.cc/virtual/2023/poster/25042}, } @inproceedings{neurips2023_mariogpt_68648, title = {{MarioGPT: Open-Ended Text2Level Generation through Large Language Models}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2023}, url = {https://nips.cc/virtual/2023/poster/71191}, } @inproceedings{iclr2025_bigcodebench_82551, title = {{BigCodeBench: Benchmarking Code Generation with Diverse Function Calls and Complex Instructions}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, url = {https://iclr.cc/virtual/2025/poster/29245}, } @inproceedings{aaai2025_codehalu_8442, title = {{CodeHalu: Investigating Code Hallucinations in LLMs via Execution-based Verification}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2025}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/34717}, } @inproceedings{aaai2025_can_8379, title = {{Can LLMs Obfuscate Code? A Systematic Analysis of Large Language Models into Assembly Code Obfuscation}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2025}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/34672}, } @inproceedings{neurips2022_fault_64684, title = {{Fault-Aware Neural Code Rankers}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2022}, url = {https://nips.cc/virtual/2022/poster/53688}, } @inproceedings{neurips2024_semcoder_72031, title = {{SemCoder: Training Code Language Models with Comprehensive Semantics Reasoning}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2024}, url = {https://neurips.cc/virtual/2024/poster/95287}, } @inproceedings{neurips2021_latent_62597, title = {{Latent Execution for Neural Program Synthesis Beyond Domain-Specific Languages}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2021}, url = {https://nips.cc/virtual/2021/poster/28205}, } @inproceedings{neurips2024_gavel_73870, title = {{GAVEL: Generating Games via Evolution and Language Models}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2024}, url = {https://neurips.cc/virtual/2024/poster/93639}, } @inproceedings{aaai2024_benchmarking_5278, title = {{Benchmarking Large Language Models on Controllable Generation under Diversified Instructions}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2024}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/29734}, } @inproceedings{aaai2025_domaineval_8617, title = {{DOMAINEVAL: An Auto-Constructed Benchmark for Multi-Domain Code Generation}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2025}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/34811}, } @inproceedings{cvpr2025_blendergym_25596, title = {{BlenderGym: Benchmarking Foundational Model Systems for Graphics Editing}}, booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, year = {2025}, url = {https://cvpr.thecvf.com/virtual/2025/poster/33899}, } @inproceedings{neurips2020_program_60624, title = {{Program Synthesis with Pragmatic Communication}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2020}, url = {https://nips.cc/virtual/2020/poster/18171}, } @inproceedings{neurips2021_pipeline_63493, title = {{Pipeline Combinators for Gradual AutoML}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2021}, url = {https://nips.cc/virtual/2021/poster/25952}, } @inproceedings{aaai2025_text2midi_10961, title = {{Text2midi: Generating Symbolic Music from Captions}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2025}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/34516}, } @inproceedings{neurips2024_ledex_73059, title = {{LeDex: Training LLMs to Better Self-Debug and Explain Code}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2024}, url = {https://neurips.cc/virtual/2024/poster/94367}, } @inproceedings{cvpr2025_postero_27286, title = {{PosterO: Structuring Layout Trees to Enable Language Models in Generalized Content-Aware Layout Generation}}, booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, year = {2025}, url = {https://cvpr.thecvf.com/virtual/2025/poster/35211}, } @inproceedings{iclr2024_is_35644, title = {{Is Self-Repair a Silver Bullet for Code Generation?}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2024}, url = {https://iclr.cc/virtual/2024/poster/17429}, } @inproceedings{iclr2024_mathcoder_35684, title = {{MathCoder: Seamless Code Integration in LLMs for Enhanced Mathematical Reasoning}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2024}, url = {https://iclr.cc/virtual/2024/poster/17389}, } @inproceedings{neurips2024_code_73867, title = {{Code Repair with LLMs gives an Exploration-Exploitation Tradeoff}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2024}, url = {https://neurips.cc/virtual/2024/poster/93642}, } @inproceedings{neurips2024_stackeval_70402, title = {{StackEval: Benchmarking LLMs in Coding Assistance}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2024}, url = {https://neurips.cc/virtual/2024/poster/97856}, } @inproceedings{iclr2025_mupt_83094, title = {{MuPT: A Generative Symbolic Music Pretrained Transformer}}, booktitle = {International Conference on Learning Representations (ICLR)}, year = {2025}, url = {https://iclr.cc/virtual/2025/poster/28712}, } @inproceedings{aaai2024_bridging_5316, title = {{Bridging the Gap between Source Code and Requirements Using GPT (Student Abstract)}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2024}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/30526}, } @inproceedings{aaai2025_counterexample_8546, title = {{Counterexample Guided Program Repair Using Zero-Shot Learning and MaxSAT-based Fault Localization}}, booktitle = {AAAI Conference on Artificial Intelligence (AAAI)}, year = {2025}, url = {https://ojs.aaai.org/index.php/AAAI/article/view/32046}, } @inproceedings{neurips2024_evocodebench_73629, title = {{EvoCodeBench: An Evolving Code Generation Benchmark with Domain-Specific Evaluations}}, booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year = {2024}, url = {https://neurips.cc/virtual/2024/poster/97531}, }