% Key papers for Attention head mechanisms (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{neurips2023_representational_66708,
  title = {{Representational Strengths and Limitations of Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/72920},
}

@inproceedings{icml2024_attnlrp_44972,
  title = {{AttnLRP: Attention-Aware Layer-Wise Relevance Propagation for Transformers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33480},
}

@inproceedings{iclr2024_analyzing_35182,
  title = {{Analyzing Feed-Forward Blocks in Transformers through the Lens of Attention Maps}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/17891},
}

@inproceedings{iclr2024_abstractors_34623,
  title = {{Abstractors and relational cross-attention: An inductive bias for explicit relational reasoning in Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18451},
}

@inproceedings{aaai2021_on_1090,
  title = {{On Scalar Embedding of Relative Positions in Attention Models}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2021},
  url = {https://aaai.org/papers/14050-on-scalar-embedding-of-relative-positions-in-attention-models/},
}

@inproceedings{neurips2022_attention_64169,
  title = {{Attention-based Neural Cellular Automata}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/54709},
}

@inproceedings{neurips2024_evidence_70793,
  title = {{Evidence of Learned Look-Ahead in a Chess-Playing Neural Network}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96336},
}

@inproceedings{icml2024_improving_44405,
  title = {{Improving Transformers with Dynamically Composable Multi-Head Attention}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34047},
}

@inproceedings{aaai2025_on_10180,
  title = {{On the Power of Convolution-Augmented Transformer}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/34024},
}

@inproceedings{iclr2024_memorization_34268,
  title = {{Memorization Capacity of Multi-Head Attention in Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18807},
}

@inproceedings{icml2024_transformers_45758,
  title = {{Transformers Learn Nonlinear Features In Context: Nonconvex Mean-field Dynamics on the Attention Landscape}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/32694},
}

@inproceedings{icml2025_fourier_47762,
  title = {{Fourier Position Embedding: Enhancing Attention’s Periodic Extension for Length Generalization}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/44838},
}

@inproceedings{icml2025_mind_48785,
  title = {{Mind the Gap: a Spectral Analysis of Rank Collapse and Signal Propagation in Attention Layers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/43837},
}

@inproceedings{neurips2024_talking_71695,
  title = {{Talking Heads: Understanding Inter-Layer Communication in Transformer Language Models}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95588},
}

@inproceedings{neurips2025_memory_76382,
  title = {{Memory Mosaics at scale}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=IfD2MKTmWv},
}

@inproceedings{neurips2025_what_76934,
  title = {{What are you sinking? A geometric approach on attention sink}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=OiC78C68sJ},
}

@inproceedings{neurips2025_the_78887,
  title = {{The emergence of sparse attention: impact of data distribution and benefits of repetition}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=jMhRbV47pS},
}

@inproceedings{iclr2024_stack_34626,
  title = {{Stack Attention: Improving the Ability of Transformers to Model Hierarchical Patterns}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18448},
}

@inproceedings{icml2023_monotonic_41629,
  title = {{Monotonic Location Attention for Length Generalization}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2023},
  url = {https://icml.cc/virtual/2023/poster/23965},
}

@inproceedings{icml2024_towards_44059,
  title = {{Towards Understanding the Word Sensitivity of Attention Layers: A Study via Random Features}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34393},
}

@inproceedings{icml2025_positional_45852,
  title = {{Positional Attention: Expressivity and Learnability of Algorithmic Computation}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46691},
}

@inproceedings{icml2025_taming_45866,
  title = {{Taming Knowledge Conflicts in Language Models}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46677},
}

@inproceedings{icml2025_in_46659,
  title = {{In-Context Denoising with One-Layer Transformers: Connections between Attention and Associative Memory Retrieval}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/45913},
}

@inproceedings{icml2025_kv_47001,
  title = {{KV Shifting Attention Enhances Language Modeling}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/45582},
}