% Key papers for Transformer expressivity and learnability (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{aaai2022_frozen_2133,
  title = {{Frozen Pretrained Transformers as Universal Computation Engines}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2022},
  url = {https://aaai.org/papers/07628-frozen-pretrained-transformers-as-universal-computation-engines/},
}

@inproceedings{aaai2023_graphix_3802,
  title = {{Graphix-T5: Mixing Pre-trained Transformers with Graph-Aware Layers for Text-to-SQL Parsing}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2023},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/26536},
}

@inproceedings{neurips2023_vanillanet_68058,
  title = {{VanillaNet: the Power of Minimalism in Deep Learning}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/71722},
}

@inproceedings{neurips2019_novel_58695,
  title = {{Novel positional encodings to enable tree-based transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2019},
  url = {https://nips.cc/virtual/2019/poster/14174},
}

@inproceedings{iclr2022_equivariant_31864,
  title = {{Equivariant Transformers for Neural Network based Molecular Potentials}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2022},
  url = {https://iclr.cc/virtual/2022/poster/6416},
}

@inproceedings{iclr2022_relating_30982,
  title = {{Relating transformers to models and neural representations of the hippocampal formation}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2022},
  url = {https://iclr.cc/virtual/2022/poster/6743},
}

@inproceedings{iclr2020_are_29389,
  title = {{Are Transformers universal approximators of sequence-to-sequence functions?}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=ByxRM0Ntvr},
}

@inproceedings{iclr2020_robustness_29328,
  title = {{Robustness Verification for Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=BJxwPJHFwS},
}

@inproceedings{neurips2023_the_67383,
  title = {{The Impact of Positional Encoding on Length Generalization in Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/72346},
}

@inproceedings{icml2021_relative_39870,
  title = {{Relative Positional Encoding for Transformers with Linear Complexity}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2021},
  url = {https://icml.cc/virtual/2021/poster/9021},
}

@inproceedings{cvpr2024_permutation_24452,
  title = {{Permutation Equivariance of Transformers and Its Applications}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2024},
  url = {https://cvpr.thecvf.com/virtual/2024/poster/31170},
}

@inproceedings{aaai2022_learning_2316,
  title = {{Learning Bounded Context-Free-Grammar via LSTM and the Transformer: Difference and the Explanations}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2022},
  url = {https://aaai.org/papers/08267-learning-bounded-context-free-grammar-via-lstm-and-the-transformer-difference-and-the-explanations/},
}

@inproceedings{iclr2023_learning_32867,
  title = {{Learning to Grow Pretrained Models for Efficient Transformer Training}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2023},
  url = {https://iclr.cc/virtual/2023/poster/11759},
}

@inproceedings{iclr2023_characterizing_33264,
  title = {{Characterizing intrinsic compositionality in transformers with Tree Projections}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2023},
  url = {https://iclr.cc/virtual/2023/poster/10816},
}

@inproceedings{neurips2023_learning_68026,
  title = {{Learning Transformer Programs}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/71753},
}

@inproceedings{neurips2022_your_64750,
  title = {{Your Transformer May Not be as Powerful as You Expect}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/55137},
}

@inproceedings{neurips2022_tokenmixup_65453,
  title = {{TokenMixup: Efficient Attention-guided Token-level Data Augmentation for Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/54623},
}

@inproceedings{iclr2023_transformers_32257,
  title = {{Transformers Learn Shortcuts to Automata}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2023},
  url = {https://iclr.cc/virtual/2023/poster/11118},
}

@inproceedings{neurips2023_quantizable_69676,
  title = {{Quantizable Transformers: Removing Outliers by Helping Attention Heads Do Nothing}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/70261},
}

@inproceedings{aaai2025_transformer_11103,
  title = {{Transformer Layers as Painters}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/34708},
}

@inproceedings{iclr2024_what_33839,
  title = {{What Algorithms can Transformers Learn? A Study in Length Generalization}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/19236},
}

@inproceedings{aaai2025_lazydit_9716,
  title = {{LazyDiT: Lazy Learning for the Acceleration of Diffusion Transformers}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/34248},
}

@inproceedings{icml2024_converting_43641,
  title = {{Converting Transformers to Polynomial Form for Secure Inference Over Homomorphic Encryption}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34811},
}

@inproceedings{neurips2024_transformers_72834,
  title = {{Transformers Can Do Arithmetic with the Right Embeddings}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94565},
}

@inproceedings{iclr2024_chain_33551,
  title = {{Chain of Thought Empowers Transformers to Solve Inherently Serial Problems}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/19524},
}

@inproceedings{icml2024_how_45139,
  title = {{How Transformers Learn Causal Structure with Gradient Descent}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33313},
}

@inproceedings{neurips2024_understanding_70965,
  title = {{Understanding Transformer Reasoning Capabilities via Graph Algorithms}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96239},
}

@inproceedings{iclr2024_circuit_34903,
  title = {{Circuit Component Reuse Across Tasks in Transformer Language Models}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18171},
}

@inproceedings{icml2024_eureka_44007,
  title = {{Eureka-Moments in Transformers: Multi-Step Tasks Reveal Softmax Induced Optimization Problems}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34445},
}

@inproceedings{icml2024_repeat_44925,
  title = {{Repeat After Me: Transformers are Better than State Space Models at Copying}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33527},
}

@inproceedings{neurips2024_masked_71249,
  title = {{Masked Hard-Attention Transformers Recognize Exactly the Star-Free Languages}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95994},
}

@inproceedings{aaai2024_exploring_5924,
  title = {{Exploring Transformer Extrapolation}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/29855},
}

@inproceedings{aaai2024_towards_7593,
  title = {{Towards Equipping Transformer with the Ability of Systematic Compositionality}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/29788},
}

@inproceedings{iclr2024_when_34472,
  title = {{When can transformers reason with abstract symbols?}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18602},
}

@inproceedings{iclr2024_logical_34933,
  title = {{Logical Languages Accepted by Transformer Encoders with Hard Attention}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18141},
}

@inproceedings{iclr2024_functional_35380,
  title = {{Functional Interpolation for Relative Positions improves Long Context Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/17693},
}

@inproceedings{icml2024_transformers_44356,
  title = {{Transformers, parallel computation, and logarithmic depth}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34096},
}

@inproceedings{icml2024_the_44377,
  title = {{The Illusion of State in State-Space Models}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34075},
}