% Key papers for Attention head and block redesign (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{neurips2021_transformer_62928,
  title = {{Transformer in Transformer}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/28439},
}

@inproceedings{cvpr2022_vision_20184,
  title = {{Vision Transformer With Deformable Attention}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2022},
}

@inproceedings{cvpr2022_mixformer_19344,
  title = {{MixFormer: End-to-End Tracking With Iterative Mixed Attention}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2022},
}

@inproceedings{cvpr2021_transformer_18047,
  title = {{Transformer Interpretability Beyond Attention Visualization}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2021},
}

@inproceedings{aaai2021_self_1350,
  title = {{Self-Attention Attribution: Interpreting Information Interactions Inside Transformer}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2021},
  url = {https://aaai.org/papers/12963-self-attention-attribution-interpreting-information-interactions-inside-transformer/},
}

@inproceedings{iclr2020_poly_29703,
  title = {{Poly-encoders: Architectures and Pre-training Strategies for Fast and Accurate Multi-sentence Scoring}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=SkxgnnNFvH},
}

@inproceedings{cvpr2022_lite_19260,
  title = {{Lite Vision Transformer With Enhanced Self-Attention}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2022},
}

@inproceedings{cvpr2023_slide_22222,
  title = {{Slide-Transformer: Hierarchical Vision Transformer With Local Self-Attention}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2023},
  url = {https://cvpr.thecvf.com/virtual/2023/poster/21912},
}

@inproceedings{neurips2019_a_57843,
  title = {{A Tensorized Transformer for Language Modeling}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2019},
  url = {https://nips.cc/virtual/2019/poster/13422},
}

@inproceedings{iclr2020_on_29286,
  title = {{On Identifiability in Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=BJg1f6EFDB},
}

@inproceedings{aaai2023_demt_3411,
  title = {{DeMT: Deformable Mixer Transformer for Multi-Task Learning of Dense Prediction}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2023},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/25411},
}

@inproceedings{neurips2021_soft_63283,
  title = {{SOFT: Softmax-free Transformer with Linear Complexity}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/28110},
}

@inproceedings{neurips2021_choose_63331,
  title = {{Choose a Transformer: Fourier or Galerkin}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/27535},
}

@inproceedings{icml2022_flowformer_40579,
  title = {{Flowformer: Linearizing Transformers with Conservation Flows}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2022},
  url = {https://icml.cc/virtual/2022/poster/17257},
}

@inproceedings{neurips2021_learnable_62200,
  title = {{Learnable Fourier Features for Multi-dimensional Spatial Positional Encoding}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/25950},
}

@inproceedings{icml2021_linear_39565,
  title = {{Linear Transformers Are Secretly Fast Weight Programmers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2021},
  url = {https://icml.cc/virtual/2021/poster/10587},
}

@inproceedings{cvpr2024_unifying_25141,
  title = {{Unifying Top-down and Bottom-up Scanpath Prediction Using Transformers}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2024},
  url = {https://cvpr.thecvf.com/virtual/2024/poster/29719},
}

@inproceedings{cvpr2024_mlp_24065,
  title = {{MLP Can Be A Good Transformer Learner}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2024},
  url = {https://cvpr.thecvf.com/virtual/2024/poster/29486},
}

@inproceedings{aaai2023_paraformer_4344,
  title = {{ParaFormer: Parallel Attention Transformer for Efficient Feature Matching}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2023},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/25275},
}

@inproceedings{cvpr2025_linear_26799,
  title = {{Linear Attention Modeling for Learned Image Compression}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2025},
  url = {https://cvpr.thecvf.com/virtual/2025/poster/33192},
}

@inproceedings{aaai2024_rethinking_7127,
  title = {{Rethinking Attention: Exploring Shallow Feed-Forward Neural Networks as an Alternative to Attention Layers in Transformers (Student Abstract)}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/30436},
}

@inproceedings{cvpr2025_breaking_25614,
  title = {{Breaking the Low-Rank Dilemma of Linear Attention}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2025},
  url = {https://cvpr.thecvf.com/virtual/2025/poster/33147},
}

@inproceedings{neurips2024_ditfastattn_70489,
  title = {{DiTFastAttn: Attention Compression for Diffusion Transformer Models}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96619},
}

@inproceedings{aaai2024_setformer_7276,
  title = {{SeTformer Is What You Need for Vision and Language}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/28272},
}

@inproceedings{iclr2025_token_83300,
  title = {{Token Statistics Transformer: Linear-Time Attention via Variational Rate Reduction}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/28513},
}

@inproceedings{neurips2024_selective_72104,
  title = {{Selective Attention: Enhancing Transformer through Principled Context Control}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95224},
}

@inproceedings{neurips2025_faster_77627,
  title = {{Faster Video Diffusion with Trainable Sparse Attention}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=VrYCLQ5inI},
}

@inproceedings{iclr2025_polaformer_83217,
  title = {{PolaFormer: Polarity-aware Linear Attention for Vision Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/28592},
}

@inproceedings{cvpr2024_mean_24128,
  title = {{Mean-Shift Feature Transformer}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2024},
  url = {https://cvpr.thecvf.com/virtual/2024/poster/31128},
}

@inproceedings{cvpr2025_dic_25920,
  title = {{DiC: Rethinking Conv3x3 Designs in Diffusion Models}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2025},
  url = {https://cvpr.thecvf.com/virtual/2025/poster/34504},
}

@inproceedings{neurips2024_infinite_73922,
  title = {{Infinite Limits of Multi-head Transformer Dynamics}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/93589},
}

@inproceedings{aaai2024_cached_5366,
  title = {{Cached Transformers: Improving Transformers with Differentiable Memory Cachde}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/29636},
}

@inproceedings{aaai2024_target_7495,
  title = {{Target Focused Shallow Transformer Framework for Efficient Visual Tracking}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/30405},
}

@inproceedings{neurips2024_graph_73244,
  title = {{Graph Convolutions Enrich the Self-Attention in Transformers!}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94193},
}

@inproceedings{cvpr2025_associative_25518,
  title = {{Associative Transformer}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2025},
  url = {https://cvpr.thecvf.com/virtual/2025/poster/35076},
}

@inproceedings{iclr2024_linear_33661,
  title = {{Linear Log-Normal Attention with Unbiased Concentration}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/19414},
}

@inproceedings{icml2024_outlier_45191,
  title = {{Outlier-Efficient Hopfield Layers for Large Transformer-Based Models}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33261},
}