% Key papers for Transformer block and depth design (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{iclr2020_reducing_29722,
  title = {{Reducing Transformer Depth on Demand with Structured Dropout}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=SylO2yStDr},
}

@inproceedings{cvpr2023_generalized_21067,
  title = {{Generalized Relation Modeling for Transformer Tracking}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2023},
  url = {https://cvpr.thecvf.com/virtual/2023/poster/22874},
}

@inproceedings{icml2020_stabilizing_38784,
  title = {{Stabilizing Transformers for Reinforcement Learning}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2020},
  url = {https://icml.cc/virtual/2020/poster/6176},
}

@inproceedings{cvpr2020_sketchformer_16255,
  title = {{Sketchformer: Transformer-Based Representation for Sketched Structure}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2020},
}

@inproceedings{cvpr2022_reduce_19691,
  title = {{Reduce Information Loss in Transformers for Pluralistic Image Inpainting}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2022},
}

@inproceedings{iclr2020_transformer_29740,
  title = {{Transformer-XH: Multi-Evidence Reasoning with eXtra Hop Attention}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=r1eIiCNYwS},
}

@inproceedings{iclr2020_depth_29625,
  title = {{Depth-Adaptive Transformer}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=SJg7KhVKPH},
}

@inproceedings{neurips2022_data_65839,
  title = {{Data Distributional Properties Drive Emergent In-Context Learning in Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/54758},
}

@inproceedings{icml2020_improving_38308,
  title = {{Improving Transformer Optimization Through Better Initialization}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2020},
  url = {https://icml.cc/virtual/2020/poster/6684},
}

@inproceedings{neurips2021_hash_63060,
  title = {{Hash Layers For Large Sparse Models}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/28763},
}

@inproceedings{iclr2021_delight_30712,
  title = {{DeLighT: Deep and Light-weight Transformer}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2021},
  url = {https://iclr.cc/virtual/2021/poster/2739},
}

@inproceedings{aaai2022_scaling_2700,
  title = {{Scaling Up Influence Functions}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2022},
  url = {https://aaai.org/papers/08179-scaling-up-influence-functions/},
}

@inproceedings{neurips2023_mixformerv2_67055,
  title = {{MixFormerV2: Efficient Fully Transformer Tracking}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/72644},
}

@inproceedings{aaai2021_an_123,
  title = {{An Efficient Transformer Decoder with Compressed Sub-layers}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2021},
  url = {https://aaai.org/papers/13315-an-efficient-transformer-decoder-with-compressed-sub-layers/},
}

@inproceedings{aaai2021_faster_606,
  title = {{Faster Depth-Adaptive Transformers}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2021},
  url = {https://aaai.org/papers/13424-faster-depth-adaptive-transformers/},
}

@inproceedings{icml2022_xai_41360,
  title = {{XAI for Transformers: Better Explanations through Conservative Propagation}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2022},
  url = {https://icml.cc/virtual/2022/poster/17871},
}

@inproceedings{neurips2021_integrating_61406,
  title = {{Integrating Tree Path in Transformer for Code Representation}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/26141},
}

@inproceedings{icml2020_learning_38406,
  title = {{Learning to Encode Position for Transformer with Continuous Dynamical Model}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2020},
  url = {https://icml.cc/virtual/2020/poster/5906},
}

@inproceedings{neurips2020_deep_59668,
  title = {{Deep Transformers with Latent Depth}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2020},
  url = {https://nips.cc/virtual/2020/poster/18951},
}

@inproceedings{cvpr2023_svgformer_22119,
  title = {{SVGformer: Representation Learning for Continuous Vector Graphics Using Transformers}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2023},
  url = {https://cvpr.thecvf.com/virtual/2023/poster/21584},
}

@inproceedings{iclr2025_kolmogorov_81186,
  title = {{Kolmogorov-Arnold Transformer}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/30584},
}

@inproceedings{iclr2024_simplifying_34446,
  title = {{Simplifying Transformer Blocks}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18629},
}

@inproceedings{neurips2024_learning_72791,
  title = {{Learning-to-Cache: Accelerating Diffusion Transformer via Layer Caching}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94605},
}

@inproceedings{neurips2024_fasterdit_73043,
  title = {{FasterDiT: Towards Faster Diffusion Transformers Training without Architecture Modification}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94380},
}

@inproceedings{aaai2025_transformer_10915,
  title = {{TRANSFORMER EXPLAINER: Interactive Learning of Text-Generative Models}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/35347},
}

@inproceedings{iclr2024_xformer_35541,
  title = {{Xformer: Hybrid X-Shaped Transformer for Image Denoising}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/17532},
}

@inproceedings{neurips2024_don_72893,
  title = {{Don't Look Twice: Faster Video Transformers with Run-Length Tokenization}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94514},
}

@inproceedings{neurips2024_kraken_73515,
  title = {{Kraken: Inherently Parallel Transformers For Efficient Multi-Device Inference}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/93961},
}

@inproceedings{cvpr2025_layer_26726,
  title = {{Layer- and Timestep-Adaptive Differentiable Token Compression Ratios for Efficient Diffusion Transformers}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2025},
  url = {https://cvpr.thecvf.com/virtual/2025/poster/34258},
}

@inproceedings{neurips2024_nimbus_71321,
  title = {{Nimbus: Secure and Efficient Two-Party Inference for Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95926},
}

@inproceedings{aaai2025_adaptive_8024,
  title = {{Adaptive Computation Modules: Granular Conditional Computation for Efficient Inference}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/35453},
}

@inproceedings{neurips2024_protransformer_72408,
  title = {{ProTransformer: Robustify Transformers via Plug-and-Play Paradigm}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94947},
}

@inproceedings{neurips2025_unleashing_79698,
  title = {{Unleashing Diffusion Transformers for Visual Correspondence by Modulating Massive Activations}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=s3MwCBuqav},
}

@inproceedings{iclr2024_the_34299,
  title = {{The Expressive Power of Transformers with Chain of Thought}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18776},
}

@inproceedings{icml2024_learning_44079,
  title = {{Learning Cognitive Maps from Transformer Representations for Efficient Planning in Partially Observed Environments}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34373},
}

@inproceedings{neurips2024_transformers_70554,
  title = {{Transformers on Markov data: Constant depth suffices}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96560},
}

@inproceedings{neurips2024_unveil_71256,
  title = {{Unveil Benign Overfitting for Transformer in Vision: Training Dynamics, Convergence, and Generalization}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95987},
}

@inproceedings{neurips2024_non_71879,
  title = {{Non-asymptotic Convergence of Training Transformers for Next-token Prediction}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95424},
}

@inproceedings{neurips2025_depass_78663,
  title = {{DePass: Unified Feature Attributing by Simple Decomposed Forward Pass}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=h0aA8xpL4U},
}

@inproceedings{aaai2024_memory_6623,
  title = {{Memory-Augmenting Decoder-Only Language Models through Encoders (Student Abstract)}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/30444},
}