% Key papers for Sparse efficient transformers for long context (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{aaai2021_nystr_1075,
  title = {{Nyströmformer: A Nyström-based Algorithm for Approximating Self-Attention}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2021},
  url = {https://aaai.org/papers/14138-nystromformer-a-nystrom-based-algorithm-for-approximating-self-attention/},
}

@inproceedings{iclr2020_reformer_29836,
  title = {{Reformer: The Efficient Transformer}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=rkgNKkHtvB},
}

@inproceedings{iclr2019_music_29145,
  title = {{Music Transformer: Generating Music with Long-Term Structure}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2019},
  url = {https://iclr.cc/virtual/2019/poster/1008},
}

@inproceedings{icml2020_transformers_38887,
  title = {{Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2020},
  url = {https://icml.cc/virtual/2020/poster/6257},
}

@inproceedings{neurips2020_big_59467,
  title = {{Big Bird: Transformers for Longer Sequences}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2020},
  url = {https://nips.cc/virtual/2020/poster/18916},
}

@inproceedings{iclr2021_long_30649,
  title = {{Long Range Arena : A Benchmark for Efficient Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2021},
  url = {https://iclr.cc/virtual/2021/poster/3225},
}

@inproceedings{iclr2024_flashattention_35184,
  title = {{FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/17889},
}

@inproceedings{neurips2020_cogltx_59542,
  title = {{CogLTX: Applying BERT to Long Texts}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2020},
  url = {https://nips.cc/virtual/2020/poster/17152},
}

@inproceedings{icml2023_hyena_41427,
  title = {{Hyena Hierarchy: Towards Larger Convolutional Language Models}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2023},
  url = {https://icml.cc/virtual/2023/poster/24143},
}

@inproceedings{neurips2021_long_62020,
  title = {{Long-Short Transformer: Efficient Transformers for Language and Vision}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/26816},
}

@inproceedings{iclr2020_compressive_29720,
  title = {{Compressive Transformers for Long-Range Sequence Modelling}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=SylKikSYDH},
}

@inproceedings{icml2022_transformer_41290,
  title = {{Transformer Quality in Linear Time}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2022},
  url = {https://icml.cc/virtual/2022/poster/17273},
}

@inproceedings{iclr2022_train_31269,
  title = {{Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2022},
  url = {https://iclr.cc/virtual/2022/poster/6261},
}

@inproceedings{neurips2021_combiner_62008,
  title = {{Combiner: Full Attention Transformer with Sparse Computation Cost}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2021},
  url = {https://nips.cc/virtual/2021/poster/27023},
}

@inproceedings{neurips2022_block_66282,
  title = {{Block-Recurrent Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/54463},
}

@inproceedings{neurips2022_recurrent_65082,
  title = {{Recurrent Memory Transformer}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/52983},
}

@inproceedings{neurips2023_unlimiformer_69299,
  title = {{Unlimiformer: Long-Range Transformers with Unlimited Length Input}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/70601},
}

@inproceedings{iclr2025_vision_82207,
  title = {{Vision-LSTM: xLSTM as Generic Vision Backbone}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/29584},
}

@inproceedings{iclr2024_ringattention_34611,
  title = {{RingAttention with Blockwise Transformers for Near-Infinite Context}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18463},
}

@inproceedings{neurips2023_megabyte_67693,
  title = {{MEGABYTE: Predicting Million-byte Sequences with Multiscale Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/72057},
}

@inproceedings{aaai2024_beyond_5281,
  title = {{Beyond Attention: Breaking the Limits of Transformer Context Length with Recurrent Memory}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/29722},
}

@inproceedings{neurips2024_parallelizing_74544,
  title = {{Parallelizing Linear Transformers with the Delta Rule over Sequence Length}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/93040},
}

@inproceedings{neurips2024_approximation_72793,
  title = {{Approximation Rate of the Transformer Architecture for Sequence Modeling}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94603},
}

@inproceedings{aaai2024_proxyformer_7026,
  title = {{Proxyformer: Nyström-Based Linear Transformer with Trainable Proxy Tokens}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2024},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/29244},
}

@inproceedings{neurips2024_transformers_71260,
  title = {{Transformers to SSMs: Distilling Quadratic Knowledge to Subquadratic Models}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95984},
}

@inproceedings{iclr2024_lmuformer_35245,
  title = {{LMUFormer: Low Complexity Yet Powerful Spiking Model With Legendre Memory Units}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/17828},
}

@inproceedings{icml2025_a_46862,
  title = {{A Large Recurrent Action Model: xLSTM enables Fast Inference for Robotics Tasks}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/45715},
}

@inproceedings{iclr2024_flashfftconv_34930,
  title = {{FlashFFTConv: Efficient Convolutions for Long Sequences with Tensor Cores}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/18144},
}

@inproceedings{icml2025_dsp_47871,
  title = {{DSP: Dynamic Sequence Parallelism for Multi-Dimensional Transformers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/44732},
}

@inproceedings{neurips2024_understanding_70178,
  title = {{Understanding the Expressive Power and Mechanisms of Transformer for Sequence Modeling}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96906},
}

@inproceedings{neurips2024_mini_70265,
  title = {{Mini-Sequence Transformers: Optimizing Intermediate Memory for Long Sequences Training}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96824},
}

@inproceedings{neurips2025_two_77792,
  title = {{Two Heads are Better than One: Simulating Large Transformers with Small Ones}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=Xeb2EYBKkr},
}

@inproceedings{iclr2025_radar_82579,
  title = {{Radar: Fast Long-Context Decoding for Any Transformer}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/29218},
}

@inproceedings{iclr2024_iceformer_33686,
  title = {{IceFormer: Accelerated Inference with Long-Sequence Transformers on CPUs}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/19389},
}

@inproceedings{icml2024_leapformer_44642,
  title = {{LeaPformer: Enabling Linear Transformers for Autoregressive and Simultaneous Tasks via Learned Proportions}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33810},
}

@inproceedings{icml2025_context_46476,
  title = {{Context Matters: Query-aware Dynamic Long Sequence Modeling of Gigapixel Images}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46090},
}

@inproceedings{icml2025_curse_46512,
  title = {{Curse of High Dimensionality Issue in Transformer for Long Context Modeling}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46056},
}