% Key papers for Sparse MoE as regularization and attention (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{iclr2021_gshard_30658,
  title = {{GShard: Scaling Giant Models with Conditional Computation and Automatic Sharding}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2021},
  url = {https://iclr.cc/virtual/2021/poster/3196},
}

@inproceedings{iclr2017_outrageously_28205,
  title = {{Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2017},
  url = {https://openreview.net/forum?id=B1ckMDqlg},
}

@inproceedings{cvpr2023_mod_21585,
  title = {{Mod-Squad: Designing Mixtures of Experts As Modular Multi-Task Learners}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2023},
  url = {https://cvpr.thecvf.com/virtual/2023/poster/21853},
}

@inproceedings{cvpr2017_hard_12327,
  title = {{Hard Mixtures of Experts for Large Scale Weakly Supervised Vision}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2017},
}

@inproceedings{aaai2022_go_2158,
  title = {{Go Wider Instead of Deeper}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2022},
  url = {https://aaai.org/papers/08779-go-wider-instead-of-deeper/},
}

@inproceedings{neurips2020_towards_60985,
  title = {{Towards Crowdsourced Training of Large Neural Networks using Decentralized Mixture-of-Experts}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2020},
  url = {https://nips.cc/virtual/2020/poster/16871},
}

@inproceedings{iclr2023_sparse_32624,
  title = {{Sparse Upcycling: Training Mixture-of-Experts from Dense Checkpoints}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2023},
  url = {https://iclr.cc/virtual/2023/poster/11064},
}

@inproceedings{icml2022_gating_40604,
  title = {{Gating Dropout: Communication-efficient Regularization for Sparsely Activated Transformers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2022},
  url = {https://icml.cc/virtual/2022/poster/17415},
}

@inproceedings{aaai2025_rethinking_10540,
  title = {{Rethinking U-Net: Task-Adaptive Mixture of Skip Connections for Enhanced Medical Image Segmentation}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/32627},
}

@inproceedings{neurips2024_switchhead_70720,
  title = {{SwitchHead: Accelerating Transformers with Mixture-of-Experts Attention}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96404},
}

@inproceedings{icml2022_collaboration_40339,
  title = {{Collaboration of Experts: Achieving 80\% Top-1 Accuracy on ImageNet with 100M FLOPs}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2022},
  url = {https://icml.cc/virtual/2022/poster/17451},
}

@inproceedings{iclr2023_sparse_33356,
  title = {{Sparse MoE as the New Dropout: Scaling Dense and Self-Slimmable Transformers}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2023},
  url = {https://iclr.cc/virtual/2023/poster/11542},
}

@inproceedings{neurips2022_spatial_64223,
  title = {{Spatial Mixture-of-Experts}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/54113},
}

@inproceedings{neurips2023_shiftaddvit_69132,
  title = {{ShiftAddViT: Mixture of Multiplication Primitives Towards Efficient Vision Transformer}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2023},
  url = {https://nips.cc/virtual/2023/poster/70751},
}

@inproceedings{aaai2025_bagging_8227,
  title = {{Bagging-Expert Network for Multi-Task Learning: A Depolarization Solution in Multi-Gate Mixture-of-Experts}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2025},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/34395},
}

@inproceedings{icml2024_mvmoe_45256,
  title = {{MVMoE: Multi-Task Vehicle Routing Solver with Mixture-of-Experts}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33196},
}

@inproceedings{icml2025_moh_48027,
  title = {{MoH: Multi-Head Attention as Mixture-of-Head Attention}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/44579},
}

@inproceedings{neurips2022_a_64195,
  title = {{A Theoretical View on Sparsely Activated Networks}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2022},
  url = {https://nips.cc/virtual/2022/poster/53046},
}

@inproceedings{neurips2024_how_70571,
  title = {{How does Architecture Influence the Base Capabilities of Pre-trained Language Models? A Case Study Based on FFN-Wider and MoE Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96546},
}

@inproceedings{neurips2024_moeut_72795,
  title = {{MoEUT: Mixture-of-Experts Universal Transformers}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94601},
}

@inproceedings{neurips2025_mamba_80473,
  title = {{Mamba Goes HoME: Hierarchical Soft Mixture-of-Experts for 3D Medical Image Segmentation}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=ztgYn0Uk94},
}

@inproceedings{cvpr2025_efficient_26090,
  title = {{Efficient Data Driven Mixture-of-Expert Extraction from Trained Networks}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2025},
  url = {https://cvpr.thecvf.com/virtual/2025/poster/32663},
}

@inproceedings{iclr2025_mixture_80643,
  title = {{Mixture of In-Context Prompters for Tabular PFNs}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/31124},
}

@inproceedings{icml2024_exploring_44120,
  title = {{Exploring the Benefit of Activation Sparsity in Pre-training}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34332},
}

@inproceedings{icml2025_expert_46340,
  title = {{Expert Race: A Flexible Routing Strategy for Scaling Diffusion Transformer with Mixture of Experts}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46221},
}

@inproceedings{icml2025_mastering_48686,
  title = {{Mastering Massive Multi-Task Reinforcement Learning via Mixture-of-Expert Decision Transformer}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/43936},
}

@inproceedings{icml2026_spannorm_50165,
  title = {{SpanNorm: Reconciling Training Stability and Performance in Deep Transformers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2026},
  url = {https://openreview.net/forum?id=9bLiqb6Vec},
}

@inproceedings{icml2026_sparkling_52078,
  title = {{SPARKLING: Balancing Signal Preservation and Symmetry Breaking for Width-Progressive Learning}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2026},
  url = {https://openreview.net/forum?id=SXJBFREkWi},
}

@inproceedings{neurips2024_spiking_72547,
  title = {{Spiking Transformer with Experts Mixture}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/94824},
}

@inproceedings{neurips2025_localist_74815,
  title = {{Localist Topographic Expert Routing: A Barrel Cortex-Inspired Modular Network for Sensorimotor Processing}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=1Y8MXuJlIY},
}

@inproceedings{neurips2025_mixture_76995,
  title = {{Mixture-of-Experts Operator Transformer for Large-Scale PDE Pre-Training}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=PNgG4H3q9D},
}

@inproceedings{neurips2025_brain_77123,
  title = {{Brain-Like Processing Pathways Form in Models With Heterogeneous Experts}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=Qm6ah1hpFA},
}

@inproceedings{neurips2025_on_77194,
  title = {{On Linear Mode Connectivity of Mixture-of-Experts Architectures}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=RF3miSqdXa},
}