% Key papers for Memory-efficient LLM optimizers (scime atlas, https://atlas.scime.ai). Authors are not included: add them from the paper pages.

@inproceedings{icml2017_neural_36307,
  title = {{Neural Optimizer Search with Reinforcement Learning}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2017},
  url = {https://icml.cc/virtual/2017/poster/842},
}

@inproceedings{cvpr2020_augment_15166,
  title = {{Augment Your Batch: Improving Generalization Through Instance Repetition}},
  booktitle = {IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  year = {2020},
}

@inproceedings{iclr2020_large_29730,
  title = {{Large Batch Optimization for Deep Learning: Training BERT in 76 minutes}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2020},
  url = {https://openreview.net/forum?id=Syx4wnEtvH},
}

@inproceedings{icml2018_a_36476,
  title = {{A Progressive Batching L-BFGS Method for Machine Learning}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2018},
  url = {https://icml.cc/virtual/2018/poster/2059},
}

@inproceedings{icml2017_learning_36253,
  title = {{Learning Gradient Descent: Better Generalization and Longer Horizons}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2017},
  url = {https://icml.cc/virtual/2017/poster/875},
}

@inproceedings{icml2017_efficient_36172,
  title = {{Efficient Orthogonal Parametrisation of Recurrent Neural Networks Using Householder Reflections}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2017},
  url = {https://icml.cc/virtual/2017/poster/670},
}

@inproceedings{icml2019_understanding_37833,
  title = {{Understanding and correcting pathologies in the training of learned optimizers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2019},
  url = {https://icml.cc/virtual/2019/poster/3868},
}

@inproceedings{iclr2024_sophia_33587,
  title = {{Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2024},
  url = {https://iclr.cc/virtual/2024/poster/19488},
}

@inproceedings{icml2020_training_38882,
  title = {{Training Neural Networks for and by Interpolation}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2020},
  url = {https://icml.cc/virtual/2020/poster/6207},
}

@inproceedings{neurips2019_memory_58586,
  title = {{Memory Efficient Adaptive Optimization}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2019},
  url = {https://nips.cc/virtual/2019/poster/14003},
}

@inproceedings{icml2019_compressing_37241,
  title = {{Compressing Gradient Optimizers via Count-Sketches}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2019},
  url = {https://icml.cc/virtual/2019/poster/3666},
}

@inproceedings{aaai2020_how_101059,
  title = {{How the Duration of the Learning Period Affects the Performance of Random Gradient Selection Hyper-Heuristics}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2020},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/5617},
}

@inproceedings{aaai2021_large_852,
  title = {{Large Batch Optimization for Deep Learning Using New Complete Layer-Wise Adaptive Rate Scaling}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2021},
  url = {https://aaai.org/papers/07883-large-batch-optimization-for-deep-learning-using-new-complete-layer-wise-adaptive-rate-scaling/},
}

@inproceedings{neurips2019_which_59201,
  title = {{Which Algorithmic Choices Matter at Which Batch Sizes?  Insights From a Noisy Quadratic Model}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2019},
  url = {https://nips.cc/virtual/2019/poster/13889},
}

@inproceedings{aaai2019_scalable_101560,
  title = {{Scalable and Efficient Pairwise Learning to Achieve Statistical Accuracy}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2019},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/4254},
}

@inproceedings{aaai2020_inefficiency_102084,
  title = {{Inefficiency of K-FAC for Large Batch Size Training}},
  booktitle = {AAAI Conference on Artificial Intelligence (AAAI)},
  year = {2020},
  url = {https://ojs.aaai.org/index.php/AAAI/article/view/5946},
}

@inproceedings{icml2020_history_38284,
  title = {{History-Gradient Aided Batch Size Adaptation for Variance Reduced Algorithms}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2020},
  url = {https://icml.cc/virtual/2020/poster/6310},
}

@inproceedings{neurips2020_learning_60205,
  title = {{Learning compositional functions via multiplicative weight updates}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2020},
  url = {https://nips.cc/virtual/2020/poster/18945},
}

@inproceedings{neurips2025_small_75169,
  title = {{Small Batch Size Training for Language Models: When Vanilla SGD Works, and Why Gradient Accumulation is Wasteful}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=52Ehpe0Lu5},
}

@inproceedings{iclr2021_understanding_30674,
  title = {{Understanding the effects of data parallelism and sparsity on neural network training}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2021},
  url = {https://iclr.cc/virtual/2021/poster/3069},
}

@inproceedings{neurips2024_memory_71983,
  title = {{Memory-Efficient LLM Training with Online Subspace Descent}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/95328},
}

@inproceedings{icml2024_scaling_43266,
  title = {{Scaling Exponents Across Parameterizations and Optimizers}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/35186},
}

@inproceedings{icml2024_online_44715,
  title = {{Online Learning and Information Exponents: The Importance of Batch size \& Time/Complexity Tradeoffs}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33737},
}

@inproceedings{icml2025_frugal_46431,
  title = {{FRUGAL: Memory-Efficient Optimization by Reducing State Overhead for Scalable Training}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46134},
}

@inproceedings{neurips2024_towards_70511,
  title = {{Towards Exact Gradient-based Training on Analog In-memory Computing}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2024},
  url = {https://neurips.cc/virtual/2024/poster/96597},
}

@inproceedings{neurips2025_pike_80214,
  title = {{PiKE: Adaptive Data Mixing for Large-Scale Multi-Task Learning Under Low Gradient Conflicts}},
  booktitle = {Advances in Neural Information Processing Systems (NeurIPS)},
  year = {2025},
  url = {https://openreview.net/forum?id=xNJenVNmzL},
}

@inproceedings{iclr2025_revisit_83502,
  title = {{Revisit Micro-batch Clipping: Adaptive Data Pruning via Gradient Manipulation}},
  booktitle = {International Conference on Learning Representations (ICLR)},
  year = {2025},
  url = {https://iclr.cc/virtual/2025/poster/28314},
}

@inproceedings{icml2024_when_43501,
  title = {{When Will Gradient Regularization Be Harmful?}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/34951},
}

@inproceedings{icml2024_subsampling_45027,
  title = {{Subsampling is not Magic: Why Large Batch Sizes Work for Differentially Private Stochastic Optimisation}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2024},
  url = {https://icml.cc/virtual/2024/poster/33425},
}

@inproceedings{icml2025_subspace_46059,
  title = {{Subspace Optimization for Large Language Models with Convergence Guarantees}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46498},
}

@inproceedings{icml2025_nesterov_46316,
  title = {{Nesterov Method for Asynchronous Pipeline Parallel Optimization}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46245},
}

@inproceedings{icml2025_swan_46502,
  title = {{SWAN: SGD with Normalization and Whitening Enables Stateless LLM Training}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/46065},
}

@inproceedings{icml2025_lora_46962,
  title = {{LoRA-One: One-Step Full Gradient Could Suffice for Fine-Tuning Large Language Models, Provably and Efficiently}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/45618},
}

@inproceedings{icml2025_mars_47108,
  title = {{MARS: Unleashing the Power of Variance Reduction for Training Large Models}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/45479},
}

@inproceedings{icml2025_one_47441,
  title = {{One-Step Generalization Ratio Guided Optimization for Domain Generalization}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/45152},
}

@inproceedings{icml2025_on_48622,
  title = {{On the Duality between Gradient Transformations and Adapters}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2025},
  url = {https://icml.cc/virtual/2025/poster/43999},
}

@inproceedings{icml2026_mitigating_49489,
  title = {{Mitigating Staleness in Asynchronous Pipeline Parallelism via Basis Rotation}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2026},
  url = {https://openreview.net/forum?id=31iky6wFu5},
}

@inproceedings{icml2026_gradpower_51021,
  title = {{GradPower: Powering Gradients for Faster Language Model Pre-Training}},
  booktitle = {International Conference on Machine Learning (ICML)},
  year = {2026},
  url = {https://openreview.net/forum?id=I9pDKCWYXk},
}