first commit

iclr-blogposts · fabianp · Mar 22, 2024 · Nov 25, 2023 · Nov 26, 2023 · Nov 26, 2023
commit 0263cddf1784230c9c514c2aab670f5dbed530c4
diff --git a/.tool-versions b/.tool-versions
@@ -0,0 +1 @@
+ruby 3.0.5
diff --git a/_posts/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo.md b/_posts/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo.md
diff --git a/assets/bibliography/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo.bib b/assets/bibliography/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo.bib
@@ -0,0 +1,51 @@
+@article{Ziegler2019fine,
+  title={Fine-tuning language models from human preferences},
+  author={Ziegler, Daniel M and Stiennon, Nisan and Wu, Jeffrey and Brown, Tom B and Radford, Alec and Amodei, Dario and Christiano, Paul and Irving, Geoffrey},
+  journal={arXiv preprint arXiv:1909.08593},
+  year={2019}
+}
+
+@inproceedings{Shengyi2022the37implementation,
+  author = {Huang, Shengyi and Dossa, Rousslan Fernand Julien and Raffin, Antonin and Kanervisto, Anssi and Wang, Weixun},
+  title = {The 37 Implementation Details of Proximal Policy Optimization},
+  booktitle = {ICLR Blog Track},
+  year = {2022},
+  note = {https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/},
+  url  = {https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/}
+}
+
+@inproceedings{Wolf2020transformers,
+    title = "Transformers: State-of-the-Art Natural Language Processing",
+    author = "Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush",
+    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
+    month = oct,
+    year = "2020",
+    address = "Online",
+    publisher = "Association for Computational Linguistics",
+    url = "https://www.aclweb.org/anthology/2020.emnlp-demos.6",
+    pages = "38--45"
+}
+
+@article{Paszke2019pytorch,
+  title={Pytorch: An imperative style, high-performance deep learning library},
+  author={Paszke, Adam and Gross, Sam and Massa, Francisco and Lerer, Adam and Bradbury, James and Chanan, Gregory and Killeen, Trevor and Lin, Zeming and Gimelshein, Natalia and Antiga, Luca and others},
+  journal={Advances in neural information processing systems},
+  volume={32},
+  year={2019}
+}
+
+@software{Jax2018github,
+  author = {James Bradbury and Roy Frostig and Peter Hawkins and Matthew James Johnson and Chris Leary and Dougal Maclaurin and George Necula and Adam Paszke and Jake Vander{P}las and Skye Wanderman-{M}ilne and Qiao Zhang},
+  title = {{JAX}: composable transformations of {P}ython+{N}um{P}y programs},
+  url = {http://github.com/google/jax},
+  version = {0.3.13},
+  year = {2018},
+}
+
+@InProceedings{Kingma2015adam,
+  author    = {Kingma, Diederik and Ba, Jimmy},
+  booktitle = {International Conference on Learning Representations (ICLR)},
+  title     = {Adam: A Method for Stochastic Optimization},
+  year      = {2015},
+  address   = {San Diega, CA, USA},
+}
diff --git a/assets/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/adam_gpt2.png b/assets/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/adam_gpt2.png
diff --git a/...s/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/adam_gpt2_xl.png b/...s/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/adam_gpt2_xl.png
diff --git a/...img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/curve-matching.png b/...img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/curve-matching.png
diff --git a/...05-07-the-n-implementation-details-of-rlhf-with-ppo/descriptiveness-samples.png b/...05-07-the-n-implementation-details-of-rlhf-with-ppo/descriptiveness-samples.png
diff --git a/...-05-07-the-n-implementation-details-of-rlhf-with-ppo/norma_const_comparison.png b/...-05-07-the-n-implementation-details-of-rlhf-with-ppo/norma_const_comparison.png
diff --git a/assets/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/tldr1.png b/assets/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/tldr1.png
diff --git a/assets/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/tldr2.png b/assets/img/2024-05-07-the-n-implementation-details-of-rlhf-with-ppo/tldr2.png