google/bigbird-roberta-base (Fine-Tuned for Medical QA)google/bigbird-roberta-base | Step | Training Loss | Validation Loss | Exact Match | F1 |
|---|---|---|---|---|
| 400 | No log | 200.938553 | 61.78 | 79.78 |
| 800 | No log | 200.406921 | 81.98 | 90.12 |
| 1200 | No log | 200.226486 | 90.38 | 94.86 |
| 1600 | No log | 200.168106 | 92.93 | 96.30 |
| 2000 | No log | 200.107483 | 95.59 | 97.74 |
| 2400 | No log | 200.063904 | 97.08 | 98.63 |
| 2800 | No log | 200.044708 | 97.91 | 98.99 |
Best checkpoint: Step 2800
@misc{zaheer2021big,
title={Big Bird: Transformers for Longer Sequences},
author={Manzil Zaheer and Guru Guruganesh and Avinava Dubey and Joshua Ainslie and Chris Alberti and Santiago Ontanon and Philip Pham and Anirudh Ravula and Qifan Wang and Li Yang and Amr Ahmed},
year={2021},
eprint={2007.14062},
archivePrefix={arXiv},
primaryClass={cs.LG}
}
@article{zaheer2020bigbird,
title={Big bird: Transformers for longer sequences},
author={Zaheer, Manzil and Guruganesh, Guru and Dubey, Avinava and Ainslie, Joshua and Alberti, Chris and Ontanon, Santiago and Pham, Philip and Ravula, Anirudh and Wang, Qifan and Yang, Li and others},
journal={Advances in Neural Information Processing Systems},
volume={33},
year={2020}
}