Re-packed weights of Qwen/Qwen3-235B-A22B-Instruct-2507 using Charles Goddard's remote code implementation of scattermoe, including scripts to convert to and from standard Qwen3MoeForCausalLM. Thank you to intervitens for assistance with memory-efficient conversion scripts!
This is intended to be used as a drop-in replacement for efficient training using any transformers-based training repository.
Optional monkeypatches included for Liger Kernel and Cut Cross-Entropy. Simply rename the relevant modeling file to modeling_qwen3_shared_moe.py.
@misc{qwen3technicalreport,
title={Qwen3 Technical Report},
author={Qwen Team},
year={2025},
eprint={2505.09388},
archivePrefix={arXiv},
primaryClass={cs.CL},
url={https://arxiv.org/abs/2505.09388},
}
@misc{tan2024scatteredmixtureofexpertsimplementation,
title={Scattered Mixture-of-Experts Implementation},
author={Shawn Tan and Yikang Shen and Rameswar Panda and Aaron Courville},
year={2024},
eprint={2403.08245},
archivePrefix={arXiv},
primaryClass={cs.LG},
url={https://arxiv.org/abs/2403.08245},
}
@misc{hsu2025ligerkernelefficienttriton,
title={Liger Kernel: Efficient Triton Kernels for LLM Training},
author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
year={2025},
eprint={2410.10989},
archivePrefix={arXiv},
primaryClass={cs.LG},
url={https://arxiv.org/abs/2410.10989},
}
@misc{wijmans2025cutlosseslargevocabularylanguage,
title={Cut Your Losses in Large-Vocabulary Language Models},
author={Erik Wijmans and Brody Huval and Alexander Hertzberg and Vladlen Koltun and Philipp Krähenbühl},
year={2025},
eprint={2411.09009},
archivePrefix={arXiv},
primaryClass={cs.LG},
url={https://arxiv.org/abs/2411.09009},
}
model-00011-of-00095.safetensors 5.0 GBmodel-00012-of-00095.safetensors 5.0 GBmodel-00013-of-00095.safetensors 5.0 GBmodel-00014-of-00095.safetensors 5.0 GBmodel-00015-of-00095.safetensors 5.0 GBmodel-00016-of-00095.safetensors 5.0 GBmodel-00017-of-00095.safetensors 5.0 GBmodel-00018-of-00095.safetensors 5.0 GBmodel-00019-of-00095.safetensors 5.0 GBmodel-00020-of-00095.safetensors 5.0 GBmodel-00021-of-00095.safetensors 5.0 GBmodel-00022-of-00095.safetensors 5.0 GBmodel-00023-of-00095.safetensors 5.0 GBmodel-00024-of-00095.safetensors 5.0 GBmodel-00025-of-00095.safetensors 5.0 GBmodel-00026-of-00095.safetensors 5.0 GBmodel-00027-of-00095.safetensors 5.0 GBmodel-00028-of-00095.safetensors 5.0 GBmodel-00029-of-00095.safetensors 5.0 GBmodel-00030-of-00095.safetensors 5.0 GBmodel-00031-of-00095.safetensors 5.0 GBmodel-00032-of-00095.safetensors 5.0 GBmodel-00033-of-00095.safetensors 5.0 GBmodel-00034-of-00095.safetensors 5.0 GBmodel-00035-of-00095.safetensors 5.0 GBmodel-00036-of-00095.safetensors 5.0 GBmodel-00037-of-00095.safetensors 5.0 GBmodel-00038-of-00095.safetensors 5.0 GBmodel-00039-of-00095.safetensors 5.0 GBmodel-00040-of-00095.safetensors 5.0 GBmodel-00041-of-00095.safetensors 5.0 GBmodel-00042-of-00095.safetensors 5.0 GBmodel-00043-of-00095.safetensors 5.0 GBmodel-00044-of-00095.safetensors 5.0 GBmodel-00045-of-00095.safetensors 5.0 GBmodel-00046-of-00095.safetensors 5.0 GBmodel-00047-of-00095.safetensors 5.0 GBmodel-00048-of-00095.safetensors 5.0 GBmodel-00049-of-00095.safetensors 5.0 GBmodel-00050-of-00095.safetensors 5.0 GBmodel-00051-of-00095.safetensors 5.0 GBmodel-00052-of-00095.safetensors 5.0 GBmodel-00053-of-00095.safetensors 5.0 GBmodel-00054-of-00095.safetensors 5.0 GBmodel-00055-of-00095.safetensors 5.0 GBmodel-00056-of-00095.safetensors 5.0 GBmodel-00057-of-00095.safetensors 5.0 GBmodel-00058-of-00095.safetensors 5.0 GBmodel-00059-of-00095.safetensors 5.0 GBmodel-00060-of-00095.safetensors 5.0 GBmodel-00061-of-00095.safetensors 5.0 GBmodel-00062-of-00095.safetensors 5.0 GBmodel-00063-of-00095.safetensors 5.0 GBmodel-00064-of-00095.safetensors 5.0 GBmodel-00065-of-00095.safetensors 5.0 GBmodel-00066-of-00095.safetensors 5.0 GBmodel-00067-of-00095.safetensors 5.0 GBmodel-00068-of-00095.safetensors 5.0 GBmodel-00069-of-00095.safetensors 5.0 GBmodel-00070-of-00095.safetensors 5.0 GBmodel-00071-of-00095.safetensors 5.0 GBmodel-00072-of-00095.safetensors 5.0 GBmodel-00073-of-00095.safetensors 5.0 GBmodel-00074-of-00095.safetensors 5.0 GBmodel-00075-of-00095.safetensors 5.0 GBmodel-00076-of-00095.safetensors 5.0 GBmodel-00077-of-00095.safetensors 5.0 GBmodel-00078-of-00095.safetensors 5.0 GBmodel-00079-of-00095.safetensors 5.0 GBmodel-00080-of-00095.safetensors 5.0 GBmodel-00081-of-00095.safetensors 5.0 GBmodel-00082-of-00095.safetensors 5.0 GBmodel-00083-of-00095.safetensors 5.0 GBmodel-00084-of-00095.safetensors 5.0 GBmodel-00085-of-00095.safetensors 5.0 GBmodel-00086-of-00095.safetensors 5.0 GBmodel-00087-of-00095.safetensors 5.0 GBmodel-00088-of-00095.safetensors 5.0 GBmodel-00089-of-00095.safetensors 5.0 GBmodel-00090-of-00095.safetensors 5.0 GBmodel-00091-of-00095.safetensors 5.0 GBmodel-00092-of-00095.safetensors 5.0 GBmodel-00093-of-00095.safetensors 5.0 GBmodel-00094-of-00095.safetensors 5.0 GBmodel-00002-of-00095.safetensors 5.0 GBmodel-00003-of-00095.safetensors 5.0 GBmodel-00004-of-00095.safetensors 5.0 GBmodel-00005-of-00095.safetensors 5.0 GBmodel-00006-of-00095.safetensors 5.0 GBmodel-00007-of-00095.safetensors 5.0 GBmodel-00008-of-00095.safetensors 5.0 GBmodel-00009-of-00095.safetensors 5.0 GBmodel-00010-of-00095.safetensors 5.0 GBmodel-00001-of-00095.safetensors 4.6 GBmodel-00095-of-00095.safetensors 2.9 GB有文件?提交种子。
提交种子