--- base_model: - meta-llama/Meta-Llama-3-8B - meta-llama/Meta-Llama-3-8B-Instruct library_name: transformers tags: - mergekit - merge license: other license_name: llama3 license_link: LICENSE extra_gated_prompt: >- ### META LLAMA 3 COMMUNITY LICENSE AGREEMENT --- # Llama-3-DARE-v2-8B This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit). ## Merge Details ### Merge Method This model was merged using the [DARE](https://arxiv.org/abs/2311.03099) [TIES](https://arxiv.org/abs/2306.01708) merge method using [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) as a base. ### Models Merged The following models were included in the merge: * [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) ### Configuration The following YAML configuration was used to produce this model: ```yaml base_model: model: path: meta-llama/Meta-Llama-3-8B dtype: bfloat16 merge_method: dare_ties parameters: int8_mask: 1.0 normalize: 0.0 slices: - sources: - layer_range: [0, 32] model: model: path: meta-llama/Meta-Llama-3-8B-Instruct parameters: density: - filter: embed_token value: 0.1312973174759711 - filter: model.norm value: 0.9822573068643683 - filter: lm_head value: 0.9822573068643683 - filter: layers.0. value: 0.7585980531146381 - filter: layers.1. value: 0.561884676687423 - filter: layers.2. value: 0.9813332103529979 - filter: layers.3. value: 0.21930890320643737 - filter: layers.4. value: 0.8438595439416953 - filter: layers.5. value: 0.7505829508553786 - filter: layers.6. value: 0.25806652766624183 - filter: layers.7. value: 0.09440963517101368 - filter: layers.8. value: 0.7755855137764502 - filter: layers.9. value: 0.980153628060705 - filter: layers.10. value: 0.19005899180649255 - filter: layers.11. value: 0.8607970083829785 - filter: layers.12. value: 0.397155695730673 - filter: layers.13. value: 0.13006504372382097 - filter: layers.14. value: 0.09190339054787032 - filter: layers.15. value: 0.14507208971010788 - filter: layers.16. value: 0.8783567570968138 - filter: layers.17. value: 0.12105889524706802 - filter: layers.18. value: 0.8613122534393256 - filter: layers.19. value: 0.04810799757747608 - filter: layers.20. value: 0.5576780215726541 - filter: layers.21. value: 0.027665563208104402 - filter: layers.22. value: 0.7853866990645277 - filter: layers.23. value: 0.8674754718436442 - filter: layers.24. value: 0.915444005146838 - filter: layers.25. value: 0.8408012482479134 - filter: layers.26. value: 0.021388326989086454 - filter: layers.27. value: 0.8382317625625855 - filter: layers.28. value: 0.0418890647830244 - filter: layers.29. value: 0.16347844552378538 - filter: layers.30. value: 0.3152357116732385 - filter: layers.31. value: 0.24560833074353305 weight: - filter: embed_token value: 0.07729925635324639 - filter: model.norm. value: 0.8178953444711945 - filter: lm_head value: 0.8178953444711945 - filter: layers.0. value: 0.7744866238001816 - filter: layers.1. value: 0.9434402042495054 - filter: layers.2. value: 0.7277347873270702 - filter: layers.3. value: 0.7344542377948238 - filter: layers.4. value: 0.7789550773026275 - filter: layers.5. value: 0.9977070050825179 - filter: layers.6. value: 0.9790331299458723 - filter: layers.7. value: 0.936694814017198 - filter: layers.8. value: 0.8022245662093622 - filter: layers.9. value: 0.9207818987772527 - filter: layers.10. value: 0.9177346250309115 - filter: layers.11. value: 0.9705518504378707 - filter: layers.12. value: 0.6962192295710468 - filter: layers.13. value: 0.6601913931204128 - filter: layers.14. value: 0.7700302171860424 - filter: layers.15. value: 0.8764914331693985 - filter: layers.16. value: 0.1680474521036102 - filter: layers.17. value: 0.7252502118522168 - filter: layers.18. value: 0.974742254778171 - filter: layers.19. value: 0.6195931122778885 - filter: layers.20. value: 0.8808789662609544 - filter: layers.21. value: 0.10257515634421632 - filter: layers.22. value: 0.04739908873803297 - filter: layers.23. value: 0.9609916339993614 - filter: layers.24. value: 0.919873783591387 - filter: layers.25. value: 0.8920927224651306 - filter: layers.26. value: 0.3163445951176943 - filter: layers.27. value: 0.9183620719063076 - filter: layers.28. value: 0.7696116230696886 - filter: layers.29. value: 0.8092141535971384 - filter: layers.30. value: 0.9831761728133218 - filter: layers.31. value: 0.8368550180785097 - layer_range: [0, 32] model: model: path: meta-llama/Meta-Llama-3-8B parameters: density: - filter: embed_token value: 0.17966590577034688 - filter: model.norm value: 0.21103453748756532 - filter: lm_head value: 0.21103453748756532 - filter: layers.0. value: 0.1258111935267289 - filter: layers.1. value: 0.14712518700082466 - filter: layers.2. value: 0.15119918492525058 - filter: layers.3. value: 0.2786737466113946 - filter: layers.4. value: 0.7941421047966343 - filter: layers.5. value: 0.14996187005944175 - filter: layers.6. value: 0.21464495779281623 - filter: layers.7. value: 0.9544388237818261 - filter: layers.8. value: 0.6823542810137563 - filter: layers.9. value: 0.12285240879708 - filter: layers.10. value: 0.9560979622228943 - filter: layers.11. value: 0.18774045206190226 - filter: layers.12. value: 0.040667440606417166 - filter: layers.13. value: 0.06577308115912563 - filter: layers.14. value: 0.10898066770591856 - filter: layers.15. value: 0.7988665902205875 - filter: layers.16. value: 0.6902648842556923 - filter: layers.17. value: 0.21929644905849668 - filter: layers.18. value: 0.031818815520317414 - filter: layers.19. value: 0.09373597067550557 - filter: layers.20. value: 0.05663841107031176 - filter: layers.21. value: 0.573566646668988 - filter: layers.22. value: 0.2100277505470118 - filter: layers.23. value: 0.14741793214499152 - filter: layers.24. value: 0.13421247708500247 - filter: layers.25. value: 0.3923811614341299 - filter: layers.26. value: 0.7128080834275775 - filter: layers.27. value: 0.7173419548629656 - filter: layers.28. value: 0.25148835909796347 - filter: layers.29. value: 0.01041679056502079 - filter: layers.30. value: 0.04502964289907793 - filter: layers.31. value: 0.7933910983298705 weight: - filter: embed_token value: 0.1379551165348899 - filter: model.norm. value: 0.1145495237372664 - filter: lm_head value: 0.1145495237372664 - filter: layers.0. value: 0.058088970620673676 - filter: layers.1. value: 0.9812897914515345 - filter: layers.2. value: 0.7981160133946364 - filter: layers.3. value: 0.7634141926997863 - filter: layers.4. value: 0.16065704042361334 - filter: layers.5. value: 0.11985625529888592 - filter: layers.6. value: 0.12337476562239155 - filter: layers.7. value: 0.24780333206355964 - filter: layers.8. value: 0.031049663247481193 - filter: layers.9. value: 0.9399411230471199 - filter: layers.10. value: 0.059148520666240975 - filter: layers.11. value: 0.7231307420560733 - filter: layers.12. value: 0.8604431238597543 - filter: layers.13. value: 0.7874665970878372 - filter: layers.14. value: 0.18061875009463604 - filter: layers.15. value: 0.09518784175500823 - filter: layers.16. value: 0.27798323894923355 - filter: layers.17. value: 0.9413932817490506 - filter: layers.18. value: 0.2512810899514295 - filter: layers.19. value: 0.11709350712163674 - filter: layers.20. value: 0.6739674929530162 - filter: layers.21. value: 0.21208528089924455 - filter: layers.22. value: 0.811536887881144 - filter: layers.23. value: 0.6200821391879456 - filter: layers.24. value: 0.7663192062414923 - filter: layers.25. value: 0.8657672217068108 - filter: layers.26. value: 0.10630047742154969 - filter: layers.27. value: 0.9641883915469491 - filter: layers.28. value: 0.9807033575417018 - filter: layers.29. value: 0.04810198321663159 - filter: layers.30. value: 0.8249574979326807 - filter: layers.31. value: 0.01564957234872716 ```